autoencoder_averageValues.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "autoencoder_averageValues.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SQsrrdhN6GMB"
      },
      "source": [
        "# Import libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4RHnFs37CroL"
      },
      "source": [
        "import os\n",
        "import random\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from math import sqrt\n",
        "import math\n",
        "import os\n",
        "import random\n",
        "\n",
        "from sklearn.metrics import mean_squared_error\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.utils.data\n",
        "import torch.optim as optim"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VTg5htrjS620"
      },
      "source": [
        "!pip install autoimpute\n",
        "from autoimpute.imputations import MultipleImputer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0QVAgGyBFNzr"
      },
      "source": [
        "# Missingness method"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oiaPNpCjDLzj"
      },
      "source": [
        "def missing_method(test_data, train_data, num_embeddings) :\n",
        "    \n",
        "    # test data\n",
        "    test_data = test_data.copy()\n",
        "    test_rows, test_cols = test_data.shape\n",
        "    test_cols -= num_embeddings\n",
        "\n",
        "    # train data\n",
        "    train_data = train_data.copy()\n",
        "    train_rows, train_cols = train_data.shape\n",
        "    train_cols -= num_embeddings\n",
        "\n",
        "    # missingness threshold\n",
        "    t = 0.2\n",
        "\n",
        "    # uniform random vector, missing values where v<=t\n",
        "    # test data corruption\n",
        "    # embedding columns do not have any missing values\n",
        "    v = np.random.uniform(size=(test_rows, test_cols))\n",
        "    embeddings_mask = np.zeros((test_rows, num_embeddings), dtype=bool)\n",
        "    mask = (v<=t)\n",
        "    mask = np.c_[mask, embeddings_mask]\n",
        "    test_data[mask] = np.NAN\n",
        "\n",
        "    # train data corruption - this is used for training MultipleImputer for imputing mean/median values in the dataset\n",
        "    v_train = np.random.uniform(size=(train_rows, train_cols))\n",
        "    embeddings_mask = np.zeros((train_rows, num_embeddings), dtype=bool)\n",
        "    mask_train = (v_train <= t)\n",
        "    mask_train = np.c_[mask_train, embeddings_mask]\n",
        "    train_data[mask_train] = np.NAN\n",
        "        \n",
        "    return test_data, train_data, mask"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "81RxIikoPXt0"
      },
      "source": [
        "# Imputation Methods"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y2t6xbd8MXYF"
      },
      "source": [
        "## Autoencoder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yUtv4xx5MdXR"
      },
      "source": [
        "class Autoencoder(nn.Module):\n",
        "    def __init__(self, dim):\n",
        "        super(Autoencoder, self).__init__()\n",
        "        self.dim = dim\n",
        "        \n",
        "        self.drop_out = nn.Dropout(p=0.2)\n",
        "        \n",
        "        # encoder architecture\n",
        "        self.encoder = nn.Sequential(\n",
        "            nn.Linear(dim, int(dim*0.7)),\n",
        "            nn.Tanh(),\n",
        "            nn.Linear(int(dim*0.7), int(dim*0.5)),\n",
        "            nn.Tanh(),\n",
        "            nn.Linear(int(dim*0.5), int(dim*0.2))\n",
        "        )\n",
        "            \n",
        "        # decoder architecture\n",
        "        self.decoder = nn.Sequential(\n",
        "            nn.Linear(int(dim*0.2), int(dim*0.5)),\n",
        "            nn.Tanh(),\n",
        "            nn.Linear(int(dim*0.5), int(dim*0.7)),\n",
        "            nn.Tanh(),\n",
        "            nn.Linear(int(dim*0.7), dim)\n",
        "        )\n",
        "        \n",
        "    def forward(self, x):\n",
        "        x = x.view(-1, self.dim)\n",
        "\n",
        "        # adding dropout to introduce input corruption during training\n",
        "        x_missed = self.drop_out(x)\n",
        "        \n",
        "        z = self.encoder(x_missed)\n",
        "        out = self.decoder(z)\n",
        "        \n",
        "        out = out.view(-1, self.dim)\n",
        "        \n",
        "        return out"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Clzk4cPA61ep"
      },
      "source": [
        "def training(num_epochs, model, train_loader, criterion, optimizer):\n",
        "  for epoch in range(num_epochs):\n",
        "      loss = 0\n",
        "      for i, batch_features in enumerate(train_loader):\n",
        "          # load it to the active device\n",
        "          batch_features = batch_features.to(device)\n",
        "        \n",
        "          # reset the gradients back to zero\n",
        "          optimizer.zero_grad()\n",
        "        \n",
        "          # compute reconstructions\n",
        "          outputs = model(batch_features)\n",
        "        \n",
        "          # compute training reconstruction loss\n",
        "          train_loss = criterion(outputs, batch_features)\n",
        "        \n",
        "          # compute accumulated gradients\n",
        "          train_loss.backward()\n",
        "        \n",
        "          # perform parameter update based on current gradients\n",
        "          optimizer.step()\n",
        "        \n",
        "          # add the mini-batch training loss to epoch loss\n",
        "          loss += train_loss.item()\n",
        "    \n",
        "      # compute the epoch training loss\n",
        "      loss = loss / len(train_loader)\n",
        "    \n",
        "      #  display the epoch training loss\n",
        "      print(\"epoch : {}/{}, recon loss = {:.8f}\".format(epoch + 1, num_epochs, loss))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9mGXPjhx6_hT"
      },
      "source": [
        "## Mean and Median Imputation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tikiUrxd6-PY"
      },
      "source": [
        "# takes the test dataset and the dataset containing the average/median values by category\n",
        "# imputes the values found in the dataset by catogory in place of the NaN values in the test dataset\n",
        "# returns pandas DataFrame\n",
        "\n",
        "def impute_traditional_by_category(X_test, df, indices):\n",
        "  for i in indices:\n",
        "    temp1 = X_test.loc[[i]]\n",
        "    temp2 = df[df['fdc_id'] == i]\n",
        "    del temp2['fdc_id']\n",
        "    for col in temp1.columns:\n",
        "      if math.isnan(temp1[col].values[0]):\n",
        "        temp1[col] = temp2[col].values[0]\n",
        "    X_test.loc[[i]] = temp1\n",
        "\n",
        "  return X_test"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t2DkyZo4TIDi"
      },
      "source": [
        "# uses a multiple imputer to fill the NaN values in the test dataset, calculates mean/median by column\n",
        "# returns pandas DataFrame\n",
        "\n",
        "def impute_traditional_all_foods(train, test, method):\n",
        "  imputer = MultipleImputer(1, strategy=method, return_list=True)\n",
        "  imputer.fit(train)\n",
        "  data = imputer.transform(test)\n",
        "  return data[0][1]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YMAG40X48Qk2"
      },
      "source": [
        "# Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "trOiLwNF8P_L"
      },
      "source": [
        "def rmse_error(test_data, imputed_data, num_cols, mask):\n",
        "  rmse_sum = 0\n",
        "\n",
        "  for i in range(num_cols):\n",
        "    y_actual = test_data[:,i][mask[:,i]]\n",
        "    y_predicted = imputed_data[:,i][mask[:,i]]\n",
        "\n",
        "    rmse = sqrt(mean_squared_error(y_actual, y_predicted))\n",
        "    rmse_sum += rmse\n",
        "\n",
        "  return rmse_sum"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZhKAV25rDBgO"
      },
      "source": [
        "# Data Preparation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kXCoADDk2Qlu"
      },
      "source": [
        "num_epochs = 200\n",
        "test_size = 0.2\n",
        "use_cuda = False\n",
        "batch_size  = 1\n",
        "\n",
        "data_mean = 'datasets/average datasets/traditionalMethods/food_mean.csv'\n",
        "data_median = 'datasets/average datasets/traditionalMethods/food_median.csv'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 350
        },
        "id": "P2fuX35m71S5",
        "outputId": "03f869af-68c4-471e-ed7b-540e6fc149ae"
      },
      "source": [
        "df_mean = pd.read_csv(data_mean)\n",
        "del df_mean['food_category_id']\n",
        "del df_mean['Unnamed: 0']\n",
        "df_mean.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>fdc_id</th>\n",
              "      <th>1002</th>\n",
              "      <th>1003</th>\n",
              "      <th>1004</th>\n",
              "      <th>1005</th>\n",
              "      <th>1007</th>\n",
              "      <th>1008</th>\n",
              "      <th>1009</th>\n",
              "      <th>1010</th>\n",
              "      <th>1011</th>\n",
              "      <th>1012</th>\n",
              "      <th>1013</th>\n",
              "      <th>1014</th>\n",
              "      <th>1024</th>\n",
              "      <th>1032</th>\n",
              "      <th>1039</th>\n",
              "      <th>1050</th>\n",
              "      <th>1051</th>\n",
              "      <th>1062</th>\n",
              "      <th>1063</th>\n",
              "      <th>1075</th>\n",
              "      <th>1079</th>\n",
              "      <th>1082</th>\n",
              "      <th>1084</th>\n",
              "      <th>1085</th>\n",
              "      <th>1087</th>\n",
              "      <th>1089</th>\n",
              "      <th>1090</th>\n",
              "      <th>1091</th>\n",
              "      <th>1092</th>\n",
              "      <th>1093</th>\n",
              "      <th>1094</th>\n",
              "      <th>1095</th>\n",
              "      <th>1097</th>\n",
              "      <th>1098</th>\n",
              "      <th>1100</th>\n",
              "      <th>1101</th>\n",
              "      <th>1102</th>\n",
              "      <th>1103</th>\n",
              "      <th>1105</th>\n",
              "      <th>...</th>\n",
              "      <th>1323</th>\n",
              "      <th>1325</th>\n",
              "      <th>1329</th>\n",
              "      <th>1330</th>\n",
              "      <th>1331</th>\n",
              "      <th>1333</th>\n",
              "      <th>1334</th>\n",
              "      <th>1335</th>\n",
              "      <th>1404</th>\n",
              "      <th>1405</th>\n",
              "      <th>1406</th>\n",
              "      <th>1409</th>\n",
              "      <th>1411</th>\n",
              "      <th>1414</th>\n",
              "      <th>2003</th>\n",
              "      <th>2004</th>\n",
              "      <th>2005</th>\n",
              "      <th>2006</th>\n",
              "      <th>2007</th>\n",
              "      <th>2008</th>\n",
              "      <th>2009</th>\n",
              "      <th>2010</th>\n",
              "      <th>2012</th>\n",
              "      <th>2013</th>\n",
              "      <th>2014</th>\n",
              "      <th>2015</th>\n",
              "      <th>2016</th>\n",
              "      <th>2018</th>\n",
              "      <th>2019</th>\n",
              "      <th>2020</th>\n",
              "      <th>2021</th>\n",
              "      <th>2022</th>\n",
              "      <th>2023</th>\n",
              "      <th>2024</th>\n",
              "      <th>2025</th>\n",
              "      <th>2026</th>\n",
              "      <th>2028</th>\n",
              "      <th>2029</th>\n",
              "      <th>2032</th>\n",
              "      <th>2033</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>1.310500e+04</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>1.310500e+04</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>4.807517e+05</td>\n",
              "      <td>0.243665</td>\n",
              "      <td>1.942243</td>\n",
              "      <td>1.824697</td>\n",
              "      <td>0.164999</td>\n",
              "      <td>0.472742</td>\n",
              "      <td>1.999714</td>\n",
              "      <td>2.561062</td>\n",
              "      <td>0.147034</td>\n",
              "      <td>0.118187</td>\n",
              "      <td>0.120046</td>\n",
              "      <td>0.046356</td>\n",
              "      <td>0.008754</td>\n",
              "      <td>0.000476</td>\n",
              "      <td>0.681114</td>\n",
              "      <td>0.728577</td>\n",
              "      <td>0.049253</td>\n",
              "      <td>8.349769</td>\n",
              "      <td>8.369340</td>\n",
              "      <td>0.055411</td>\n",
              "      <td>0.002459</td>\n",
              "      <td>0.350355</td>\n",
              "      <td>0.005555</td>\n",
              "      <td>0.021892</td>\n",
              "      <td>0.106772</td>\n",
              "      <td>27.659061</td>\n",
              "      <td>0.476038</td>\n",
              "      <td>14.595597</td>\n",
              "      <td>55.746662</td>\n",
              "      <td>125.003892</td>\n",
              "      <td>133.394887</td>\n",
              "      <td>14.730179</td>\n",
              "      <td>0.384163</td>\n",
              "      <td>2.983579</td>\n",
              "      <td>0.077504</td>\n",
              "      <td>12.187135</td>\n",
              "      <td>0.149979</td>\n",
              "      <td>42.944411</td>\n",
              "      <td>0.840258</td>\n",
              "      <td>2.086684</td>\n",
              "      <td>...</td>\n",
              "      <td>0.002480</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.001246</td>\n",
              "      <td>0.000236</td>\n",
              "      <td>7.630675e-07</td>\n",
              "      <td>0.000002</td>\n",
              "      <td>0.000065</td>\n",
              "      <td>0.000017</td>\n",
              "      <td>0.016719</td>\n",
              "      <td>0.000279</td>\n",
              "      <td>0.001158</td>\n",
              "      <td>0.000021</td>\n",
              "      <td>0.000652</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000093</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000153</td>\n",
              "      <td>0.002787</td>\n",
              "      <td>0.000194</td>\n",
              "      <td>0.005167</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000170</td>\n",
              "      <td>0.000018</td>\n",
              "      <td>0.020393</td>\n",
              "      <td>0.001910</td>\n",
              "      <td>0.000021</td>\n",
              "      <td>0.000090</td>\n",
              "      <td>0.000004</td>\n",
              "      <td>0.000217</td>\n",
              "      <td>0.000025</td>\n",
              "      <td>0.000044</td>\n",
              "      <td>0.000039</td>\n",
              "      <td>0.000110</td>\n",
              "      <td>0.298283</td>\n",
              "      <td>0.004426</td>\n",
              "      <td>0.005723</td>\n",
              "      <td>0.022570</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.544018e+05</td>\n",
              "      <td>0.220034</td>\n",
              "      <td>5.131636</td>\n",
              "      <td>1.626531</td>\n",
              "      <td>0.445364</td>\n",
              "      <td>1.407237</td>\n",
              "      <td>2.322706</td>\n",
              "      <td>7.750389</td>\n",
              "      <td>1.676538</td>\n",
              "      <td>0.170909</td>\n",
              "      <td>0.170768</td>\n",
              "      <td>0.066607</td>\n",
              "      <td>0.022466</td>\n",
              "      <td>0.001082</td>\n",
              "      <td>1.547442</td>\n",
              "      <td>1.655274</td>\n",
              "      <td>0.088366</td>\n",
              "      <td>6.015826</td>\n",
              "      <td>9.725673</td>\n",
              "      <td>0.336683</td>\n",
              "      <td>0.003077</td>\n",
              "      <td>0.866652</td>\n",
              "      <td>0.009656</td>\n",
              "      <td>0.039604</td>\n",
              "      <td>0.268325</td>\n",
              "      <td>36.719285</td>\n",
              "      <td>1.102398</td>\n",
              "      <td>36.788753</td>\n",
              "      <td>104.198028</td>\n",
              "      <td>312.532122</td>\n",
              "      <td>574.034787</td>\n",
              "      <td>46.850594</td>\n",
              "      <td>0.685919</td>\n",
              "      <td>9.489527</td>\n",
              "      <td>0.216968</td>\n",
              "      <td>77.766980</td>\n",
              "      <td>0.408316</td>\n",
              "      <td>135.856952</td>\n",
              "      <td>0.727050</td>\n",
              "      <td>2.920551</td>\n",
              "      <td>...</td>\n",
              "      <td>0.004219</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.002653</td>\n",
              "      <td>0.000311</td>\n",
              "      <td>2.094377e-06</td>\n",
              "      <td>0.000006</td>\n",
              "      <td>0.000102</td>\n",
              "      <td>0.000024</td>\n",
              "      <td>0.062646</td>\n",
              "      <td>0.000978</td>\n",
              "      <td>0.001561</td>\n",
              "      <td>0.000030</td>\n",
              "      <td>0.001585</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000410</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000214</td>\n",
              "      <td>0.003980</td>\n",
              "      <td>0.000502</td>\n",
              "      <td>0.017287</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000669</td>\n",
              "      <td>0.000097</td>\n",
              "      <td>0.073219</td>\n",
              "      <td>0.008556</td>\n",
              "      <td>0.000029</td>\n",
              "      <td>0.000180</td>\n",
              "      <td>0.000018</td>\n",
              "      <td>0.000493</td>\n",
              "      <td>0.000062</td>\n",
              "      <td>0.000093</td>\n",
              "      <td>0.000208</td>\n",
              "      <td>0.000393</td>\n",
              "      <td>0.359319</td>\n",
              "      <td>0.006378</td>\n",
              "      <td>0.008247</td>\n",
              "      <td>0.037044</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>3.198770e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.111598</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000945</td>\n",
              "      <td>0.259574</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000787</td>\n",
              "      <td>1.080851</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.019685</td>\n",
              "      <td>0.001024</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000315</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>3.256480e+05</td>\n",
              "      <td>0.039103</td>\n",
              "      <td>0.032765</td>\n",
              "      <td>0.213767</td>\n",
              "      <td>0.029723</td>\n",
              "      <td>0.126017</td>\n",
              "      <td>0.852374</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.014362</td>\n",
              "      <td>0.002979</td>\n",
              "      <td>0.001510</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.002864</td>\n",
              "      <td>7.083392</td>\n",
              "      <td>3.566526</td>\n",
              "      <td>0.009194</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000399</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.011009</td>\n",
              "      <td>2.582981</td>\n",
              "      <td>0.089350</td>\n",
              "      <td>1.771225</td>\n",
              "      <td>5.056801</td>\n",
              "      <td>19.483948</td>\n",
              "      <td>11.090738</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.034903</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.002225</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.001896</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.101719</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000156</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000007</td>\n",
              "      <td>0.000024</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.004215</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000016</td>\n",
              "      <td>0.000145</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.001804</td>\n",
              "      <td>0.000368</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000002</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>3.314100e+05</td>\n",
              "      <td>0.228130</td>\n",
              "      <td>0.153873</td>\n",
              "      <td>2.072731</td>\n",
              "      <td>0.050521</td>\n",
              "      <td>0.268274</td>\n",
              "      <td>1.537129</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.014362</td>\n",
              "      <td>0.002979</td>\n",
              "      <td>0.005394</td>\n",
              "      <td>0.000066</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.002864</td>\n",
              "      <td>7.640763</td>\n",
              "      <td>6.428218</td>\n",
              "      <td>0.016087</td>\n",
              "      <td>0.001852</td>\n",
              "      <td>0.000399</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.054283</td>\n",
              "      <td>6.659193</td>\n",
              "      <td>0.113892</td>\n",
              "      <td>2.494296</td>\n",
              "      <td>27.334123</td>\n",
              "      <td>19.621362</td>\n",
              "      <td>66.594918</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.246153</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.006796</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.008731</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.210188</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.001174</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000290</td>\n",
              "      <td>0.000062</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000057</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.006797</td>\n",
              "      <td>0.000070</td>\n",
              "      <td>0.000115</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000152</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000007</td>\n",
              "      <td>0.000054</td>\n",
              "      <td>0.000988</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000008</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.002971</td>\n",
              "      <td>0.000368</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000087</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000125</td>\n",
              "      <td>0.000017</td>\n",
              "      <td>0.000047</td>\n",
              "      <td>0.000003</td>\n",
              "      <td>0.000019</td>\n",
              "      <td>0.143865</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>7.474680e+05</td>\n",
              "      <td>0.347477</td>\n",
              "      <td>0.170609</td>\n",
              "      <td>2.739955</td>\n",
              "      <td>0.197184</td>\n",
              "      <td>0.377012</td>\n",
              "      <td>1.609859</td>\n",
              "      <td>0.031965</td>\n",
              "      <td>0.130000</td>\n",
              "      <td>0.280000</td>\n",
              "      <td>0.344147</td>\n",
              "      <td>0.142326</td>\n",
              "      <td>0.001274</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.047227</td>\n",
              "      <td>8.114625</td>\n",
              "      <td>6.732629</td>\n",
              "      <td>0.048356</td>\n",
              "      <td>0.005681</td>\n",
              "      <td>0.138035</td>\n",
              "      <td>0.014350</td>\n",
              "      <td>0.045217</td>\n",
              "      <td>0.077488</td>\n",
              "      <td>40.177700</td>\n",
              "      <td>0.230601</td>\n",
              "      <td>3.248511</td>\n",
              "      <td>37.944601</td>\n",
              "      <td>32.739910</td>\n",
              "      <td>71.179343</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.246153</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.010870</td>\n",
              "      <td>5.087418</td>\n",
              "      <td>0.025827</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.210188</td>\n",
              "      <td>6.291549</td>\n",
              "      <td>...</td>\n",
              "      <td>0.002980</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.001881</td>\n",
              "      <td>0.000432</td>\n",
              "      <td>1.173709e-06</td>\n",
              "      <td>0.000004</td>\n",
              "      <td>0.000100</td>\n",
              "      <td>0.000051</td>\n",
              "      <td>0.006797</td>\n",
              "      <td>0.000089</td>\n",
              "      <td>0.001902</td>\n",
              "      <td>0.000065</td>\n",
              "      <td>0.000408</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000103</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000460</td>\n",
              "      <td>0.006659</td>\n",
              "      <td>0.000159</td>\n",
              "      <td>0.001459</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000050</td>\n",
              "      <td>0.000001</td>\n",
              "      <td>0.009273</td>\n",
              "      <td>0.000636</td>\n",
              "      <td>0.000058</td>\n",
              "      <td>0.000101</td>\n",
              "      <td>0.000001</td>\n",
              "      <td>0.000125</td>\n",
              "      <td>0.000028</td>\n",
              "      <td>0.000050</td>\n",
              "      <td>0.000003</td>\n",
              "      <td>0.000019</td>\n",
              "      <td>0.476995</td>\n",
              "      <td>0.013615</td>\n",
              "      <td>0.017606</td>\n",
              "      <td>0.048557</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.105897e+06</td>\n",
              "      <td>1.041708</td>\n",
              "      <td>17.350339</td>\n",
              "      <td>7.700000</td>\n",
              "      <td>7.377778</td>\n",
              "      <td>10.745404</td>\n",
              "      <td>28.518519</td>\n",
              "      <td>27.170017</td>\n",
              "      <td>36.970370</td>\n",
              "      <td>0.509611</td>\n",
              "      <td>0.424066</td>\n",
              "      <td>0.142326</td>\n",
              "      <td>0.099300</td>\n",
              "      <td>0.002934</td>\n",
              "      <td>4.196521</td>\n",
              "      <td>4.488952</td>\n",
              "      <td>0.442857</td>\n",
              "      <td>39.162234</td>\n",
              "      <td>119.259259</td>\n",
              "      <td>7.392593</td>\n",
              "      <td>0.015404</td>\n",
              "      <td>3.078965</td>\n",
              "      <td>0.025200</td>\n",
              "      <td>0.106441</td>\n",
              "      <td>1.839764</td>\n",
              "      <td>131.759966</td>\n",
              "      <td>3.967752</td>\n",
              "      <td>130.819593</td>\n",
              "      <td>381.446989</td>\n",
              "      <td>1116.573367</td>\n",
              "      <td>4352.914894</td>\n",
              "      <td>163.731128</td>\n",
              "      <td>2.514563</td>\n",
              "      <td>33.163528</td>\n",
              "      <td>0.765211</td>\n",
              "      <td>587.404255</td>\n",
              "      <td>1.439618</td>\n",
              "      <td>475.002375</td>\n",
              "      <td>2.564158</td>\n",
              "      <td>6.291549</td>\n",
              "      <td>...</td>\n",
              "      <td>0.018017</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.017298</td>\n",
              "      <td>0.001777</td>\n",
              "      <td>1.184834e-05</td>\n",
              "      <td>0.000043</td>\n",
              "      <td>0.000545</td>\n",
              "      <td>0.000051</td>\n",
              "      <td>0.456394</td>\n",
              "      <td>0.005574</td>\n",
              "      <td>0.007629</td>\n",
              "      <td>0.000065</td>\n",
              "      <td>0.008614</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.002992</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000460</td>\n",
              "      <td>0.016773</td>\n",
              "      <td>0.002872</td>\n",
              "      <td>0.109142</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.003732</td>\n",
              "      <td>0.000559</td>\n",
              "      <td>0.514488</td>\n",
              "      <td>0.062130</td>\n",
              "      <td>0.000076</td>\n",
              "      <td>0.000894</td>\n",
              "      <td>0.000104</td>\n",
              "      <td>0.002840</td>\n",
              "      <td>0.000416</td>\n",
              "      <td>0.000628</td>\n",
              "      <td>0.001441</td>\n",
              "      <td>0.002182</td>\n",
              "      <td>1.174141</td>\n",
              "      <td>0.013615</td>\n",
              "      <td>0.017606</td>\n",
              "      <td>0.123579</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8 rows × 199 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "             fdc_id          1002  ...          2032          2033\n",
              "count  1.310500e+04  13105.000000  ...  13105.000000  13105.000000\n",
              "mean   4.807517e+05      0.243665  ...      0.005723      0.022570\n",
              "std    2.544018e+05      0.220034  ...      0.008247      0.037044\n",
              "min    3.198770e+05      0.000000  ...      0.000000      0.000000\n",
              "25%    3.256480e+05      0.039103  ...      0.000000      0.000000\n",
              "50%    3.314100e+05      0.228130  ...      0.000000      0.000000\n",
              "75%    7.474680e+05      0.347477  ...      0.017606      0.048557\n",
              "max    1.105897e+06      1.041708  ...      0.017606      0.123579\n",
              "\n",
              "[8 rows x 199 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 350
        },
        "id": "fWnRpWu574qz",
        "outputId": "780ebf61-d3f6-4782-839e-0206dbbd1dbd"
      },
      "source": [
        "df_median = pd.read_csv(data_median, index_col=0)\n",
        "del df_median['food_category_id']\n",
        "df_median.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>fdc_id</th>\n",
              "      <th>1002</th>\n",
              "      <th>1003</th>\n",
              "      <th>1004</th>\n",
              "      <th>1005</th>\n",
              "      <th>1007</th>\n",
              "      <th>1008</th>\n",
              "      <th>1009</th>\n",
              "      <th>1010</th>\n",
              "      <th>1011</th>\n",
              "      <th>1012</th>\n",
              "      <th>1013</th>\n",
              "      <th>1014</th>\n",
              "      <th>1024</th>\n",
              "      <th>1032</th>\n",
              "      <th>1039</th>\n",
              "      <th>1050</th>\n",
              "      <th>1051</th>\n",
              "      <th>1062</th>\n",
              "      <th>1063</th>\n",
              "      <th>1075</th>\n",
              "      <th>1079</th>\n",
              "      <th>1082</th>\n",
              "      <th>1084</th>\n",
              "      <th>1085</th>\n",
              "      <th>1087</th>\n",
              "      <th>1089</th>\n",
              "      <th>1090</th>\n",
              "      <th>1091</th>\n",
              "      <th>1092</th>\n",
              "      <th>1093</th>\n",
              "      <th>1094</th>\n",
              "      <th>1095</th>\n",
              "      <th>1097</th>\n",
              "      <th>1098</th>\n",
              "      <th>1100</th>\n",
              "      <th>1101</th>\n",
              "      <th>1102</th>\n",
              "      <th>1103</th>\n",
              "      <th>1105</th>\n",
              "      <th>...</th>\n",
              "      <th>1323</th>\n",
              "      <th>1325</th>\n",
              "      <th>1329</th>\n",
              "      <th>1330</th>\n",
              "      <th>1331</th>\n",
              "      <th>1333</th>\n",
              "      <th>1334</th>\n",
              "      <th>1335</th>\n",
              "      <th>1404</th>\n",
              "      <th>1405</th>\n",
              "      <th>1406</th>\n",
              "      <th>1409</th>\n",
              "      <th>1411</th>\n",
              "      <th>1414</th>\n",
              "      <th>2003</th>\n",
              "      <th>2004</th>\n",
              "      <th>2005</th>\n",
              "      <th>2006</th>\n",
              "      <th>2007</th>\n",
              "      <th>2008</th>\n",
              "      <th>2009</th>\n",
              "      <th>2010</th>\n",
              "      <th>2012</th>\n",
              "      <th>2013</th>\n",
              "      <th>2014</th>\n",
              "      <th>2015</th>\n",
              "      <th>2016</th>\n",
              "      <th>2018</th>\n",
              "      <th>2019</th>\n",
              "      <th>2020</th>\n",
              "      <th>2021</th>\n",
              "      <th>2022</th>\n",
              "      <th>2023</th>\n",
              "      <th>2024</th>\n",
              "      <th>2025</th>\n",
              "      <th>2026</th>\n",
              "      <th>2028</th>\n",
              "      <th>2029</th>\n",
              "      <th>2032</th>\n",
              "      <th>2033</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>1.310500e+04</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>...</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "      <td>13105.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>4.807517e+05</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.417543</td>\n",
              "      <td>0.142532</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.013915</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3.310736</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.865044</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.368859</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>13.674781</td>\n",
              "      <td>0.424638</td>\n",
              "      <td>14.934300</td>\n",
              "      <td>43.813277</td>\n",
              "      <td>133.149180</td>\n",
              "      <td>0.0</td>\n",
              "      <td>19.522549</td>\n",
              "      <td>0.288790</td>\n",
              "      <td>3.508661</td>\n",
              "      <td>0.085287</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.157440</td>\n",
              "      <td>34.636780</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.544018e+05</td>\n",
              "      <td>0.0</td>\n",
              "      <td>7.085115</td>\n",
              "      <td>0.438829</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.115348</td>\n",
              "      <td>0.0</td>\n",
              "      <td>10.530080</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>7.170618</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.173188</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>43.493809</td>\n",
              "      <td>1.350597</td>\n",
              "      <td>47.499817</td>\n",
              "      <td>139.351873</td>\n",
              "      <td>423.492346</td>\n",
              "      <td>0.0</td>\n",
              "      <td>62.093135</td>\n",
              "      <td>0.918521</td>\n",
              "      <td>11.159596</td>\n",
              "      <td>0.271264</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.500751</td>\n",
              "      <td>110.165239</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>3.198770e+05</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>3.256480e+05</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>3.314100e+05</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>7.474680e+05</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.105897e+06</td>\n",
              "      <td>0.0</td>\n",
              "      <td>23.300000</td>\n",
              "      <td>2.410000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.970000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>36.800000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>60.300000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>4.100000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>152.000000</td>\n",
              "      <td>4.720000</td>\n",
              "      <td>166.000000</td>\n",
              "      <td>487.000000</td>\n",
              "      <td>1480.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>217.000000</td>\n",
              "      <td>3.210000</td>\n",
              "      <td>39.000000</td>\n",
              "      <td>0.948000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.750000</td>\n",
              "      <td>385.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8 rows × 199 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "             fdc_id     1002          1003  ...     2029     2032     2033\n",
              "count  1.310500e+04  13105.0  13105.000000  ...  13105.0  13105.0  13105.0\n",
              "mean   4.807517e+05      0.0      2.417543  ...      0.0      0.0      0.0\n",
              "std    2.544018e+05      0.0      7.085115  ...      0.0      0.0      0.0\n",
              "min    3.198770e+05      0.0      0.000000  ...      0.0      0.0      0.0\n",
              "25%    3.256480e+05      0.0      0.000000  ...      0.0      0.0      0.0\n",
              "50%    3.314100e+05      0.0      0.000000  ...      0.0      0.0      0.0\n",
              "75%    7.474680e+05      0.0      0.000000  ...      0.0      0.0      0.0\n",
              "max    1.105897e+06      0.0     23.300000  ...      0.0      0.0      0.0\n",
              "\n",
              "[8 rows x 199 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fD0MsYk9MWc0"
      },
      "source": [
        "device = torch.device(\"cuda\" if use_cuda else \"cpu\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LtfjBn3wDBtI"
      },
      "source": [
        "# 20 Columns"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n1A-kBN7UHOy"
      },
      "source": [
        "data_path = 'datasets/average datasets/byColumnsWithEmbeddings/20_columns_embeddings.csv'\n",
        "num_embeddings = 5\n",
        "num_columns = 20"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 320
        },
        "id": "KFKtYt3pXTLz",
        "outputId": "d77f3034-ba47-4552-b5f2-6d2f18954a0a"
      },
      "source": [
        "dataset = pd.read_csv(data_path, index_col=0)\n",
        "dataset[dataset.notna().all(axis=1)].describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>fdc_id</th>\n",
              "      <th>1004</th>\n",
              "      <th>1087</th>\n",
              "      <th>1090</th>\n",
              "      <th>1092</th>\n",
              "      <th>1095</th>\n",
              "      <th>1091</th>\n",
              "      <th>1089</th>\n",
              "      <th>1098</th>\n",
              "      <th>1101</th>\n",
              "      <th>1051</th>\n",
              "      <th>1007</th>\n",
              "      <th>1002</th>\n",
              "      <th>1093</th>\n",
              "      <th>1079</th>\n",
              "      <th>1003</th>\n",
              "      <th>1009</th>\n",
              "      <th>1102</th>\n",
              "      <th>1265</th>\n",
              "      <th>1266</th>\n",
              "      <th>1264</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>1.310500e+04</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>4.807517e+05</td>\n",
              "      <td>1.824697</td>\n",
              "      <td>27.659061</td>\n",
              "      <td>14.595597</td>\n",
              "      <td>125.003892</td>\n",
              "      <td>0.384163</td>\n",
              "      <td>55.746662</td>\n",
              "      <td>0.476038</td>\n",
              "      <td>0.077504</td>\n",
              "      <td>0.149979</td>\n",
              "      <td>8.349769</td>\n",
              "      <td>0.472742</td>\n",
              "      <td>0.243665</td>\n",
              "      <td>133.394887</td>\n",
              "      <td>0.350355</td>\n",
              "      <td>1.942243</td>\n",
              "      <td>2.561062</td>\n",
              "      <td>42.944411</td>\n",
              "      <td>0.308332</td>\n",
              "      <td>0.116491</td>\n",
              "      <td>0.065003</td>\n",
              "      <td>21488.189968</td>\n",
              "      <td>176.402583</td>\n",
              "      <td>0.119704</td>\n",
              "      <td>-0.093912</td>\n",
              "      <td>-0.048493</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.544018e+05</td>\n",
              "      <td>7.838466</td>\n",
              "      <td>99.924214</td>\n",
              "      <td>45.754175</td>\n",
              "      <td>382.322293</td>\n",
              "      <td>1.055340</td>\n",
              "      <td>153.432660</td>\n",
              "      <td>1.483360</td>\n",
              "      <td>0.264892</td>\n",
              "      <td>0.515558</td>\n",
              "      <td>23.549114</td>\n",
              "      <td>4.416751</td>\n",
              "      <td>0.999601</td>\n",
              "      <td>1736.846193</td>\n",
              "      <td>1.186155</td>\n",
              "      <td>6.583455</td>\n",
              "      <td>9.446549</td>\n",
              "      <td>187.200801</td>\n",
              "      <td>1.605577</td>\n",
              "      <td>0.597369</td>\n",
              "      <td>0.559044</td>\n",
              "      <td>254447.573253</td>\n",
              "      <td>3958.010969</td>\n",
              "      <td>1.112342</td>\n",
              "      <td>0.694392</td>\n",
              "      <td>0.598292</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>3.198770e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-139555.939272</td>\n",
              "      <td>-7413.329674</td>\n",
              "      <td>-3.349638</td>\n",
              "      <td>-1.585647</td>\n",
              "      <td>-1.527263</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>3.256480e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-133692.806088</td>\n",
              "      <td>-2328.507401</td>\n",
              "      <td>-0.796369</td>\n",
              "      <td>-0.667107</td>\n",
              "      <td>-0.477074</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>3.314100e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-127823.125712</td>\n",
              "      <td>-153.274226</td>\n",
              "      <td>-0.002647</td>\n",
              "      <td>-0.011945</td>\n",
              "      <td>-0.071942</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>7.474680e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>288262.097477</td>\n",
              "      <td>3032.870912</td>\n",
              "      <td>0.983044</td>\n",
              "      <td>0.441140</td>\n",
              "      <td>0.323268</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.105897e+06</td>\n",
              "      <td>99.400000</td>\n",
              "      <td>1390.000000</td>\n",
              "      <td>386.000000</td>\n",
              "      <td>2520.000000</td>\n",
              "      <td>7.660000</td>\n",
              "      <td>997.000000</td>\n",
              "      <td>12.700000</td>\n",
              "      <td>1.920000</td>\n",
              "      <td>7.870000</td>\n",
              "      <td>96.700000</td>\n",
              "      <td>99.600000</td>\n",
              "      <td>13.200000</td>\n",
              "      <td>40700.000000</td>\n",
              "      <td>11.500000</td>\n",
              "      <td>79.900000</td>\n",
              "      <td>43.900000</td>\n",
              "      <td>1790.000000</td>\n",
              "      <td>22.800000</td>\n",
              "      <td>8.610000</td>\n",
              "      <td>17.000000</td>\n",
              "      <td>646711.616853</td>\n",
              "      <td>7635.496843</td>\n",
              "      <td>2.647689</td>\n",
              "      <td>1.869347</td>\n",
              "      <td>1.616847</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             fdc_id          1004  ...             3             4\n",
              "count  1.310500e+04  13105.000000  ...  13105.000000  13105.000000\n",
              "mean   4.807517e+05      1.824697  ...     -0.093912     -0.048493\n",
              "std    2.544018e+05      7.838466  ...      0.694392      0.598292\n",
              "min    3.198770e+05      0.000000  ...     -1.585647     -1.527263\n",
              "25%    3.256480e+05      0.000000  ...     -0.667107     -0.477074\n",
              "50%    3.314100e+05      0.000000  ...     -0.011945     -0.071942\n",
              "75%    7.474680e+05      0.000000  ...      0.441140      0.323268\n",
              "max    1.105897e+06     99.400000  ...      1.869347      1.616847\n",
              "\n",
              "[8 rows x 26 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xg5tqVZLWJWn"
      },
      "source": [
        "data = dataset\n",
        "train_data, test_data = train_test_split(data, test_size=test_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 320
        },
        "id": "AX4xszZvl1MP",
        "outputId": "b5b30a17-cff5-4761-9ef7-9f49c50a67e1"
      },
      "source": [
        "train_data.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>fdc_id</th>\n",
              "      <th>1004</th>\n",
              "      <th>1087</th>\n",
              "      <th>1090</th>\n",
              "      <th>1092</th>\n",
              "      <th>1095</th>\n",
              "      <th>1091</th>\n",
              "      <th>1089</th>\n",
              "      <th>1098</th>\n",
              "      <th>1101</th>\n",
              "      <th>1051</th>\n",
              "      <th>1007</th>\n",
              "      <th>1002</th>\n",
              "      <th>1093</th>\n",
              "      <th>1079</th>\n",
              "      <th>1003</th>\n",
              "      <th>1009</th>\n",
              "      <th>1102</th>\n",
              "      <th>1265</th>\n",
              "      <th>1266</th>\n",
              "      <th>1264</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>1.048400e+04</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>4.822540e+05</td>\n",
              "      <td>1.755201</td>\n",
              "      <td>27.743609</td>\n",
              "      <td>14.484309</td>\n",
              "      <td>124.078024</td>\n",
              "      <td>0.380677</td>\n",
              "      <td>55.346433</td>\n",
              "      <td>0.474776</td>\n",
              "      <td>0.077031</td>\n",
              "      <td>0.148934</td>\n",
              "      <td>8.414429</td>\n",
              "      <td>0.476104</td>\n",
              "      <td>0.234480</td>\n",
              "      <td>139.155284</td>\n",
              "      <td>0.355341</td>\n",
              "      <td>1.915318</td>\n",
              "      <td>2.559018</td>\n",
              "      <td>42.233441</td>\n",
              "      <td>0.308766</td>\n",
              "      <td>0.118179</td>\n",
              "      <td>0.063822</td>\n",
              "      <td>22990.229720</td>\n",
              "      <td>203.642129</td>\n",
              "      <td>0.120423</td>\n",
              "      <td>-0.094587</td>\n",
              "      <td>-0.045482</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.559578e+05</td>\n",
              "      <td>7.687655</td>\n",
              "      <td>101.585144</td>\n",
              "      <td>45.636608</td>\n",
              "      <td>381.855342</td>\n",
              "      <td>1.053683</td>\n",
              "      <td>153.490134</td>\n",
              "      <td>1.485049</td>\n",
              "      <td>0.263929</td>\n",
              "      <td>0.514145</td>\n",
              "      <td>23.687789</td>\n",
              "      <td>4.515724</td>\n",
              "      <td>0.967373</td>\n",
              "      <td>1822.974657</td>\n",
              "      <td>1.197866</td>\n",
              "      <td>6.542063</td>\n",
              "      <td>9.447906</td>\n",
              "      <td>184.659671</td>\n",
              "      <td>1.595443</td>\n",
              "      <td>0.603438</td>\n",
              "      <td>0.523031</td>\n",
              "      <td>256004.025403</td>\n",
              "      <td>3967.478183</td>\n",
              "      <td>1.109581</td>\n",
              "      <td>0.694324</td>\n",
              "      <td>0.598217</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>3.198770e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-139555.939272</td>\n",
              "      <td>-7413.329674</td>\n",
              "      <td>-3.349638</td>\n",
              "      <td>-1.585647</td>\n",
              "      <td>-1.481988</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>3.256225e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-133718.782631</td>\n",
              "      <td>-2326.205329</td>\n",
              "      <td>-0.788413</td>\n",
              "      <td>-0.671057</td>\n",
              "      <td>-0.476737</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>3.314150e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-127818.032271</td>\n",
              "      <td>-111.096255</td>\n",
              "      <td>-0.002088</td>\n",
              "      <td>-0.011191</td>\n",
              "      <td>-0.071399</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>7.475152e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>288310.230486</td>\n",
              "      <td>3047.830480</td>\n",
              "      <td>0.982963</td>\n",
              "      <td>0.441077</td>\n",
              "      <td>0.330094</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.105897e+06</td>\n",
              "      <td>99.400000</td>\n",
              "      <td>1390.000000</td>\n",
              "      <td>386.000000</td>\n",
              "      <td>2510.000000</td>\n",
              "      <td>7.660000</td>\n",
              "      <td>997.000000</td>\n",
              "      <td>10.800000</td>\n",
              "      <td>1.920000</td>\n",
              "      <td>7.870000</td>\n",
              "      <td>96.700000</td>\n",
              "      <td>99.600000</td>\n",
              "      <td>13.000000</td>\n",
              "      <td>40700.000000</td>\n",
              "      <td>11.400000</td>\n",
              "      <td>79.900000</td>\n",
              "      <td>43.900000</td>\n",
              "      <td>1790.000000</td>\n",
              "      <td>22.800000</td>\n",
              "      <td>8.610000</td>\n",
              "      <td>16.800000</td>\n",
              "      <td>646711.616853</td>\n",
              "      <td>7635.496843</td>\n",
              "      <td>2.647689</td>\n",
              "      <td>1.869347</td>\n",
              "      <td>1.597387</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             fdc_id          1004  ...             3             4\n",
              "count  1.048400e+04  10484.000000  ...  10484.000000  10484.000000\n",
              "mean   4.822540e+05      1.755201  ...     -0.094587     -0.045482\n",
              "std    2.559578e+05      7.687655  ...      0.694324      0.598217\n",
              "min    3.198770e+05      0.000000  ...     -1.585647     -1.481988\n",
              "25%    3.256225e+05      0.000000  ...     -0.671057     -0.476737\n",
              "50%    3.314150e+05      0.000000  ...     -0.011191     -0.071399\n",
              "75%    7.475152e+05      0.000000  ...      0.441077      0.330094\n",
              "max    1.105897e+06     99.400000  ...      1.869347      1.597387\n",
              "\n",
              "[8 rows x 26 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 320
        },
        "id": "yUlm9SUGl4OW",
        "outputId": "62940f01-b7f4-4338-ab4b-584358717f1e"
      },
      "source": [
        "test_data.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>fdc_id</th>\n",
              "      <th>1004</th>\n",
              "      <th>1087</th>\n",
              "      <th>1090</th>\n",
              "      <th>1092</th>\n",
              "      <th>1095</th>\n",
              "      <th>1091</th>\n",
              "      <th>1089</th>\n",
              "      <th>1098</th>\n",
              "      <th>1101</th>\n",
              "      <th>1051</th>\n",
              "      <th>1007</th>\n",
              "      <th>1002</th>\n",
              "      <th>1093</th>\n",
              "      <th>1079</th>\n",
              "      <th>1003</th>\n",
              "      <th>1009</th>\n",
              "      <th>1102</th>\n",
              "      <th>1265</th>\n",
              "      <th>1266</th>\n",
              "      <th>1264</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>2.621000e+03</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>4.747426e+05</td>\n",
              "      <td>2.102678</td>\n",
              "      <td>27.320870</td>\n",
              "      <td>15.040748</td>\n",
              "      <td>128.707364</td>\n",
              "      <td>0.398108</td>\n",
              "      <td>57.347577</td>\n",
              "      <td>0.481087</td>\n",
              "      <td>0.079392</td>\n",
              "      <td>0.154159</td>\n",
              "      <td>8.091129</td>\n",
              "      <td>0.459294</td>\n",
              "      <td>0.280404</td>\n",
              "      <td>110.353300</td>\n",
              "      <td>0.330408</td>\n",
              "      <td>2.049947</td>\n",
              "      <td>2.569241</td>\n",
              "      <td>45.788287</td>\n",
              "      <td>0.306598</td>\n",
              "      <td>0.109738</td>\n",
              "      <td>0.069724</td>\n",
              "      <td>15480.030960</td>\n",
              "      <td>67.444400</td>\n",
              "      <td>0.116832</td>\n",
              "      <td>-0.091213</td>\n",
              "      <td>-0.060537</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.480369e+05</td>\n",
              "      <td>8.410532</td>\n",
              "      <td>93.001944</td>\n",
              "      <td>46.227551</td>\n",
              "      <td>384.235236</td>\n",
              "      <td>1.062026</td>\n",
              "      <td>153.221347</td>\n",
              "      <td>1.476856</td>\n",
              "      <td>0.268750</td>\n",
              "      <td>0.521250</td>\n",
              "      <td>22.988673</td>\n",
              "      <td>3.997199</td>\n",
              "      <td>1.118717</td>\n",
              "      <td>1338.044656</td>\n",
              "      <td>1.138109</td>\n",
              "      <td>6.746680</td>\n",
              "      <td>9.442917</td>\n",
              "      <td>197.048894</td>\n",
              "      <td>1.645802</td>\n",
              "      <td>0.572513</td>\n",
              "      <td>0.684504</td>\n",
              "      <td>248081.146906</td>\n",
              "      <td>3918.771433</td>\n",
              "      <td>1.123528</td>\n",
              "      <td>0.694789</td>\n",
              "      <td>0.598559</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>3.198820e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-139550.845832</td>\n",
              "      <td>-7411.367800</td>\n",
              "      <td>-2.816578</td>\n",
              "      <td>-1.537302</td>\n",
              "      <td>-1.527263</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>3.257310e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-133608.254984</td>\n",
              "      <td>-2345.183653</td>\n",
              "      <td>-0.820487</td>\n",
              "      <td>-0.646966</td>\n",
              "      <td>-0.482865</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>3.314030e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-127830.256528</td>\n",
              "      <td>-330.340365</td>\n",
              "      <td>-0.006454</td>\n",
              "      <td>-0.014591</td>\n",
              "      <td>-0.076192</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>7.472940e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>288084.845758</td>\n",
              "      <td>2974.013474</td>\n",
              "      <td>0.984178</td>\n",
              "      <td>0.441246</td>\n",
              "      <td>0.321993</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.105887e+06</td>\n",
              "      <td>82.600000</td>\n",
              "      <td>938.000000</td>\n",
              "      <td>369.000000</td>\n",
              "      <td>2520.000000</td>\n",
              "      <td>7.050000</td>\n",
              "      <td>986.000000</td>\n",
              "      <td>12.700000</td>\n",
              "      <td>1.840000</td>\n",
              "      <td>4.010000</td>\n",
              "      <td>96.000000</td>\n",
              "      <td>99.400000</td>\n",
              "      <td>13.200000</td>\n",
              "      <td>40100.000000</td>\n",
              "      <td>11.500000</td>\n",
              "      <td>48.100000</td>\n",
              "      <td>42.500000</td>\n",
              "      <td>1610.000000</td>\n",
              "      <td>22.200000</td>\n",
              "      <td>7.420000</td>\n",
              "      <td>17.000000</td>\n",
              "      <td>646701.429977</td>\n",
              "      <td>7630.592063</td>\n",
              "      <td>2.647168</td>\n",
              "      <td>1.869343</td>\n",
              "      <td>1.616847</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             fdc_id         1004  ...            3            4\n",
              "count  2.621000e+03  2621.000000  ...  2621.000000  2621.000000\n",
              "mean   4.747426e+05     2.102678  ...    -0.091213    -0.060537\n",
              "std    2.480369e+05     8.410532  ...     0.694789     0.598559\n",
              "min    3.198820e+05     0.000000  ...    -1.537302    -1.527263\n",
              "25%    3.257310e+05     0.000000  ...    -0.646966    -0.482865\n",
              "50%    3.314030e+05     0.000000  ...    -0.014591    -0.076192\n",
              "75%    7.472940e+05     0.000000  ...     0.441246     0.321993\n",
              "max    1.105887e+06    82.600000  ...     1.869343     1.616847\n",
              "\n",
              "[8 rows x 26 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WM5CUxL98PI2",
        "outputId": "9603aa43-606f-4c69-e3e9-717d46e562c3"
      },
      "source": [
        "indices = test_data['fdc_id']\n",
        "indices"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "767      321893\n",
              "11351    790103\n",
              "7753     333420\n",
              "3071     325328\n",
              "1309     322529\n",
              "          ...  \n",
              "3525     326013\n",
              "8207     334668\n",
              "11660    790480\n",
              "5439     329186\n",
              "2603     324520\n",
              "Name: fdc_id, Length: 2621, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xcr7XAhk4lzq"
      },
      "source": [
        "del train_data['fdc_id']\n",
        "del test_data['fdc_id']\n",
        "columns = train_data.columns"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4fjF5eTb_un3",
        "outputId": "1b7403f0-0cc1-4a53-c007-c5d70519d6c7"
      },
      "source": [
        "train_data = train_data.to_numpy()\n",
        "test_data.fillna(0, inplace=True)\n",
        "test_data = test_data.to_numpy()\n",
        "data = dataset.values\n",
        "rows, cols = data.shape\n",
        "cols -= 1\n",
        "\n",
        "y_test = test_data.copy()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pandas/core/frame.py:4327: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  downcast=downcast,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 320
        },
        "id": "DEfQcR8l8ZWm",
        "outputId": "50d8b406-cdc3-48ed-86f2-cb6bbffb9414"
      },
      "source": [
        "missed_data, missed_data_train, mask = missing_method(test_data, train_data, num_embeddings)\n",
        "pd.DataFrame(missed_data).describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>2128.000000</td>\n",
              "      <td>2103.000000</td>\n",
              "      <td>2081.000000</td>\n",
              "      <td>2088.000000</td>\n",
              "      <td>2107.000000</td>\n",
              "      <td>2127.000000</td>\n",
              "      <td>2096.000000</td>\n",
              "      <td>2105.000000</td>\n",
              "      <td>2098.000000</td>\n",
              "      <td>2116.000000</td>\n",
              "      <td>2100.000000</td>\n",
              "      <td>2067.000000</td>\n",
              "      <td>2093.000000</td>\n",
              "      <td>2115.000000</td>\n",
              "      <td>2080.000000</td>\n",
              "      <td>2085.000000</td>\n",
              "      <td>2138.000000</td>\n",
              "      <td>2111.000000</td>\n",
              "      <td>2069.000000</td>\n",
              "      <td>2088.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>2.127702</td>\n",
              "      <td>28.920114</td>\n",
              "      <td>14.914512</td>\n",
              "      <td>126.250000</td>\n",
              "      <td>0.408519</td>\n",
              "      <td>58.964269</td>\n",
              "      <td>0.489957</td>\n",
              "      <td>0.079634</td>\n",
              "      <td>0.159527</td>\n",
              "      <td>8.198341</td>\n",
              "      <td>0.446043</td>\n",
              "      <td>0.306473</td>\n",
              "      <td>86.846632</td>\n",
              "      <td>0.330875</td>\n",
              "      <td>2.031240</td>\n",
              "      <td>2.564427</td>\n",
              "      <td>45.905753</td>\n",
              "      <td>0.324708</td>\n",
              "      <td>0.111738</td>\n",
              "      <td>0.067591</td>\n",
              "      <td>15480.030960</td>\n",
              "      <td>67.444400</td>\n",
              "      <td>0.116832</td>\n",
              "      <td>-0.091213</td>\n",
              "      <td>-0.060537</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>8.600823</td>\n",
              "      <td>95.757893</td>\n",
              "      <td>45.769414</td>\n",
              "      <td>380.203401</td>\n",
              "      <td>1.074592</td>\n",
              "      <td>155.364526</td>\n",
              "      <td>1.495226</td>\n",
              "      <td>0.270365</td>\n",
              "      <td>0.526523</td>\n",
              "      <td>23.213875</td>\n",
              "      <td>3.877333</td>\n",
              "      <td>1.184570</td>\n",
              "      <td>879.867413</td>\n",
              "      <td>1.127328</td>\n",
              "      <td>6.722095</td>\n",
              "      <td>9.429371</td>\n",
              "      <td>198.052557</td>\n",
              "      <td>1.712762</td>\n",
              "      <td>0.574624</td>\n",
              "      <td>0.700947</td>\n",
              "      <td>248081.146906</td>\n",
              "      <td>3918.771433</td>\n",
              "      <td>1.123528</td>\n",
              "      <td>0.694789</td>\n",
              "      <td>0.598559</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-139550.845832</td>\n",
              "      <td>-7411.367800</td>\n",
              "      <td>-2.816578</td>\n",
              "      <td>-1.537302</td>\n",
              "      <td>-1.527263</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-133608.254984</td>\n",
              "      <td>-2345.183653</td>\n",
              "      <td>-0.820487</td>\n",
              "      <td>-0.646966</td>\n",
              "      <td>-0.482865</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-127830.256528</td>\n",
              "      <td>-330.340365</td>\n",
              "      <td>-0.006454</td>\n",
              "      <td>-0.014591</td>\n",
              "      <td>-0.076192</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>288084.845758</td>\n",
              "      <td>2974.013474</td>\n",
              "      <td>0.984178</td>\n",
              "      <td>0.441246</td>\n",
              "      <td>0.321993</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>82.600000</td>\n",
              "      <td>938.000000</td>\n",
              "      <td>369.000000</td>\n",
              "      <td>2030.000000</td>\n",
              "      <td>6.630000</td>\n",
              "      <td>986.000000</td>\n",
              "      <td>12.700000</td>\n",
              "      <td>1.840000</td>\n",
              "      <td>4.010000</td>\n",
              "      <td>96.000000</td>\n",
              "      <td>99.400000</td>\n",
              "      <td>13.200000</td>\n",
              "      <td>38200.000000</td>\n",
              "      <td>11.000000</td>\n",
              "      <td>48.100000</td>\n",
              "      <td>42.500000</td>\n",
              "      <td>1610.000000</td>\n",
              "      <td>22.200000</td>\n",
              "      <td>7.420000</td>\n",
              "      <td>17.000000</td>\n",
              "      <td>646701.429977</td>\n",
              "      <td>7630.592063</td>\n",
              "      <td>2.647168</td>\n",
              "      <td>1.869343</td>\n",
              "      <td>1.616847</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                0            1   ...           23           24\n",
              "count  2128.000000  2103.000000  ...  2621.000000  2621.000000\n",
              "mean      2.127702    28.920114  ...    -0.091213    -0.060537\n",
              "std       8.600823    95.757893  ...     0.694789     0.598559\n",
              "min       0.000000     0.000000  ...    -1.537302    -1.527263\n",
              "25%       0.000000     0.000000  ...    -0.646966    -0.482865\n",
              "50%       0.000000     0.000000  ...    -0.014591    -0.076192\n",
              "75%       0.000000     0.000000  ...     0.441246     0.321993\n",
              "max      82.600000   938.000000  ...     1.869343     1.616847\n",
              "\n",
              "[8 rows x 25 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aYb-iCxcZh6B"
      },
      "source": [
        "# traditional methods test sets - impute across all foods\n",
        "X_train = pd.DataFrame(missed_data_train, columns = columns)\n",
        "X_test_mean_all = pd.DataFrame(missed_data, columns = columns)\n",
        "X_test_median_all = pd.DataFrame(missed_data, columns = columns)\n",
        "\n",
        "X_test_mean_all = impute_traditional_all_foods(X_train, X_test_mean_all, 'mean')\n",
        "X_test_median_all = impute_traditional_all_foods(X_train, X_test_median_all, 'median')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vhJHJEEe9PGZ"
      },
      "source": [
        "# Traditional methods test sets - impute by food category\n",
        "X_test_mean = pd.DataFrame(missed_data, columns = columns, index=indices)\n",
        "X_test_median = X_test_mean.copy()\n",
        "df_mean_cat = df_mean[df_mean['fdc_id'].isin(indices)]\n",
        "X_test_mean = impute_traditional_by_category(X_test_mean, df_mean_cat, indices)\n",
        "df_median_cat = df_median[df_median['fdc_id'].isin(indices)]\n",
        "X_test_median = impute_traditional_by_category(X_test_median, df_median_cat, indices)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y3ccnRgs80DK"
      },
      "source": [
        "scaler = MinMaxScaler()\n",
        "scaler.fit(train_data)\n",
        "train_data = scaler.transform(train_data)\n",
        "test_data = scaler.transform(test_data)\n",
        "X_test_mean = scaler.transform(X_test_mean)\n",
        "X_test_median = scaler.transform(X_test_median)\n",
        "X_test_mean_all = scaler.transform(X_test_mean_all)\n",
        "X_test_median_all = scaler.transform(X_test_median_all)\n",
        "\n",
        "y_test = scaler.transform(y_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MpX4gbuaG4Uh"
      },
      "source": [
        "# datasets without embeddings for autoencoder\n",
        "train_data_noEmbeddings = pd.DataFrame(train_data, columns = columns)\n",
        "train_data_noEmbeddings = np.array(train_data_noEmbeddings.iloc[:, :-num_embeddings]) # df = df.iloc[: , :-1]\n",
        "test_mean_noEmbeddings = np.array(pd.DataFrame(X_test_mean, columns = columns).iloc[:, :-num_embeddings])\n",
        "test_median_noEmbeddings = np.array(pd.DataFrame(X_test_median, columns = columns).iloc[:, :-num_embeddings])\n",
        "test_mean_noEmbeddings_all = np.array(pd.DataFrame(X_test_mean_all, columns = columns).iloc[:, :-num_embeddings])\n",
        "test_median_noEmbeddings_all = np.array(pd.DataFrame(X_test_median_all, columns = columns).iloc[:, :-num_embeddings])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6XqXk22U8kqI"
      },
      "source": [
        "missed_data = X_test_mean_all\n",
        "missed_data = torch.from_numpy(missed_data).float()\n",
        "\n",
        "train_data = torch.from_numpy(train_data).float()\n",
        "\n",
        "train_loader = torch.utils.data.DataLoader(dataset=train_data,\n",
        "                                           batch_size=batch_size,\n",
        "                                           shuffle=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WzVcSeRmIjVD"
      },
      "source": [
        "missed_data_noEm = test_mean_noEmbeddings_all\n",
        "missed_data_noEm = torch.from_numpy(missed_data_noEm).float()\n",
        "\n",
        "train_data_noEmbeddings = torch.from_numpy(train_data_noEmbeddings).float()\n",
        "\n",
        "train_loader_noEm = torch.utils.data.DataLoader(dataset=train_data_noEmbeddings,\n",
        "                                           batch_size=batch_size,\n",
        "                                           shuffle=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eokKmxMvNzAE"
      },
      "source": [
        "## Autoencoder - without embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rM8VyvOtNzAG"
      },
      "source": [
        "model = Autoencoder(dim=cols-num_embeddings).to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1_3MxF8_NzAG"
      },
      "source": [
        "criterion = nn.MSELoss()\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wVg3Xo7FNzAH",
        "outputId": "05e29b2d-48ab-4926-b363-0aa1969b9158"
      },
      "source": [
        "training(num_epochs, model, train_loader_noEm, criterion, optimizer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch : 1/200, recon loss = 0.01529659\n",
            "epoch : 2/200, recon loss = 0.01418361\n",
            "epoch : 3/200, recon loss = 0.01256301\n",
            "epoch : 4/200, recon loss = 0.00741972\n",
            "epoch : 5/200, recon loss = 0.00631517\n",
            "epoch : 6/200, recon loss = 0.00621053\n",
            "epoch : 7/200, recon loss = 0.00613695\n",
            "epoch : 8/200, recon loss = 0.00609543\n",
            "epoch : 9/200, recon loss = 0.00604837\n",
            "epoch : 10/200, recon loss = 0.00601578\n",
            "epoch : 11/200, recon loss = 0.00595585\n",
            "epoch : 12/200, recon loss = 0.00587455\n",
            "epoch : 13/200, recon loss = 0.00579482\n",
            "epoch : 14/200, recon loss = 0.00552471\n",
            "epoch : 15/200, recon loss = 0.00499643\n",
            "epoch : 16/200, recon loss = 0.00423957\n",
            "epoch : 17/200, recon loss = 0.00383543\n",
            "epoch : 18/200, recon loss = 0.00374121\n",
            "epoch : 19/200, recon loss = 0.00365074\n",
            "epoch : 20/200, recon loss = 0.00368156\n",
            "epoch : 21/200, recon loss = 0.00358289\n",
            "epoch : 22/200, recon loss = 0.00359027\n",
            "epoch : 23/200, recon loss = 0.00357586\n",
            "epoch : 24/200, recon loss = 0.00363234\n",
            "epoch : 25/200, recon loss = 0.00358037\n",
            "epoch : 26/200, recon loss = 0.00361901\n",
            "epoch : 27/200, recon loss = 0.00356193\n",
            "epoch : 28/200, recon loss = 0.00356568\n",
            "epoch : 29/200, recon loss = 0.00355075\n",
            "epoch : 30/200, recon loss = 0.00347731\n",
            "epoch : 31/200, recon loss = 0.00356659\n",
            "epoch : 32/200, recon loss = 0.00356901\n",
            "epoch : 33/200, recon loss = 0.00357996\n",
            "epoch : 34/200, recon loss = 0.00354756\n",
            "epoch : 35/200, recon loss = 0.00349406\n",
            "epoch : 36/200, recon loss = 0.00352814\n",
            "epoch : 37/200, recon loss = 0.00349242\n",
            "epoch : 38/200, recon loss = 0.00348654\n",
            "epoch : 39/200, recon loss = 0.00342107\n",
            "epoch : 40/200, recon loss = 0.00338915\n",
            "epoch : 41/200, recon loss = 0.00339701\n",
            "epoch : 42/200, recon loss = 0.00337867\n",
            "epoch : 43/200, recon loss = 0.00339119\n",
            "epoch : 44/200, recon loss = 0.00326997\n",
            "epoch : 45/200, recon loss = 0.00336804\n",
            "epoch : 46/200, recon loss = 0.00331924\n",
            "epoch : 47/200, recon loss = 0.00326054\n",
            "epoch : 48/200, recon loss = 0.00326493\n",
            "epoch : 49/200, recon loss = 0.00325295\n",
            "epoch : 50/200, recon loss = 0.00321275\n",
            "epoch : 51/200, recon loss = 0.00321477\n",
            "epoch : 52/200, recon loss = 0.00321952\n",
            "epoch : 53/200, recon loss = 0.00315956\n",
            "epoch : 54/200, recon loss = 0.00327865\n",
            "epoch : 55/200, recon loss = 0.00313236\n",
            "epoch : 56/200, recon loss = 0.00314966\n",
            "epoch : 57/200, recon loss = 0.00317731\n",
            "epoch : 58/200, recon loss = 0.00305913\n",
            "epoch : 59/200, recon loss = 0.00311269\n",
            "epoch : 60/200, recon loss = 0.00316203\n",
            "epoch : 61/200, recon loss = 0.00316552\n",
            "epoch : 62/200, recon loss = 0.00315728\n",
            "epoch : 63/200, recon loss = 0.00310613\n",
            "epoch : 64/200, recon loss = 0.00314826\n",
            "epoch : 65/200, recon loss = 0.00312036\n",
            "epoch : 66/200, recon loss = 0.00314044\n",
            "epoch : 67/200, recon loss = 0.00313432\n",
            "epoch : 68/200, recon loss = 0.00304161\n",
            "epoch : 69/200, recon loss = 0.00309938\n",
            "epoch : 70/200, recon loss = 0.00316542\n",
            "epoch : 71/200, recon loss = 0.00305694\n",
            "epoch : 72/200, recon loss = 0.00306639\n",
            "epoch : 73/200, recon loss = 0.00298936\n",
            "epoch : 74/200, recon loss = 0.00304411\n",
            "epoch : 75/200, recon loss = 0.00299398\n",
            "epoch : 76/200, recon loss = 0.00300023\n",
            "epoch : 77/200, recon loss = 0.00294223\n",
            "epoch : 78/200, recon loss = 0.00294996\n",
            "epoch : 79/200, recon loss = 0.00287462\n",
            "epoch : 80/200, recon loss = 0.00294051\n",
            "epoch : 81/200, recon loss = 0.00286975\n",
            "epoch : 82/200, recon loss = 0.00282970\n",
            "epoch : 83/200, recon loss = 0.00284495\n",
            "epoch : 84/200, recon loss = 0.00281083\n",
            "epoch : 85/200, recon loss = 0.00283525\n",
            "epoch : 86/200, recon loss = 0.00291235\n",
            "epoch : 87/200, recon loss = 0.00283175\n",
            "epoch : 88/200, recon loss = 0.00282405\n",
            "epoch : 89/200, recon loss = 0.00279671\n",
            "epoch : 90/200, recon loss = 0.00280930\n",
            "epoch : 91/200, recon loss = 0.00274064\n",
            "epoch : 92/200, recon loss = 0.00268617\n",
            "epoch : 93/200, recon loss = 0.00267742\n",
            "epoch : 94/200, recon loss = 0.00266130\n",
            "epoch : 95/200, recon loss = 0.00264651\n",
            "epoch : 96/200, recon loss = 0.00273397\n",
            "epoch : 97/200, recon loss = 0.00265009\n",
            "epoch : 98/200, recon loss = 0.00262780\n",
            "epoch : 99/200, recon loss = 0.00265736\n",
            "epoch : 100/200, recon loss = 0.00258397\n",
            "epoch : 101/200, recon loss = 0.00257734\n",
            "epoch : 102/200, recon loss = 0.00265085\n",
            "epoch : 103/200, recon loss = 0.00256665\n",
            "epoch : 104/200, recon loss = 0.00258667\n",
            "epoch : 105/200, recon loss = 0.00257868\n",
            "epoch : 106/200, recon loss = 0.00258239\n",
            "epoch : 107/200, recon loss = 0.00257354\n",
            "epoch : 108/200, recon loss = 0.00260246\n",
            "epoch : 109/200, recon loss = 0.00259534\n",
            "epoch : 110/200, recon loss = 0.00257851\n",
            "epoch : 111/200, recon loss = 0.00260734\n",
            "epoch : 112/200, recon loss = 0.00254082\n",
            "epoch : 113/200, recon loss = 0.00255362\n",
            "epoch : 114/200, recon loss = 0.00257659\n",
            "epoch : 115/200, recon loss = 0.00257222\n",
            "epoch : 116/200, recon loss = 0.00251067\n",
            "epoch : 117/200, recon loss = 0.00256375\n",
            "epoch : 118/200, recon loss = 0.00255726\n",
            "epoch : 119/200, recon loss = 0.00251717\n",
            "epoch : 120/200, recon loss = 0.00253556\n",
            "epoch : 121/200, recon loss = 0.00248572\n",
            "epoch : 122/200, recon loss = 0.00243706\n",
            "epoch : 123/200, recon loss = 0.00259976\n",
            "epoch : 124/200, recon loss = 0.00252416\n",
            "epoch : 125/200, recon loss = 0.00250352\n",
            "epoch : 126/200, recon loss = 0.00248095\n",
            "epoch : 127/200, recon loss = 0.00248479\n",
            "epoch : 128/200, recon loss = 0.00251395\n",
            "epoch : 129/200, recon loss = 0.00238728\n",
            "epoch : 130/200, recon loss = 0.00248144\n",
            "epoch : 131/200, recon loss = 0.00244850\n",
            "epoch : 132/200, recon loss = 0.00249565\n",
            "epoch : 133/200, recon loss = 0.00245970\n",
            "epoch : 134/200, recon loss = 0.00251310\n",
            "epoch : 135/200, recon loss = 0.00239378\n",
            "epoch : 136/200, recon loss = 0.00245166\n",
            "epoch : 137/200, recon loss = 0.00248742\n",
            "epoch : 138/200, recon loss = 0.00254730\n",
            "epoch : 139/200, recon loss = 0.00248121\n",
            "epoch : 140/200, recon loss = 0.00246449\n",
            "epoch : 141/200, recon loss = 0.00250556\n",
            "epoch : 142/200, recon loss = 0.00246898\n",
            "epoch : 143/200, recon loss = 0.00237883\n",
            "epoch : 144/200, recon loss = 0.00241567\n",
            "epoch : 145/200, recon loss = 0.00244860\n",
            "epoch : 146/200, recon loss = 0.00233672\n",
            "epoch : 147/200, recon loss = 0.00243933\n",
            "epoch : 148/200, recon loss = 0.00244019\n",
            "epoch : 149/200, recon loss = 0.00251157\n",
            "epoch : 150/200, recon loss = 0.00243268\n",
            "epoch : 151/200, recon loss = 0.00249384\n",
            "epoch : 152/200, recon loss = 0.00240569\n",
            "epoch : 153/200, recon loss = 0.00248105\n",
            "epoch : 154/200, recon loss = 0.00239403\n",
            "epoch : 155/200, recon loss = 0.00245728\n",
            "epoch : 156/200, recon loss = 0.00240158\n",
            "epoch : 157/200, recon loss = 0.00240813\n",
            "epoch : 158/200, recon loss = 0.00246930\n",
            "epoch : 159/200, recon loss = 0.00238822\n",
            "epoch : 160/200, recon loss = 0.00241885\n",
            "epoch : 161/200, recon loss = 0.00246324\n",
            "epoch : 162/200, recon loss = 0.00241411\n",
            "epoch : 163/200, recon loss = 0.00239834\n",
            "epoch : 164/200, recon loss = 0.00237652\n",
            "epoch : 165/200, recon loss = 0.00235018\n",
            "epoch : 166/200, recon loss = 0.00236071\n",
            "epoch : 167/200, recon loss = 0.00246373\n",
            "epoch : 168/200, recon loss = 0.00237401\n",
            "epoch : 169/200, recon loss = 0.00246428\n",
            "epoch : 170/200, recon loss = 0.00235223\n",
            "epoch : 171/200, recon loss = 0.00246059\n",
            "epoch : 172/200, recon loss = 0.00231749\n",
            "epoch : 173/200, recon loss = 0.00229723\n",
            "epoch : 174/200, recon loss = 0.00231485\n",
            "epoch : 175/200, recon loss = 0.00226630\n",
            "epoch : 176/200, recon loss = 0.00234642\n",
            "epoch : 177/200, recon loss = 0.00233884\n",
            "epoch : 178/200, recon loss = 0.00233287\n",
            "epoch : 179/200, recon loss = 0.00230329\n",
            "epoch : 180/200, recon loss = 0.00229421\n",
            "epoch : 181/200, recon loss = 0.00226529\n",
            "epoch : 182/200, recon loss = 0.00225463\n",
            "epoch : 183/200, recon loss = 0.00221869\n",
            "epoch : 184/200, recon loss = 0.00228770\n",
            "epoch : 185/200, recon loss = 0.00224054\n",
            "epoch : 186/200, recon loss = 0.00234184\n",
            "epoch : 187/200, recon loss = 0.00228063\n",
            "epoch : 188/200, recon loss = 0.00229618\n",
            "epoch : 189/200, recon loss = 0.00234750\n",
            "epoch : 190/200, recon loss = 0.00230442\n",
            "epoch : 191/200, recon loss = 0.00232226\n",
            "epoch : 192/200, recon loss = 0.00227061\n",
            "epoch : 193/200, recon loss = 0.00228813\n",
            "epoch : 194/200, recon loss = 0.00227572\n",
            "epoch : 195/200, recon loss = 0.00222775\n",
            "epoch : 196/200, recon loss = 0.00218679\n",
            "epoch : 197/200, recon loss = 0.00223982\n",
            "epoch : 198/200, recon loss = 0.00218159\n",
            "epoch : 199/200, recon loss = 0.00226458\n",
            "epoch : 200/200, recon loss = 0.00224014\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UICdguBRNzAI",
        "outputId": "8c73bd88-06c7-406f-c132-c0501241c969"
      },
      "source": [
        "model.eval()\n",
        "rmse_sum = 0\n",
        "\n",
        "filled_data = model(missed_data_noEm.to(device))\n",
        "filled_data = filled_data.cpu().detach().numpy()\n",
        "\n",
        "rmse_sum = rmse_error(test_data, filled_data, cols-num_embeddings, mask)\n",
        "\n",
        "print(rmse_sum)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1.1560837637956913\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "53SnBv08Djyw"
      },
      "source": [
        "## Autoencoder - with embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MHlvlNIQMhdB"
      },
      "source": [
        "model = Autoencoder(dim=cols).to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BWQO79SQMjMr"
      },
      "source": [
        "criterion = nn.MSELoss()\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VU-8toTp8uaf",
        "outputId": "fb29d285-9e82-4da8-fd10-f79359f53420"
      },
      "source": [
        "training(num_epochs, model, train_loader, criterion, optimizer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch : 1/200, recon loss = 0.02462699\n",
            "epoch : 2/200, recon loss = 0.02235821\n",
            "epoch : 3/200, recon loss = 0.01862748\n",
            "epoch : 4/200, recon loss = 0.01543054\n",
            "epoch : 5/200, recon loss = 0.01503805\n",
            "epoch : 6/200, recon loss = 0.01475612\n",
            "epoch : 7/200, recon loss = 0.01449390\n",
            "epoch : 8/200, recon loss = 0.01418221\n",
            "epoch : 9/200, recon loss = 0.01377359\n",
            "epoch : 10/200, recon loss = 0.01337756\n",
            "epoch : 11/200, recon loss = 0.01296752\n",
            "epoch : 12/200, recon loss = 0.01257062\n",
            "epoch : 13/200, recon loss = 0.01242249\n",
            "epoch : 14/200, recon loss = 0.01220449\n",
            "epoch : 15/200, recon loss = 0.01187324\n",
            "epoch : 16/200, recon loss = 0.01163460\n",
            "epoch : 17/200, recon loss = 0.01161640\n",
            "epoch : 18/200, recon loss = 0.01143047\n",
            "epoch : 19/200, recon loss = 0.01141854\n",
            "epoch : 20/200, recon loss = 0.01130841\n",
            "epoch : 21/200, recon loss = 0.01125204\n",
            "epoch : 22/200, recon loss = 0.01104177\n",
            "epoch : 23/200, recon loss = 0.01103405\n",
            "epoch : 24/200, recon loss = 0.01080708\n",
            "epoch : 25/200, recon loss = 0.01060850\n",
            "epoch : 26/200, recon loss = 0.01026164\n",
            "epoch : 27/200, recon loss = 0.00981712\n",
            "epoch : 28/200, recon loss = 0.00954586\n",
            "epoch : 29/200, recon loss = 0.00947938\n",
            "epoch : 30/200, recon loss = 0.00952424\n",
            "epoch : 31/200, recon loss = 0.00947761\n",
            "epoch : 32/200, recon loss = 0.00948457\n",
            "epoch : 33/200, recon loss = 0.00938222\n",
            "epoch : 34/200, recon loss = 0.00941109\n",
            "epoch : 35/200, recon loss = 0.00925815\n",
            "epoch : 36/200, recon loss = 0.00931108\n",
            "epoch : 37/200, recon loss = 0.00941178\n",
            "epoch : 38/200, recon loss = 0.00935604\n",
            "epoch : 39/200, recon loss = 0.00926853\n",
            "epoch : 40/200, recon loss = 0.00917071\n",
            "epoch : 41/200, recon loss = 0.00920460\n",
            "epoch : 42/200, recon loss = 0.00920066\n",
            "epoch : 43/200, recon loss = 0.00915973\n",
            "epoch : 44/200, recon loss = 0.00919501\n",
            "epoch : 45/200, recon loss = 0.00918443\n",
            "epoch : 46/200, recon loss = 0.00912805\n",
            "epoch : 47/200, recon loss = 0.00917617\n",
            "epoch : 48/200, recon loss = 0.00912234\n",
            "epoch : 49/200, recon loss = 0.00911362\n",
            "epoch : 50/200, recon loss = 0.00914372\n",
            "epoch : 51/200, recon loss = 0.00908275\n",
            "epoch : 52/200, recon loss = 0.00916205\n",
            "epoch : 53/200, recon loss = 0.00903487\n",
            "epoch : 54/200, recon loss = 0.00904979\n",
            "epoch : 55/200, recon loss = 0.00903259\n",
            "epoch : 56/200, recon loss = 0.00893210\n",
            "epoch : 57/200, recon loss = 0.00903584\n",
            "epoch : 58/200, recon loss = 0.00907020\n",
            "epoch : 59/200, recon loss = 0.00889975\n",
            "epoch : 60/200, recon loss = 0.00891423\n",
            "epoch : 61/200, recon loss = 0.00888835\n",
            "epoch : 62/200, recon loss = 0.00884640\n",
            "epoch : 63/200, recon loss = 0.00884991\n",
            "epoch : 64/200, recon loss = 0.00883464\n",
            "epoch : 65/200, recon loss = 0.00874033\n",
            "epoch : 66/200, recon loss = 0.00849342\n",
            "epoch : 67/200, recon loss = 0.00860803\n",
            "epoch : 68/200, recon loss = 0.00861911\n",
            "epoch : 69/200, recon loss = 0.00849395\n",
            "epoch : 70/200, recon loss = 0.00838853\n",
            "epoch : 71/200, recon loss = 0.00851806\n",
            "epoch : 72/200, recon loss = 0.00841693\n",
            "epoch : 73/200, recon loss = 0.00832423\n",
            "epoch : 74/200, recon loss = 0.00840511\n",
            "epoch : 75/200, recon loss = 0.00840695\n",
            "epoch : 76/200, recon loss = 0.00827532\n",
            "epoch : 77/200, recon loss = 0.00824986\n",
            "epoch : 78/200, recon loss = 0.00827097\n",
            "epoch : 79/200, recon loss = 0.00826076\n",
            "epoch : 80/200, recon loss = 0.00835802\n",
            "epoch : 81/200, recon loss = 0.00828620\n",
            "epoch : 82/200, recon loss = 0.00824827\n",
            "epoch : 83/200, recon loss = 0.00824308\n",
            "epoch : 84/200, recon loss = 0.00815728\n",
            "epoch : 85/200, recon loss = 0.00819288\n",
            "epoch : 86/200, recon loss = 0.00810587\n",
            "epoch : 87/200, recon loss = 0.00814267\n",
            "epoch : 88/200, recon loss = 0.00811354\n",
            "epoch : 89/200, recon loss = 0.00821356\n",
            "epoch : 90/200, recon loss = 0.00819361\n",
            "epoch : 91/200, recon loss = 0.00816482\n",
            "epoch : 92/200, recon loss = 0.00815447\n",
            "epoch : 93/200, recon loss = 0.00804176\n",
            "epoch : 94/200, recon loss = 0.00804832\n",
            "epoch : 95/200, recon loss = 0.00811293\n",
            "epoch : 96/200, recon loss = 0.00801699\n",
            "epoch : 97/200, recon loss = 0.00808026\n",
            "epoch : 98/200, recon loss = 0.00798273\n",
            "epoch : 99/200, recon loss = 0.00811468\n",
            "epoch : 100/200, recon loss = 0.00798728\n",
            "epoch : 101/200, recon loss = 0.00809939\n",
            "epoch : 102/200, recon loss = 0.00795306\n",
            "epoch : 103/200, recon loss = 0.00800519\n",
            "epoch : 104/200, recon loss = 0.00794681\n",
            "epoch : 105/200, recon loss = 0.00807104\n",
            "epoch : 106/200, recon loss = 0.00786865\n",
            "epoch : 107/200, recon loss = 0.00793924\n",
            "epoch : 108/200, recon loss = 0.00788103\n",
            "epoch : 109/200, recon loss = 0.00794954\n",
            "epoch : 110/200, recon loss = 0.00791131\n",
            "epoch : 111/200, recon loss = 0.00790524\n",
            "epoch : 112/200, recon loss = 0.00793596\n",
            "epoch : 113/200, recon loss = 0.00791548\n",
            "epoch : 114/200, recon loss = 0.00786770\n",
            "epoch : 115/200, recon loss = 0.00784341\n",
            "epoch : 116/200, recon loss = 0.00785853\n",
            "epoch : 117/200, recon loss = 0.00797379\n",
            "epoch : 118/200, recon loss = 0.00786290\n",
            "epoch : 119/200, recon loss = 0.00794955\n",
            "epoch : 120/200, recon loss = 0.00781784\n",
            "epoch : 121/200, recon loss = 0.00775221\n",
            "epoch : 122/200, recon loss = 0.00778242\n",
            "epoch : 123/200, recon loss = 0.00782076\n",
            "epoch : 124/200, recon loss = 0.00776110\n",
            "epoch : 125/200, recon loss = 0.00773938\n",
            "epoch : 126/200, recon loss = 0.00779080\n",
            "epoch : 127/200, recon loss = 0.00783833\n",
            "epoch : 128/200, recon loss = 0.00780842\n",
            "epoch : 129/200, recon loss = 0.00778433\n",
            "epoch : 130/200, recon loss = 0.00783878\n",
            "epoch : 131/200, recon loss = 0.00768622\n",
            "epoch : 132/200, recon loss = 0.00762573\n",
            "epoch : 133/200, recon loss = 0.00764479\n",
            "epoch : 134/200, recon loss = 0.00759381\n",
            "epoch : 135/200, recon loss = 0.00771142\n",
            "epoch : 136/200, recon loss = 0.00758814\n",
            "epoch : 137/200, recon loss = 0.00754386\n",
            "epoch : 138/200, recon loss = 0.00779751\n",
            "epoch : 139/200, recon loss = 0.00765887\n",
            "epoch : 140/200, recon loss = 0.00766696\n",
            "epoch : 141/200, recon loss = 0.00768512\n",
            "epoch : 142/200, recon loss = 0.00763591\n",
            "epoch : 143/200, recon loss = 0.00770543\n",
            "epoch : 144/200, recon loss = 0.00750795\n",
            "epoch : 145/200, recon loss = 0.00757236\n",
            "epoch : 146/200, recon loss = 0.00761061\n",
            "epoch : 147/200, recon loss = 0.00742506\n",
            "epoch : 148/200, recon loss = 0.00747640\n",
            "epoch : 149/200, recon loss = 0.00750437\n",
            "epoch : 150/200, recon loss = 0.00755549\n",
            "epoch : 151/200, recon loss = 0.00742539\n",
            "epoch : 152/200, recon loss = 0.00755042\n",
            "epoch : 153/200, recon loss = 0.00750514\n",
            "epoch : 154/200, recon loss = 0.00756225\n",
            "epoch : 155/200, recon loss = 0.00730778\n",
            "epoch : 156/200, recon loss = 0.00732570\n",
            "epoch : 157/200, recon loss = 0.00735011\n",
            "epoch : 158/200, recon loss = 0.00737500\n",
            "epoch : 159/200, recon loss = 0.00732432\n",
            "epoch : 160/200, recon loss = 0.00722528\n",
            "epoch : 161/200, recon loss = 0.00743025\n",
            "epoch : 162/200, recon loss = 0.00733753\n",
            "epoch : 163/200, recon loss = 0.00733324\n",
            "epoch : 164/200, recon loss = 0.00734304\n",
            "epoch : 165/200, recon loss = 0.00738944\n",
            "epoch : 166/200, recon loss = 0.00742122\n",
            "epoch : 167/200, recon loss = 0.00725200\n",
            "epoch : 168/200, recon loss = 0.00734607\n",
            "epoch : 169/200, recon loss = 0.00732896\n",
            "epoch : 170/200, recon loss = 0.00735913\n",
            "epoch : 171/200, recon loss = 0.00729164\n",
            "epoch : 172/200, recon loss = 0.00723489\n",
            "epoch : 173/200, recon loss = 0.00727279\n",
            "epoch : 174/200, recon loss = 0.00732470\n",
            "epoch : 175/200, recon loss = 0.00721102\n",
            "epoch : 176/200, recon loss = 0.00723209\n",
            "epoch : 177/200, recon loss = 0.00715017\n",
            "epoch : 178/200, recon loss = 0.00722512\n",
            "epoch : 179/200, recon loss = 0.00722883\n",
            "epoch : 180/200, recon loss = 0.00716355\n",
            "epoch : 181/200, recon loss = 0.00722190\n",
            "epoch : 182/200, recon loss = 0.00707275\n",
            "epoch : 183/200, recon loss = 0.00718705\n",
            "epoch : 184/200, recon loss = 0.00713989\n",
            "epoch : 185/200, recon loss = 0.00718942\n",
            "epoch : 186/200, recon loss = 0.00720409\n",
            "epoch : 187/200, recon loss = 0.00708039\n",
            "epoch : 188/200, recon loss = 0.00710923\n",
            "epoch : 189/200, recon loss = 0.00701262\n",
            "epoch : 190/200, recon loss = 0.00707492\n",
            "epoch : 191/200, recon loss = 0.00708343\n",
            "epoch : 192/200, recon loss = 0.00706145\n",
            "epoch : 193/200, recon loss = 0.00712369\n",
            "epoch : 194/200, recon loss = 0.00702630\n",
            "epoch : 195/200, recon loss = 0.00701115\n",
            "epoch : 196/200, recon loss = 0.00684515\n",
            "epoch : 197/200, recon loss = 0.00696891\n",
            "epoch : 198/200, recon loss = 0.00697485\n",
            "epoch : 199/200, recon loss = 0.00715696\n",
            "epoch : 200/200, recon loss = 0.00688790\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ti_oIYsMy_VX",
        "outputId": "f854591b-7d7c-4db0-faab-3ec7cd1523ae"
      },
      "source": [
        "model.eval()\n",
        "rmse_sum = 0\n",
        "\n",
        "filled_data = model(missed_data.to(device))\n",
        "filled_data = filled_data.cpu().detach().numpy()\n",
        "\n",
        "rmse_sum = rmse_error(test_data, filled_data, cols, mask)\n",
        "\n",
        "print(rmse_sum)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1.1978401246034787\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aOzbxC05OVm8"
      },
      "source": [
        "## Mean and Median Imputation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZyHymoFCOndq"
      },
      "source": [
        "### By category"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DrdgZGsp9c96",
        "outputId": "4927df2b-013d-4107-a02b-10a3cb074556"
      },
      "source": [
        "imputed = X_test_mean\n",
        "error = rmse_error(y_test, imputed, cols-num_embeddings, mask)\n",
        "print(error)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1.5219354232992517\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n33BfUPv9n4n",
        "outputId": "0f9b91d1-ccff-4917-dd9b-6412532c2329"
      },
      "source": [
        "imputed = X_test_median\n",
        "error = rmse_error(y_test, imputed, cols-num_embeddings, mask)\n",
        "print(error)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1.6189507261319402\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y4NmI9IPHlkh"
      },
      "source": [
        "### Across all foods"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rx6OuY_gHlkh",
        "outputId": "84d4778e-8dc0-41e5-ca65-0f77622058b9"
      },
      "source": [
        "imputed = X_test_mean_all\n",
        "error = rmse_error(y_test, imputed, cols-num_embeddings, mask)\n",
        "print(error)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2.0916638036635202\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "o_sHss96HzSZ",
        "outputId": "14230eec-9153-476d-8722-ec99b5dca16d"
      },
      "source": [
        "imputed = X_test_median_all\n",
        "error = rmse_error(y_test, imputed, cols-num_embeddings, mask)\n",
        "print(error)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2.178197307340046\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q2Eks0aAT6zF"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FYHt2bqD69IB"
      },
      "source": [
        "# 40 Columns"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yJn7xM9469Id"
      },
      "source": [
        "data_path = 'datasets/average datasets/byColumnsWithEmbeddings/40_columns_embeddings.csv'\n",
        "num_embeddings = 10\n",
        "num_columns = 40"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 320
        },
        "id": "i03rZynj69If",
        "outputId": "f8afd7de-d152-4329-fcc5-fe56e55925c0"
      },
      "source": [
        "dataset = pd.read_csv(data_path, index_col=0)\n",
        "dataset[dataset.notna().all(axis=1)].describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>fdc_id</th>\n",
              "      <th>1004</th>\n",
              "      <th>1087</th>\n",
              "      <th>1090</th>\n",
              "      <th>1092</th>\n",
              "      <th>1095</th>\n",
              "      <th>1091</th>\n",
              "      <th>1089</th>\n",
              "      <th>1098</th>\n",
              "      <th>1101</th>\n",
              "      <th>1051</th>\n",
              "      <th>1007</th>\n",
              "      <th>1002</th>\n",
              "      <th>1093</th>\n",
              "      <th>1079</th>\n",
              "      <th>1003</th>\n",
              "      <th>1009</th>\n",
              "      <th>1102</th>\n",
              "      <th>1265</th>\n",
              "      <th>1266</th>\n",
              "      <th>1264</th>\n",
              "      <th>1315</th>\n",
              "      <th>1316</th>\n",
              "      <th>1314</th>\n",
              "      <th>1094</th>\n",
              "      <th>1097</th>\n",
              "      <th>1137</th>\n",
              "      <th>1146</th>\n",
              "      <th>1300</th>\n",
              "      <th>1299</th>\n",
              "      <th>1404</th>\n",
              "      <th>1267</th>\n",
              "      <th>1273</th>\n",
              "      <th>1323</th>\n",
              "      <th>1167</th>\n",
              "      <th>1271</th>\n",
              "      <th>1304</th>\n",
              "      <th>1166</th>\n",
              "      <th>1175</th>\n",
              "      <th>1263</th>\n",
              "      <th>1165</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>1.310500e+04</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.00000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>4.807517e+05</td>\n",
              "      <td>1.824697</td>\n",
              "      <td>27.659061</td>\n",
              "      <td>14.595597</td>\n",
              "      <td>125.003892</td>\n",
              "      <td>0.384163</td>\n",
              "      <td>55.746662</td>\n",
              "      <td>0.476038</td>\n",
              "      <td>0.077504</td>\n",
              "      <td>0.149979</td>\n",
              "      <td>8.349769</td>\n",
              "      <td>0.472742</td>\n",
              "      <td>0.243665</td>\n",
              "      <td>133.394887</td>\n",
              "      <td>0.350355</td>\n",
              "      <td>1.942243</td>\n",
              "      <td>2.561062</td>\n",
              "      <td>42.944411</td>\n",
              "      <td>0.308332</td>\n",
              "      <td>0.116491</td>\n",
              "      <td>0.065003</td>\n",
              "      <td>0.605730</td>\n",
              "      <td>0.211003</td>\n",
              "      <td>0.023788</td>\n",
              "      <td>14.730179</td>\n",
              "      <td>2.983579</td>\n",
              "      <td>78.72934</td>\n",
              "      <td>12.225029</td>\n",
              "      <td>0.005267</td>\n",
              "      <td>0.005886</td>\n",
              "      <td>0.016719</td>\n",
              "      <td>0.004499</td>\n",
              "      <td>0.003870</td>\n",
              "      <td>0.002480</td>\n",
              "      <td>0.118839</td>\n",
              "      <td>0.002989</td>\n",
              "      <td>0.015767</td>\n",
              "      <td>0.012258</td>\n",
              "      <td>0.008632</td>\n",
              "      <td>0.038133</td>\n",
              "      <td>0.006928</td>\n",
              "      <td>21488.189968</td>\n",
              "      <td>176.402583</td>\n",
              "      <td>0.119704</td>\n",
              "      <td>-0.093912</td>\n",
              "      <td>-0.048493</td>\n",
              "      <td>-0.007831</td>\n",
              "      <td>0.051473</td>\n",
              "      <td>0.023110</td>\n",
              "      <td>0.063160</td>\n",
              "      <td>0.043647</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.544018e+05</td>\n",
              "      <td>7.838466</td>\n",
              "      <td>99.924214</td>\n",
              "      <td>45.754175</td>\n",
              "      <td>382.322293</td>\n",
              "      <td>1.055340</td>\n",
              "      <td>153.432660</td>\n",
              "      <td>1.483360</td>\n",
              "      <td>0.264892</td>\n",
              "      <td>0.515558</td>\n",
              "      <td>23.549114</td>\n",
              "      <td>4.416751</td>\n",
              "      <td>0.999601</td>\n",
              "      <td>1736.846193</td>\n",
              "      <td>1.186155</td>\n",
              "      <td>6.583455</td>\n",
              "      <td>9.446549</td>\n",
              "      <td>187.200801</td>\n",
              "      <td>1.605577</td>\n",
              "      <td>0.597369</td>\n",
              "      <td>0.559044</td>\n",
              "      <td>4.440086</td>\n",
              "      <td>1.941342</td>\n",
              "      <td>0.148635</td>\n",
              "      <td>56.459805</td>\n",
              "      <td>11.848737</td>\n",
              "      <td>302.68343</td>\n",
              "      <td>49.040700</td>\n",
              "      <td>0.030796</td>\n",
              "      <td>0.043100</td>\n",
              "      <td>0.223785</td>\n",
              "      <td>0.039142</td>\n",
              "      <td>0.065227</td>\n",
              "      <td>0.015745</td>\n",
              "      <td>0.828604</td>\n",
              "      <td>0.017050</td>\n",
              "      <td>0.106190</td>\n",
              "      <td>0.071481</td>\n",
              "      <td>0.054902</td>\n",
              "      <td>0.974266</td>\n",
              "      <td>0.056734</td>\n",
              "      <td>254447.573253</td>\n",
              "      <td>3958.010969</td>\n",
              "      <td>1.112342</td>\n",
              "      <td>0.694392</td>\n",
              "      <td>0.598293</td>\n",
              "      <td>0.511537</td>\n",
              "      <td>0.453723</td>\n",
              "      <td>0.451006</td>\n",
              "      <td>0.417450</td>\n",
              "      <td>0.426074</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>3.198770e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-139555.939272</td>\n",
              "      <td>-7413.329674</td>\n",
              "      <td>-3.349638</td>\n",
              "      <td>-1.585647</td>\n",
              "      <td>-1.527264</td>\n",
              "      <td>-1.442526</td>\n",
              "      <td>-1.189873</td>\n",
              "      <td>-1.064457</td>\n",
              "      <td>-1.076825</td>\n",
              "      <td>-0.829557</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>3.256480e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-133692.806088</td>\n",
              "      <td>-2328.507401</td>\n",
              "      <td>-0.796369</td>\n",
              "      <td>-0.667107</td>\n",
              "      <td>-0.477074</td>\n",
              "      <td>-0.343775</td>\n",
              "      <td>-0.236110</td>\n",
              "      <td>-0.286783</td>\n",
              "      <td>-0.177237</td>\n",
              "      <td>-0.292813</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>3.314100e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-127823.125712</td>\n",
              "      <td>-153.274226</td>\n",
              "      <td>-0.002647</td>\n",
              "      <td>-0.011945</td>\n",
              "      <td>-0.071966</td>\n",
              "      <td>-0.022448</td>\n",
              "      <td>0.012030</td>\n",
              "      <td>0.040891</td>\n",
              "      <td>0.040572</td>\n",
              "      <td>-0.032707</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>7.474680e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>288262.097477</td>\n",
              "      <td>3032.870912</td>\n",
              "      <td>0.983044</td>\n",
              "      <td>0.441140</td>\n",
              "      <td>0.323270</td>\n",
              "      <td>0.289041</td>\n",
              "      <td>0.257717</td>\n",
              "      <td>0.227563</td>\n",
              "      <td>0.309656</td>\n",
              "      <td>0.274336</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.105897e+06</td>\n",
              "      <td>99.400000</td>\n",
              "      <td>1390.000000</td>\n",
              "      <td>386.000000</td>\n",
              "      <td>2520.000000</td>\n",
              "      <td>7.660000</td>\n",
              "      <td>997.000000</td>\n",
              "      <td>12.700000</td>\n",
              "      <td>1.920000</td>\n",
              "      <td>7.870000</td>\n",
              "      <td>96.700000</td>\n",
              "      <td>99.600000</td>\n",
              "      <td>13.200000</td>\n",
              "      <td>40700.000000</td>\n",
              "      <td>11.500000</td>\n",
              "      <td>79.900000</td>\n",
              "      <td>43.900000</td>\n",
              "      <td>1790.000000</td>\n",
              "      <td>22.800000</td>\n",
              "      <td>8.610000</td>\n",
              "      <td>17.000000</td>\n",
              "      <td>73.200000</td>\n",
              "      <td>52.600000</td>\n",
              "      <td>3.400000</td>\n",
              "      <td>311.000000</td>\n",
              "      <td>93.000000</td>\n",
              "      <td>1710.00000</td>\n",
              "      <td>449.000000</td>\n",
              "      <td>0.485000</td>\n",
              "      <td>0.850000</td>\n",
              "      <td>7.950000</td>\n",
              "      <td>0.774000</td>\n",
              "      <td>1.830000</td>\n",
              "      <td>0.303000</td>\n",
              "      <td>14.000000</td>\n",
              "      <td>0.222000</td>\n",
              "      <td>1.930000</td>\n",
              "      <td>1.860000</td>\n",
              "      <td>0.989000</td>\n",
              "      <td>42.400000</td>\n",
              "      <td>1.920000</td>\n",
              "      <td>646711.616853</td>\n",
              "      <td>7635.496843</td>\n",
              "      <td>2.647689</td>\n",
              "      <td>1.869347</td>\n",
              "      <td>1.616847</td>\n",
              "      <td>2.041569</td>\n",
              "      <td>1.700620</td>\n",
              "      <td>1.764974</td>\n",
              "      <td>1.547869</td>\n",
              "      <td>1.348304</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             fdc_id          1004  ...             8             9\n",
              "count  1.310500e+04  13105.000000  ...  13105.000000  13105.000000\n",
              "mean   4.807517e+05      1.824697  ...      0.063160      0.043647\n",
              "std    2.544018e+05      7.838466  ...      0.417450      0.426074\n",
              "min    3.198770e+05      0.000000  ...     -1.076825     -0.829557\n",
              "25%    3.256480e+05      0.000000  ...     -0.177237     -0.292813\n",
              "50%    3.314100e+05      0.000000  ...      0.040572     -0.032707\n",
              "75%    7.474680e+05      0.000000  ...      0.309656      0.274336\n",
              "max    1.105897e+06     99.400000  ...      1.547869      1.348304\n",
              "\n",
              "[8 rows x 51 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n-0aV0Ps69If"
      },
      "source": [
        "data = dataset\n",
        "train_data, test_data = train_test_split(data, test_size=test_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 320
        },
        "id": "XFJI5r0O69Ig",
        "outputId": "764e6239-f857-4416-af3f-4b39a43dd10e"
      },
      "source": [
        "train_data.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>fdc_id</th>\n",
              "      <th>1004</th>\n",
              "      <th>1087</th>\n",
              "      <th>1090</th>\n",
              "      <th>1092</th>\n",
              "      <th>1095</th>\n",
              "      <th>1091</th>\n",
              "      <th>1089</th>\n",
              "      <th>1098</th>\n",
              "      <th>1101</th>\n",
              "      <th>1051</th>\n",
              "      <th>1007</th>\n",
              "      <th>1002</th>\n",
              "      <th>1093</th>\n",
              "      <th>1079</th>\n",
              "      <th>1003</th>\n",
              "      <th>1009</th>\n",
              "      <th>1102</th>\n",
              "      <th>1265</th>\n",
              "      <th>1266</th>\n",
              "      <th>1264</th>\n",
              "      <th>1315</th>\n",
              "      <th>1316</th>\n",
              "      <th>1314</th>\n",
              "      <th>1094</th>\n",
              "      <th>1097</th>\n",
              "      <th>1137</th>\n",
              "      <th>1146</th>\n",
              "      <th>1300</th>\n",
              "      <th>1299</th>\n",
              "      <th>1404</th>\n",
              "      <th>1267</th>\n",
              "      <th>1273</th>\n",
              "      <th>1323</th>\n",
              "      <th>1167</th>\n",
              "      <th>1271</th>\n",
              "      <th>1304</th>\n",
              "      <th>1166</th>\n",
              "      <th>1175</th>\n",
              "      <th>1263</th>\n",
              "      <th>1165</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>1.048400e+04</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>4.837058e+05</td>\n",
              "      <td>1.887024</td>\n",
              "      <td>27.318962</td>\n",
              "      <td>14.395221</td>\n",
              "      <td>123.668447</td>\n",
              "      <td>0.380351</td>\n",
              "      <td>55.097482</td>\n",
              "      <td>0.471182</td>\n",
              "      <td>0.076515</td>\n",
              "      <td>0.148187</td>\n",
              "      <td>8.491958</td>\n",
              "      <td>0.488262</td>\n",
              "      <td>0.250330</td>\n",
              "      <td>122.514594</td>\n",
              "      <td>0.342884</td>\n",
              "      <td>1.927489</td>\n",
              "      <td>2.517378</td>\n",
              "      <td>42.457030</td>\n",
              "      <td>0.315906</td>\n",
              "      <td>0.118237</td>\n",
              "      <td>0.069707</td>\n",
              "      <td>0.614797</td>\n",
              "      <td>0.199738</td>\n",
              "      <td>0.024070</td>\n",
              "      <td>14.494945</td>\n",
              "      <td>2.935969</td>\n",
              "      <td>77.261351</td>\n",
              "      <td>11.975630</td>\n",
              "      <td>0.005308</td>\n",
              "      <td>0.006134</td>\n",
              "      <td>0.016559</td>\n",
              "      <td>0.004545</td>\n",
              "      <td>0.003895</td>\n",
              "      <td>0.002414</td>\n",
              "      <td>0.122711</td>\n",
              "      <td>0.003012</td>\n",
              "      <td>0.015654</td>\n",
              "      <td>0.012506</td>\n",
              "      <td>0.008658</td>\n",
              "      <td>0.044606</td>\n",
              "      <td>0.007359</td>\n",
              "      <td>24442.898852</td>\n",
              "      <td>170.589040</td>\n",
              "      <td>0.119443</td>\n",
              "      <td>-0.093123</td>\n",
              "      <td>-0.052680</td>\n",
              "      <td>-0.011590</td>\n",
              "      <td>0.052038</td>\n",
              "      <td>0.022672</td>\n",
              "      <td>0.063588</td>\n",
              "      <td>0.045477</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.553333e+05</td>\n",
              "      <td>8.019547</td>\n",
              "      <td>99.862708</td>\n",
              "      <td>45.531646</td>\n",
              "      <td>380.806938</td>\n",
              "      <td>1.051379</td>\n",
              "      <td>152.479658</td>\n",
              "      <td>1.480380</td>\n",
              "      <td>0.263713</td>\n",
              "      <td>0.515968</td>\n",
              "      <td>23.679751</td>\n",
              "      <td>4.518943</td>\n",
              "      <td>1.014003</td>\n",
              "      <td>1618.739582</td>\n",
              "      <td>1.172874</td>\n",
              "      <td>6.563284</td>\n",
              "      <td>9.370949</td>\n",
              "      <td>186.008101</td>\n",
              "      <td>1.641078</td>\n",
              "      <td>0.605666</td>\n",
              "      <td>0.602119</td>\n",
              "      <td>4.534074</td>\n",
              "      <td>1.776330</td>\n",
              "      <td>0.150727</td>\n",
              "      <td>56.099160</td>\n",
              "      <td>11.764307</td>\n",
              "      <td>299.832117</td>\n",
              "      <td>48.511157</td>\n",
              "      <td>0.030991</td>\n",
              "      <td>0.044737</td>\n",
              "      <td>0.216067</td>\n",
              "      <td>0.039463</td>\n",
              "      <td>0.066432</td>\n",
              "      <td>0.015003</td>\n",
              "      <td>0.842898</td>\n",
              "      <td>0.017028</td>\n",
              "      <td>0.104826</td>\n",
              "      <td>0.073419</td>\n",
              "      <td>0.055229</td>\n",
              "      <td>1.087993</td>\n",
              "      <td>0.059635</td>\n",
              "      <td>255379.454754</td>\n",
              "      <td>3933.834962</td>\n",
              "      <td>1.111404</td>\n",
              "      <td>0.692907</td>\n",
              "      <td>0.599123</td>\n",
              "      <td>0.506658</td>\n",
              "      <td>0.454648</td>\n",
              "      <td>0.452795</td>\n",
              "      <td>0.421476</td>\n",
              "      <td>0.425888</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>3.198770e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-139555.939272</td>\n",
              "      <td>-7413.329674</td>\n",
              "      <td>-3.349638</td>\n",
              "      <td>-1.577342</td>\n",
              "      <td>-1.527264</td>\n",
              "      <td>-1.442526</td>\n",
              "      <td>-1.189873</td>\n",
              "      <td>-1.006986</td>\n",
              "      <td>-1.076825</td>\n",
              "      <td>-0.829557</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>3.256918e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-133648.238488</td>\n",
              "      <td>-2307.662167</td>\n",
              "      <td>-0.798023</td>\n",
              "      <td>-0.663338</td>\n",
              "      <td>-0.477425</td>\n",
              "      <td>-0.344402</td>\n",
              "      <td>-0.236082</td>\n",
              "      <td>-0.286880</td>\n",
              "      <td>-0.177357</td>\n",
              "      <td>-0.292446</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>3.314685e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-127763.532461</td>\n",
              "      <td>-159.406808</td>\n",
              "      <td>0.006781</td>\n",
              "      <td>-0.011148</td>\n",
              "      <td>-0.072708</td>\n",
              "      <td>-0.023806</td>\n",
              "      <td>0.011936</td>\n",
              "      <td>0.040795</td>\n",
              "      <td>0.041052</td>\n",
              "      <td>-0.032263</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>7.476032e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>288399.875032</td>\n",
              "      <td>3016.236880</td>\n",
              "      <td>0.981714</td>\n",
              "      <td>0.441011</td>\n",
              "      <td>0.322800</td>\n",
              "      <td>0.288232</td>\n",
              "      <td>0.261384</td>\n",
              "      <td>0.227468</td>\n",
              "      <td>0.311777</td>\n",
              "      <td>0.276468</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.105896e+06</td>\n",
              "      <td>99.200000</td>\n",
              "      <td>1380.000000</td>\n",
              "      <td>386.000000</td>\n",
              "      <td>2520.000000</td>\n",
              "      <td>7.660000</td>\n",
              "      <td>997.000000</td>\n",
              "      <td>12.700000</td>\n",
              "      <td>1.920000</td>\n",
              "      <td>7.870000</td>\n",
              "      <td>96.000000</td>\n",
              "      <td>99.600000</td>\n",
              "      <td>13.200000</td>\n",
              "      <td>40700.000000</td>\n",
              "      <td>11.500000</td>\n",
              "      <td>79.900000</td>\n",
              "      <td>43.900000</td>\n",
              "      <td>1730.000000</td>\n",
              "      <td>22.800000</td>\n",
              "      <td>8.610000</td>\n",
              "      <td>17.000000</td>\n",
              "      <td>73.200000</td>\n",
              "      <td>52.400000</td>\n",
              "      <td>3.400000</td>\n",
              "      <td>311.000000</td>\n",
              "      <td>93.000000</td>\n",
              "      <td>1710.000000</td>\n",
              "      <td>449.000000</td>\n",
              "      <td>0.485000</td>\n",
              "      <td>0.850000</td>\n",
              "      <td>7.950000</td>\n",
              "      <td>0.774000</td>\n",
              "      <td>1.830000</td>\n",
              "      <td>0.303000</td>\n",
              "      <td>14.000000</td>\n",
              "      <td>0.222000</td>\n",
              "      <td>1.930000</td>\n",
              "      <td>1.860000</td>\n",
              "      <td>0.989000</td>\n",
              "      <td>42.400000</td>\n",
              "      <td>1.920000</td>\n",
              "      <td>646710.598170</td>\n",
              "      <td>7635.496843</td>\n",
              "      <td>2.647689</td>\n",
              "      <td>1.869347</td>\n",
              "      <td>1.616847</td>\n",
              "      <td>2.028810</td>\n",
              "      <td>1.700620</td>\n",
              "      <td>1.764974</td>\n",
              "      <td>1.547869</td>\n",
              "      <td>1.228624</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             fdc_id          1004  ...             8             9\n",
              "count  1.048400e+04  10484.000000  ...  10484.000000  10484.000000\n",
              "mean   4.837058e+05      1.887024  ...      0.063588      0.045477\n",
              "std    2.553333e+05      8.019547  ...      0.421476      0.425888\n",
              "min    3.198770e+05      0.000000  ...     -1.076825     -0.829557\n",
              "25%    3.256918e+05      0.000000  ...     -0.177357     -0.292446\n",
              "50%    3.314685e+05      0.000000  ...      0.041052     -0.032263\n",
              "75%    7.476032e+05      0.000000  ...      0.311777      0.276468\n",
              "max    1.105896e+06     99.200000  ...      1.547869      1.228624\n",
              "\n",
              "[8 rows x 51 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 320
        },
        "id": "fQf5N97x69Ih",
        "outputId": "6d616e33-7d7b-487a-9443-ebca257af0ce"
      },
      "source": [
        "test_data.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>fdc_id</th>\n",
              "      <th>1004</th>\n",
              "      <th>1087</th>\n",
              "      <th>1090</th>\n",
              "      <th>1092</th>\n",
              "      <th>1095</th>\n",
              "      <th>1091</th>\n",
              "      <th>1089</th>\n",
              "      <th>1098</th>\n",
              "      <th>1101</th>\n",
              "      <th>1051</th>\n",
              "      <th>1007</th>\n",
              "      <th>1002</th>\n",
              "      <th>1093</th>\n",
              "      <th>1079</th>\n",
              "      <th>1003</th>\n",
              "      <th>1009</th>\n",
              "      <th>1102</th>\n",
              "      <th>1265</th>\n",
              "      <th>1266</th>\n",
              "      <th>1264</th>\n",
              "      <th>1315</th>\n",
              "      <th>1316</th>\n",
              "      <th>1314</th>\n",
              "      <th>1094</th>\n",
              "      <th>1097</th>\n",
              "      <th>1137</th>\n",
              "      <th>1146</th>\n",
              "      <th>1300</th>\n",
              "      <th>1299</th>\n",
              "      <th>1404</th>\n",
              "      <th>1267</th>\n",
              "      <th>1273</th>\n",
              "      <th>1323</th>\n",
              "      <th>1167</th>\n",
              "      <th>1271</th>\n",
              "      <th>1304</th>\n",
              "      <th>1166</th>\n",
              "      <th>1175</th>\n",
              "      <th>1263</th>\n",
              "      <th>1165</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>2.621000e+03</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>4.689355e+05</td>\n",
              "      <td>1.575387</td>\n",
              "      <td>29.019458</td>\n",
              "      <td>15.397100</td>\n",
              "      <td>130.345670</td>\n",
              "      <td>0.399412</td>\n",
              "      <td>58.343380</td>\n",
              "      <td>0.495464</td>\n",
              "      <td>0.081459</td>\n",
              "      <td>0.157148</td>\n",
              "      <td>7.781011</td>\n",
              "      <td>0.410660</td>\n",
              "      <td>0.217005</td>\n",
              "      <td>176.916063</td>\n",
              "      <td>0.380237</td>\n",
              "      <td>2.001263</td>\n",
              "      <td>2.735799</td>\n",
              "      <td>44.893934</td>\n",
              "      <td>0.278035</td>\n",
              "      <td>0.109507</td>\n",
              "      <td>0.046184</td>\n",
              "      <td>0.569463</td>\n",
              "      <td>0.256064</td>\n",
              "      <td>0.022656</td>\n",
              "      <td>15.671118</td>\n",
              "      <td>3.174018</td>\n",
              "      <td>84.601297</td>\n",
              "      <td>13.222625</td>\n",
              "      <td>0.005103</td>\n",
              "      <td>0.004891</td>\n",
              "      <td>0.017362</td>\n",
              "      <td>0.004317</td>\n",
              "      <td>0.003768</td>\n",
              "      <td>0.002741</td>\n",
              "      <td>0.103353</td>\n",
              "      <td>0.002899</td>\n",
              "      <td>0.016219</td>\n",
              "      <td>0.011265</td>\n",
              "      <td>0.008528</td>\n",
              "      <td>0.012241</td>\n",
              "      <td>0.005205</td>\n",
              "      <td>9669.354429</td>\n",
              "      <td>199.656755</td>\n",
              "      <td>0.120749</td>\n",
              "      <td>-0.097069</td>\n",
              "      <td>-0.031746</td>\n",
              "      <td>0.007202</td>\n",
              "      <td>0.049214</td>\n",
              "      <td>0.024860</td>\n",
              "      <td>0.061447</td>\n",
              "      <td>0.036329</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.503407e+05</td>\n",
              "      <td>7.063825</td>\n",
              "      <td>100.177400</td>\n",
              "      <td>46.633877</td>\n",
              "      <td>388.352177</td>\n",
              "      <td>1.071102</td>\n",
              "      <td>157.189722</td>\n",
              "      <td>1.495347</td>\n",
              "      <td>0.269571</td>\n",
              "      <td>0.513949</td>\n",
              "      <td>23.014814</td>\n",
              "      <td>3.982042</td>\n",
              "      <td>0.939498</td>\n",
              "      <td>2145.026530</td>\n",
              "      <td>1.237634</td>\n",
              "      <td>6.664468</td>\n",
              "      <td>9.742956</td>\n",
              "      <td>191.921278</td>\n",
              "      <td>1.454837</td>\n",
              "      <td>0.563017</td>\n",
              "      <td>0.334787</td>\n",
              "      <td>4.042945</td>\n",
              "      <td>2.494442</td>\n",
              "      <td>0.139977</td>\n",
              "      <td>57.881263</td>\n",
              "      <td>12.181039</td>\n",
              "      <td>313.819887</td>\n",
              "      <td>51.101416</td>\n",
              "      <td>0.030006</td>\n",
              "      <td>0.035801</td>\n",
              "      <td>0.252354</td>\n",
              "      <td>0.037836</td>\n",
              "      <td>0.060177</td>\n",
              "      <td>0.018420</td>\n",
              "      <td>0.768730</td>\n",
              "      <td>0.017142</td>\n",
              "      <td>0.111496</td>\n",
              "      <td>0.063141</td>\n",
              "      <td>0.053586</td>\n",
              "      <td>0.101575</td>\n",
              "      <td>0.043194</td>\n",
              "      <td>250385.040634</td>\n",
              "      <td>4053.954334</td>\n",
              "      <td>1.116300</td>\n",
              "      <td>0.700426</td>\n",
              "      <td>0.594781</td>\n",
              "      <td>0.530437</td>\n",
              "      <td>0.450082</td>\n",
              "      <td>0.443857</td>\n",
              "      <td>0.401017</td>\n",
              "      <td>0.426822</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>3.198820e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-139550.845832</td>\n",
              "      <td>-7412.348752</td>\n",
              "      <td>-2.571016</td>\n",
              "      <td>-1.585647</td>\n",
              "      <td>-1.327981</td>\n",
              "      <td>-1.442505</td>\n",
              "      <td>-1.189847</td>\n",
              "      <td>-1.064457</td>\n",
              "      <td>-1.061308</td>\n",
              "      <td>-0.808859</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>3.254200e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-133925.066958</td>\n",
              "      <td>-2397.174257</td>\n",
              "      <td>-0.788091</td>\n",
              "      <td>-0.671897</td>\n",
              "      <td>-0.475166</td>\n",
              "      <td>-0.334340</td>\n",
              "      <td>-0.238444</td>\n",
              "      <td>-0.265503</td>\n",
              "      <td>-0.170750</td>\n",
              "      <td>-0.294423</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>3.309040e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-128338.581850</td>\n",
              "      <td>-98.834426</td>\n",
              "      <td>-0.018865</td>\n",
              "      <td>-0.019108</td>\n",
              "      <td>-0.066464</td>\n",
              "      <td>-0.015501</td>\n",
              "      <td>0.012356</td>\n",
              "      <td>0.048059</td>\n",
              "      <td>0.028430</td>\n",
              "      <td>-0.034130</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>7.469940e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>287779.239352</td>\n",
              "      <td>3128.023695</td>\n",
              "      <td>0.990449</td>\n",
              "      <td>0.441551</td>\n",
              "      <td>0.339117</td>\n",
              "      <td>0.305718</td>\n",
              "      <td>0.247664</td>\n",
              "      <td>0.227888</td>\n",
              "      <td>0.300948</td>\n",
              "      <td>0.264765</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.105897e+06</td>\n",
              "      <td>99.400000</td>\n",
              "      <td>1390.000000</td>\n",
              "      <td>358.000000</td>\n",
              "      <td>1810.000000</td>\n",
              "      <td>7.460000</td>\n",
              "      <td>997.000000</td>\n",
              "      <td>9.890000</td>\n",
              "      <td>1.830000</td>\n",
              "      <td>3.900000</td>\n",
              "      <td>96.700000</td>\n",
              "      <td>99.300000</td>\n",
              "      <td>13.000000</td>\n",
              "      <td>40300.000000</td>\n",
              "      <td>11.200000</td>\n",
              "      <td>48.100000</td>\n",
              "      <td>43.200000</td>\n",
              "      <td>1790.000000</td>\n",
              "      <td>22.600000</td>\n",
              "      <td>6.860000</td>\n",
              "      <td>7.100000</td>\n",
              "      <td>69.900000</td>\n",
              "      <td>52.600000</td>\n",
              "      <td>3.320000</td>\n",
              "      <td>284.000000</td>\n",
              "      <td>79.000000</td>\n",
              "      <td>1650.000000</td>\n",
              "      <td>370.000000</td>\n",
              "      <td>0.435000</td>\n",
              "      <td>0.765000</td>\n",
              "      <td>7.680000</td>\n",
              "      <td>0.726000</td>\n",
              "      <td>1.790000</td>\n",
              "      <td>0.264000</td>\n",
              "      <td>13.400000</td>\n",
              "      <td>0.194000</td>\n",
              "      <td>1.770000</td>\n",
              "      <td>1.590000</td>\n",
              "      <td>0.787000</td>\n",
              "      <td>2.140000</td>\n",
              "      <td>1.020000</td>\n",
              "      <td>646711.616853</td>\n",
              "      <td>7630.592063</td>\n",
              "      <td>2.647168</td>\n",
              "      <td>1.869328</td>\n",
              "      <td>1.597387</td>\n",
              "      <td>2.041569</td>\n",
              "      <td>1.592385</td>\n",
              "      <td>1.742486</td>\n",
              "      <td>1.484113</td>\n",
              "      <td>1.348304</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             fdc_id         1004  ...            8            9\n",
              "count  2.621000e+03  2621.000000  ...  2621.000000  2621.000000\n",
              "mean   4.689355e+05     1.575387  ...     0.061447     0.036329\n",
              "std    2.503407e+05     7.063825  ...     0.401017     0.426822\n",
              "min    3.198820e+05     0.000000  ...    -1.061308    -0.808859\n",
              "25%    3.254200e+05     0.000000  ...    -0.170750    -0.294423\n",
              "50%    3.309040e+05     0.000000  ...     0.028430    -0.034130\n",
              "75%    7.469940e+05     0.000000  ...     0.300948     0.264765\n",
              "max    1.105897e+06    99.400000  ...     1.484113     1.348304\n",
              "\n",
              "[8 rows x 51 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wpo2ho1s69Ih",
        "outputId": "f51dde91-c297-42d2-ecd0-0cdb5521e40a"
      },
      "source": [
        "indices = test_data['fdc_id']\n",
        "indices"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1309      322529\n",
              "3496      325966\n",
              "8951      335679\n",
              "12777    1105515\n",
              "9704      747274\n",
              "          ...   \n",
              "4254      327175\n",
              "4402      327370\n",
              "8410      335104\n",
              "5694      329579\n",
              "5764      329677\n",
              "Name: fdc_id, Length: 2621, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oUMlzfoc69Ii"
      },
      "source": [
        "del train_data['fdc_id']\n",
        "del test_data['fdc_id']\n",
        "columns = train_data.columns"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1BWno_Ut69Ij",
        "outputId": "67fe055d-9e4a-4ce2-a248-391ab1c65fbf"
      },
      "source": [
        "train_data = train_data.to_numpy()\n",
        "test_data.fillna(0, inplace=True)\n",
        "test_data = test_data.to_numpy()\n",
        "data = dataset.values\n",
        "rows, cols = data.shape\n",
        "cols -= 1\n",
        "\n",
        "y_test = test_data.copy()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pandas/core/frame.py:4327: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  downcast=downcast,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 320
        },
        "id": "oTIpaMg_69Ij",
        "outputId": "dfc2e6e9-9e94-4fb9-a986-ea1384236bd8"
      },
      "source": [
        "missed_data, missed_data_train, mask = missing_method(test_data, train_data, num_embeddings)\n",
        "pd.DataFrame(missed_data).describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "      <th>30</th>\n",
              "      <th>31</th>\n",
              "      <th>32</th>\n",
              "      <th>33</th>\n",
              "      <th>34</th>\n",
              "      <th>35</th>\n",
              "      <th>36</th>\n",
              "      <th>37</th>\n",
              "      <th>38</th>\n",
              "      <th>39</th>\n",
              "      <th>40</th>\n",
              "      <th>41</th>\n",
              "      <th>42</th>\n",
              "      <th>43</th>\n",
              "      <th>44</th>\n",
              "      <th>45</th>\n",
              "      <th>46</th>\n",
              "      <th>47</th>\n",
              "      <th>48</th>\n",
              "      <th>49</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>2081.000000</td>\n",
              "      <td>2098.000000</td>\n",
              "      <td>2065.000000</td>\n",
              "      <td>2083.000000</td>\n",
              "      <td>2080.000000</td>\n",
              "      <td>2086.000000</td>\n",
              "      <td>2057.000000</td>\n",
              "      <td>2087.000000</td>\n",
              "      <td>2109.000000</td>\n",
              "      <td>2121.000000</td>\n",
              "      <td>2092.000000</td>\n",
              "      <td>2113.000000</td>\n",
              "      <td>2095.000000</td>\n",
              "      <td>2079.000000</td>\n",
              "      <td>2109.000000</td>\n",
              "      <td>2108.000000</td>\n",
              "      <td>2129.000000</td>\n",
              "      <td>2090.000000</td>\n",
              "      <td>2097.000000</td>\n",
              "      <td>2129.000000</td>\n",
              "      <td>2044.000000</td>\n",
              "      <td>2104.000000</td>\n",
              "      <td>2123.000000</td>\n",
              "      <td>2054.000000</td>\n",
              "      <td>2100.000000</td>\n",
              "      <td>2085.000000</td>\n",
              "      <td>2086.000000</td>\n",
              "      <td>2074.000000</td>\n",
              "      <td>2104.000000</td>\n",
              "      <td>2121.00000</td>\n",
              "      <td>2099.000000</td>\n",
              "      <td>2097.000000</td>\n",
              "      <td>2103.000000</td>\n",
              "      <td>2109.000000</td>\n",
              "      <td>2086.000000</td>\n",
              "      <td>2109.000000</td>\n",
              "      <td>2110.000000</td>\n",
              "      <td>2130.000000</td>\n",
              "      <td>2122.000000</td>\n",
              "      <td>2092.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>1.625132</td>\n",
              "      <td>29.615825</td>\n",
              "      <td>15.899952</td>\n",
              "      <td>124.521843</td>\n",
              "      <td>0.382135</td>\n",
              "      <td>57.976510</td>\n",
              "      <td>0.516393</td>\n",
              "      <td>0.080287</td>\n",
              "      <td>0.156195</td>\n",
              "      <td>7.633182</td>\n",
              "      <td>0.440550</td>\n",
              "      <td>0.235542</td>\n",
              "      <td>185.649642</td>\n",
              "      <td>0.373641</td>\n",
              "      <td>1.932248</td>\n",
              "      <td>2.718221</td>\n",
              "      <td>44.560263</td>\n",
              "      <td>0.288953</td>\n",
              "      <td>0.101189</td>\n",
              "      <td>0.048570</td>\n",
              "      <td>0.589414</td>\n",
              "      <td>0.248271</td>\n",
              "      <td>0.022463</td>\n",
              "      <td>16.201558</td>\n",
              "      <td>3.297667</td>\n",
              "      <td>88.138609</td>\n",
              "      <td>12.980585</td>\n",
              "      <td>0.005081</td>\n",
              "      <td>0.004599</td>\n",
              "      <td>0.01913</td>\n",
              "      <td>0.004288</td>\n",
              "      <td>0.003748</td>\n",
              "      <td>0.002780</td>\n",
              "      <td>0.106002</td>\n",
              "      <td>0.002992</td>\n",
              "      <td>0.015840</td>\n",
              "      <td>0.011400</td>\n",
              "      <td>0.007671</td>\n",
              "      <td>0.013134</td>\n",
              "      <td>0.005845</td>\n",
              "      <td>9669.354429</td>\n",
              "      <td>199.656755</td>\n",
              "      <td>0.120749</td>\n",
              "      <td>-0.097069</td>\n",
              "      <td>-0.031746</td>\n",
              "      <td>0.007202</td>\n",
              "      <td>0.049214</td>\n",
              "      <td>0.024860</td>\n",
              "      <td>0.061447</td>\n",
              "      <td>0.036329</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>7.297463</td>\n",
              "      <td>101.869652</td>\n",
              "      <td>47.249550</td>\n",
              "      <td>380.291582</td>\n",
              "      <td>1.058164</td>\n",
              "      <td>155.631801</td>\n",
              "      <td>1.532044</td>\n",
              "      <td>0.268996</td>\n",
              "      <td>0.515979</td>\n",
              "      <td>22.701211</td>\n",
              "      <td>4.427868</td>\n",
              "      <td>0.988084</td>\n",
              "      <td>2235.922331</td>\n",
              "      <td>1.221313</td>\n",
              "      <td>6.602178</td>\n",
              "      <td>9.714530</td>\n",
              "      <td>192.164300</td>\n",
              "      <td>1.452063</td>\n",
              "      <td>0.540959</td>\n",
              "      <td>0.345796</td>\n",
              "      <td>4.141566</td>\n",
              "      <td>2.465606</td>\n",
              "      <td>0.140614</td>\n",
              "      <td>58.688585</td>\n",
              "      <td>12.455273</td>\n",
              "      <td>319.828725</td>\n",
              "      <td>50.171650</td>\n",
              "      <td>0.030617</td>\n",
              "      <td>0.034833</td>\n",
              "      <td>0.27946</td>\n",
              "      <td>0.037970</td>\n",
              "      <td>0.058050</td>\n",
              "      <td>0.018405</td>\n",
              "      <td>0.792091</td>\n",
              "      <td>0.017595</td>\n",
              "      <td>0.113064</td>\n",
              "      <td>0.064075</td>\n",
              "      <td>0.048936</td>\n",
              "      <td>0.105973</td>\n",
              "      <td>0.046950</td>\n",
              "      <td>250385.040634</td>\n",
              "      <td>4053.954334</td>\n",
              "      <td>1.116300</td>\n",
              "      <td>0.700426</td>\n",
              "      <td>0.594781</td>\n",
              "      <td>0.530437</td>\n",
              "      <td>0.450082</td>\n",
              "      <td>0.443857</td>\n",
              "      <td>0.401017</td>\n",
              "      <td>0.426822</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-139550.845832</td>\n",
              "      <td>-7412.348752</td>\n",
              "      <td>-2.571016</td>\n",
              "      <td>-1.585647</td>\n",
              "      <td>-1.327981</td>\n",
              "      <td>-1.442505</td>\n",
              "      <td>-1.189847</td>\n",
              "      <td>-1.064457</td>\n",
              "      <td>-1.061308</td>\n",
              "      <td>-0.808859</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-133925.066958</td>\n",
              "      <td>-2397.174257</td>\n",
              "      <td>-0.788091</td>\n",
              "      <td>-0.671897</td>\n",
              "      <td>-0.475166</td>\n",
              "      <td>-0.334340</td>\n",
              "      <td>-0.238444</td>\n",
              "      <td>-0.265503</td>\n",
              "      <td>-0.170750</td>\n",
              "      <td>-0.294423</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-128338.581850</td>\n",
              "      <td>-98.834426</td>\n",
              "      <td>-0.018865</td>\n",
              "      <td>-0.019108</td>\n",
              "      <td>-0.066464</td>\n",
              "      <td>-0.015501</td>\n",
              "      <td>0.012356</td>\n",
              "      <td>0.048059</td>\n",
              "      <td>0.028430</td>\n",
              "      <td>-0.034130</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>287779.239352</td>\n",
              "      <td>3128.023695</td>\n",
              "      <td>0.990449</td>\n",
              "      <td>0.441551</td>\n",
              "      <td>0.339117</td>\n",
              "      <td>0.305718</td>\n",
              "      <td>0.247664</td>\n",
              "      <td>0.227888</td>\n",
              "      <td>0.300948</td>\n",
              "      <td>0.264765</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>99.400000</td>\n",
              "      <td>1390.000000</td>\n",
              "      <td>358.000000</td>\n",
              "      <td>1810.000000</td>\n",
              "      <td>7.460000</td>\n",
              "      <td>995.000000</td>\n",
              "      <td>9.890000</td>\n",
              "      <td>1.830000</td>\n",
              "      <td>3.900000</td>\n",
              "      <td>96.700000</td>\n",
              "      <td>99.300000</td>\n",
              "      <td>13.000000</td>\n",
              "      <td>40300.000000</td>\n",
              "      <td>11.200000</td>\n",
              "      <td>48.100000</td>\n",
              "      <td>42.400000</td>\n",
              "      <td>1790.000000</td>\n",
              "      <td>14.000000</td>\n",
              "      <td>6.860000</td>\n",
              "      <td>7.100000</td>\n",
              "      <td>65.000000</td>\n",
              "      <td>52.600000</td>\n",
              "      <td>3.320000</td>\n",
              "      <td>279.000000</td>\n",
              "      <td>79.000000</td>\n",
              "      <td>1640.000000</td>\n",
              "      <td>331.000000</td>\n",
              "      <td>0.435000</td>\n",
              "      <td>0.765000</td>\n",
              "      <td>7.68000</td>\n",
              "      <td>0.726000</td>\n",
              "      <td>1.790000</td>\n",
              "      <td>0.252000</td>\n",
              "      <td>13.400000</td>\n",
              "      <td>0.194000</td>\n",
              "      <td>1.770000</td>\n",
              "      <td>1.590000</td>\n",
              "      <td>0.787000</td>\n",
              "      <td>2.140000</td>\n",
              "      <td>1.020000</td>\n",
              "      <td>646711.616853</td>\n",
              "      <td>7630.592063</td>\n",
              "      <td>2.647168</td>\n",
              "      <td>1.869328</td>\n",
              "      <td>1.597387</td>\n",
              "      <td>2.041569</td>\n",
              "      <td>1.592385</td>\n",
              "      <td>1.742486</td>\n",
              "      <td>1.484113</td>\n",
              "      <td>1.348304</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                0            1   ...           48           49\n",
              "count  2081.000000  2098.000000  ...  2621.000000  2621.000000\n",
              "mean      1.625132    29.615825  ...     0.061447     0.036329\n",
              "std       7.297463   101.869652  ...     0.401017     0.426822\n",
              "min       0.000000     0.000000  ...    -1.061308    -0.808859\n",
              "25%       0.000000     0.000000  ...    -0.170750    -0.294423\n",
              "50%       0.000000     0.000000  ...     0.028430    -0.034130\n",
              "75%       0.000000     0.000000  ...     0.300948     0.264765\n",
              "max      99.400000  1390.000000  ...     1.484113     1.348304\n",
              "\n",
              "[8 rows x 50 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mll2-1ps69Ik"
      },
      "source": [
        "# traditional methods test sets - impute across all foods\n",
        "X_train = pd.DataFrame(missed_data_train, columns = columns)\n",
        "X_test_mean_all = pd.DataFrame(missed_data, columns = columns)\n",
        "X_test_median_all = pd.DataFrame(missed_data, columns = columns)\n",
        "\n",
        "X_test_mean_all = impute_traditional_all_foods(X_train, X_test_mean_all, 'mean')\n",
        "X_test_median_all = impute_traditional_all_foods(X_train, X_test_median_all, 'median')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R-6mOtQB69Il"
      },
      "source": [
        "# Traditional methods test sets - impute by food category\n",
        "X_test_mean = pd.DataFrame(missed_data, columns = columns, index=indices)\n",
        "X_test_median = X_test_mean.copy()\n",
        "df_mean_cat = df_mean[df_mean['fdc_id'].isin(indices)]\n",
        "X_test_mean = impute_traditional_by_category(X_test_mean, df_mean_cat, indices)\n",
        "df_median_cat = df_median[df_median['fdc_id'].isin(indices)]\n",
        "X_test_median = impute_traditional_by_category(X_test_median, df_median_cat, indices)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K3h7vxpv69Il"
      },
      "source": [
        "scaler = MinMaxScaler()\n",
        "scaler.fit(train_data)\n",
        "train_data = scaler.transform(train_data)\n",
        "test_data = scaler.transform(test_data)\n",
        "X_test_mean = scaler.transform(X_test_mean)\n",
        "X_test_median = scaler.transform(X_test_median)\n",
        "X_test_mean_all = scaler.transform(X_test_mean_all)\n",
        "X_test_median_all = scaler.transform(X_test_median_all)\n",
        "\n",
        "y_test = scaler.transform(y_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6M8vbFlU69Im"
      },
      "source": [
        "# datasets without embeddings for autoencoder\n",
        "train_data_noEmbeddings = pd.DataFrame(train_data, columns = columns)\n",
        "train_data_noEmbeddings = np.array(train_data_noEmbeddings.iloc[:, :-num_embeddings]) # df = df.iloc[: , :-1]\n",
        "test_mean_noEmbeddings = np.array(pd.DataFrame(X_test_mean, columns = columns).iloc[:, :-num_embeddings])\n",
        "test_median_noEmbeddings = np.array(pd.DataFrame(X_test_median, columns = columns).iloc[:, :-num_embeddings])\n",
        "test_mean_noEmbeddings_all = np.array(pd.DataFrame(X_test_mean_all, columns = columns).iloc[:, :-num_embeddings])\n",
        "test_median_noEmbeddings_all = np.array(pd.DataFrame(X_test_median_all, columns = columns).iloc[:, :-num_embeddings])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tdLkaB6m69In"
      },
      "source": [
        "missed_data = X_test_mean_all\n",
        "missed_data = torch.from_numpy(missed_data).float()\n",
        "\n",
        "train_data = torch.from_numpy(train_data).float()\n",
        "\n",
        "train_loader = torch.utils.data.DataLoader(dataset=train_data,\n",
        "                                           batch_size=batch_size,\n",
        "                                           shuffle=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TvTyk_cK69Iq"
      },
      "source": [
        "missed_data_noEm = test_mean_noEmbeddings_all\n",
        "missed_data_noEm = torch.from_numpy(missed_data_noEm).float()\n",
        "\n",
        "train_data_noEmbeddings = torch.from_numpy(train_data_noEmbeddings).float()\n",
        "\n",
        "train_loader_noEm = torch.utils.data.DataLoader(dataset=train_data_noEmbeddings,\n",
        "                                           batch_size=batch_size,\n",
        "                                           shuffle=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vFed9fbw69Ir"
      },
      "source": [
        "## Autoencoder - without embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ovkAbSBX69Ir"
      },
      "source": [
        "model = Autoencoder(dim=cols-num_embeddings).to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gv3jUOxR69Ir"
      },
      "source": [
        "criterion = nn.MSELoss()\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kR0_srb369Is",
        "outputId": "1b507715-6c8b-4c48-9510-4a253bbf1abd"
      },
      "source": [
        "training(num_epochs, model, train_loader_noEm, criterion, optimizer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch : 1/200, recon loss = 0.01119547\n",
            "epoch : 2/200, recon loss = 0.00909315\n",
            "epoch : 3/200, recon loss = 0.00602033\n",
            "epoch : 4/200, recon loss = 0.00438604\n",
            "epoch : 5/200, recon loss = 0.00417261\n",
            "epoch : 6/200, recon loss = 0.00411483\n",
            "epoch : 7/200, recon loss = 0.00406816\n",
            "epoch : 8/200, recon loss = 0.00399762\n",
            "epoch : 9/200, recon loss = 0.00392510\n",
            "epoch : 10/200, recon loss = 0.00380531\n",
            "epoch : 11/200, recon loss = 0.00364834\n",
            "epoch : 12/200, recon loss = 0.00346005\n",
            "epoch : 13/200, recon loss = 0.00331112\n",
            "epoch : 14/200, recon loss = 0.00316740\n",
            "epoch : 15/200, recon loss = 0.00311514\n",
            "epoch : 16/200, recon loss = 0.00300763\n",
            "epoch : 17/200, recon loss = 0.00295820\n",
            "epoch : 18/200, recon loss = 0.00293936\n",
            "epoch : 19/200, recon loss = 0.00289737\n",
            "epoch : 20/200, recon loss = 0.00286893\n",
            "epoch : 21/200, recon loss = 0.00285394\n",
            "epoch : 22/200, recon loss = 0.00283548\n",
            "epoch : 23/200, recon loss = 0.00280714\n",
            "epoch : 24/200, recon loss = 0.00280493\n",
            "epoch : 25/200, recon loss = 0.00275761\n",
            "epoch : 26/200, recon loss = 0.00275533\n",
            "epoch : 27/200, recon loss = 0.00270320\n",
            "epoch : 28/200, recon loss = 0.00267910\n",
            "epoch : 29/200, recon loss = 0.00264406\n",
            "epoch : 30/200, recon loss = 0.00260834\n",
            "epoch : 31/200, recon loss = 0.00255180\n",
            "epoch : 32/200, recon loss = 0.00252554\n",
            "epoch : 33/200, recon loss = 0.00247350\n",
            "epoch : 34/200, recon loss = 0.00244069\n",
            "epoch : 35/200, recon loss = 0.00245275\n",
            "epoch : 36/200, recon loss = 0.00240742\n",
            "epoch : 37/200, recon loss = 0.00237063\n",
            "epoch : 38/200, recon loss = 0.00232246\n",
            "epoch : 39/200, recon loss = 0.00237920\n",
            "epoch : 40/200, recon loss = 0.00233782\n",
            "epoch : 41/200, recon loss = 0.00224767\n",
            "epoch : 42/200, recon loss = 0.00222621\n",
            "epoch : 43/200, recon loss = 0.00219322\n",
            "epoch : 44/200, recon loss = 0.00219821\n",
            "epoch : 45/200, recon loss = 0.00217038\n",
            "epoch : 46/200, recon loss = 0.00215502\n",
            "epoch : 47/200, recon loss = 0.00216081\n",
            "epoch : 48/200, recon loss = 0.00209470\n",
            "epoch : 49/200, recon loss = 0.00209716\n",
            "epoch : 50/200, recon loss = 0.00208084\n",
            "epoch : 51/200, recon loss = 0.00206243\n",
            "epoch : 52/200, recon loss = 0.00208112\n",
            "epoch : 53/200, recon loss = 0.00207412\n",
            "epoch : 54/200, recon loss = 0.00203587\n",
            "epoch : 55/200, recon loss = 0.00203458\n",
            "epoch : 56/200, recon loss = 0.00199028\n",
            "epoch : 57/200, recon loss = 0.00202048\n",
            "epoch : 58/200, recon loss = 0.00203031\n",
            "epoch : 59/200, recon loss = 0.00202603\n",
            "epoch : 60/200, recon loss = 0.00198586\n",
            "epoch : 61/200, recon loss = 0.00202286\n",
            "epoch : 62/200, recon loss = 0.00197044\n",
            "epoch : 63/200, recon loss = 0.00199706\n",
            "epoch : 64/200, recon loss = 0.00198131\n",
            "epoch : 65/200, recon loss = 0.00202678\n",
            "epoch : 66/200, recon loss = 0.00196951\n",
            "epoch : 67/200, recon loss = 0.00197400\n",
            "epoch : 68/200, recon loss = 0.00195532\n",
            "epoch : 69/200, recon loss = 0.00192899\n",
            "epoch : 70/200, recon loss = 0.00193453\n",
            "epoch : 71/200, recon loss = 0.00193130\n",
            "epoch : 72/200, recon loss = 0.00193704\n",
            "epoch : 73/200, recon loss = 0.00194072\n",
            "epoch : 74/200, recon loss = 0.00193488\n",
            "epoch : 75/200, recon loss = 0.00196916\n",
            "epoch : 76/200, recon loss = 0.00194118\n",
            "epoch : 77/200, recon loss = 0.00194776\n",
            "epoch : 78/200, recon loss = 0.00194586\n",
            "epoch : 79/200, recon loss = 0.00191091\n",
            "epoch : 80/200, recon loss = 0.00189270\n",
            "epoch : 81/200, recon loss = 0.00193899\n",
            "epoch : 82/200, recon loss = 0.00190066\n",
            "epoch : 83/200, recon loss = 0.00190139\n",
            "epoch : 84/200, recon loss = 0.00192466\n",
            "epoch : 85/200, recon loss = 0.00188906\n",
            "epoch : 86/200, recon loss = 0.00192793\n",
            "epoch : 87/200, recon loss = 0.00189719\n",
            "epoch : 88/200, recon loss = 0.00191084\n",
            "epoch : 89/200, recon loss = 0.00189124\n",
            "epoch : 90/200, recon loss = 0.00188822\n",
            "epoch : 91/200, recon loss = 0.00188042\n",
            "epoch : 92/200, recon loss = 0.00187736\n",
            "epoch : 93/200, recon loss = 0.00188416\n",
            "epoch : 94/200, recon loss = 0.00186231\n",
            "epoch : 95/200, recon loss = 0.00184499\n",
            "epoch : 96/200, recon loss = 0.00191893\n",
            "epoch : 97/200, recon loss = 0.00183453\n",
            "epoch : 98/200, recon loss = 0.00183194\n",
            "epoch : 99/200, recon loss = 0.00184982\n",
            "epoch : 100/200, recon loss = 0.00186031\n",
            "epoch : 101/200, recon loss = 0.00181092\n",
            "epoch : 102/200, recon loss = 0.00183887\n",
            "epoch : 103/200, recon loss = 0.00184781\n",
            "epoch : 104/200, recon loss = 0.00182024\n",
            "epoch : 105/200, recon loss = 0.00180521\n",
            "epoch : 106/200, recon loss = 0.00179614\n",
            "epoch : 107/200, recon loss = 0.00179992\n",
            "epoch : 108/200, recon loss = 0.00184895\n",
            "epoch : 109/200, recon loss = 0.00180062\n",
            "epoch : 110/200, recon loss = 0.00182512\n",
            "epoch : 111/200, recon loss = 0.00176433\n",
            "epoch : 112/200, recon loss = 0.00179729\n",
            "epoch : 113/200, recon loss = 0.00175504\n",
            "epoch : 114/200, recon loss = 0.00181669\n",
            "epoch : 115/200, recon loss = 0.00179204\n",
            "epoch : 116/200, recon loss = 0.00175769\n",
            "epoch : 117/200, recon loss = 0.00177450\n",
            "epoch : 118/200, recon loss = 0.00176665\n",
            "epoch : 119/200, recon loss = 0.00176368\n",
            "epoch : 120/200, recon loss = 0.00179604\n",
            "epoch : 121/200, recon loss = 0.00174867\n",
            "epoch : 122/200, recon loss = 0.00178361\n",
            "epoch : 123/200, recon loss = 0.00176942\n",
            "epoch : 124/200, recon loss = 0.00177522\n",
            "epoch : 125/200, recon loss = 0.00173654\n",
            "epoch : 126/200, recon loss = 0.00177318\n",
            "epoch : 127/200, recon loss = 0.00174493\n",
            "epoch : 128/200, recon loss = 0.00173820\n",
            "epoch : 129/200, recon loss = 0.00176463\n",
            "epoch : 130/200, recon loss = 0.00172804\n",
            "epoch : 131/200, recon loss = 0.00177303\n",
            "epoch : 132/200, recon loss = 0.00176777\n",
            "epoch : 133/200, recon loss = 0.00173819\n",
            "epoch : 134/200, recon loss = 0.00175591\n",
            "epoch : 135/200, recon loss = 0.00175537\n",
            "epoch : 136/200, recon loss = 0.00173068\n",
            "epoch : 137/200, recon loss = 0.00172507\n",
            "epoch : 138/200, recon loss = 0.00172360\n",
            "epoch : 139/200, recon loss = 0.00169929\n",
            "epoch : 140/200, recon loss = 0.00169399\n",
            "epoch : 141/200, recon loss = 0.00175421\n",
            "epoch : 142/200, recon loss = 0.00177363\n",
            "epoch : 143/200, recon loss = 0.00172887\n",
            "epoch : 144/200, recon loss = 0.00173022\n",
            "epoch : 145/200, recon loss = 0.00175730\n",
            "epoch : 146/200, recon loss = 0.00174023\n",
            "epoch : 147/200, recon loss = 0.00172198\n",
            "epoch : 148/200, recon loss = 0.00171145\n",
            "epoch : 149/200, recon loss = 0.00171010\n",
            "epoch : 150/200, recon loss = 0.00172836\n",
            "epoch : 151/200, recon loss = 0.00172535\n",
            "epoch : 152/200, recon loss = 0.00174352\n",
            "epoch : 153/200, recon loss = 0.00174919\n",
            "epoch : 154/200, recon loss = 0.00171717\n",
            "epoch : 155/200, recon loss = 0.00170096\n",
            "epoch : 156/200, recon loss = 0.00172838\n",
            "epoch : 157/200, recon loss = 0.00171767\n",
            "epoch : 158/200, recon loss = 0.00169742\n",
            "epoch : 159/200, recon loss = 0.00168188\n",
            "epoch : 160/200, recon loss = 0.00170563\n",
            "epoch : 161/200, recon loss = 0.00170192\n",
            "epoch : 162/200, recon loss = 0.00169044\n",
            "epoch : 163/200, recon loss = 0.00172729\n",
            "epoch : 164/200, recon loss = 0.00170721\n",
            "epoch : 165/200, recon loss = 0.00171191\n",
            "epoch : 166/200, recon loss = 0.00173431\n",
            "epoch : 167/200, recon loss = 0.00169427\n",
            "epoch : 168/200, recon loss = 0.00173076\n",
            "epoch : 169/200, recon loss = 0.00168185\n",
            "epoch : 170/200, recon loss = 0.00170041\n",
            "epoch : 171/200, recon loss = 0.00166580\n",
            "epoch : 172/200, recon loss = 0.00171480\n",
            "epoch : 173/200, recon loss = 0.00164772\n",
            "epoch : 174/200, recon loss = 0.00165178\n",
            "epoch : 175/200, recon loss = 0.00167732\n",
            "epoch : 176/200, recon loss = 0.00169805\n",
            "epoch : 177/200, recon loss = 0.00165073\n",
            "epoch : 178/200, recon loss = 0.00166771\n",
            "epoch : 179/200, recon loss = 0.00171702\n",
            "epoch : 180/200, recon loss = 0.00168397\n",
            "epoch : 181/200, recon loss = 0.00170995\n",
            "epoch : 182/200, recon loss = 0.00171141\n",
            "epoch : 183/200, recon loss = 0.00165802\n",
            "epoch : 184/200, recon loss = 0.00162677\n",
            "epoch : 185/200, recon loss = 0.00166520\n",
            "epoch : 186/200, recon loss = 0.00166185\n",
            "epoch : 187/200, recon loss = 0.00165074\n",
            "epoch : 188/200, recon loss = 0.00164120\n",
            "epoch : 189/200, recon loss = 0.00166771\n",
            "epoch : 190/200, recon loss = 0.00164201\n",
            "epoch : 191/200, recon loss = 0.00163681\n",
            "epoch : 192/200, recon loss = 0.00164044\n",
            "epoch : 193/200, recon loss = 0.00165758\n",
            "epoch : 194/200, recon loss = 0.00164089\n",
            "epoch : 195/200, recon loss = 0.00163929\n",
            "epoch : 196/200, recon loss = 0.00167413\n",
            "epoch : 197/200, recon loss = 0.00165377\n",
            "epoch : 198/200, recon loss = 0.00163285\n",
            "epoch : 199/200, recon loss = 0.00164474\n",
            "epoch : 200/200, recon loss = 0.00162741\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J7U2SddY69It",
        "outputId": "30fe4d31-64ba-403f-d45c-83a22c8e02cf"
      },
      "source": [
        "model.eval()\n",
        "rmse_sum = 0\n",
        "\n",
        "filled_data = model(missed_data_noEm.to(device))\n",
        "filled_data = filled_data.cpu().detach().numpy()\n",
        "\n",
        "rmse_sum = rmse_error(test_data, filled_data, cols-num_embeddings, mask)\n",
        "\n",
        "print(rmse_sum)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1.8480004416035005\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MZpaSQVh69It"
      },
      "source": [
        "## Autoencoder - with embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pm6CXF1v69Iu"
      },
      "source": [
        "model = Autoencoder(dim=cols).to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gZ6ojm7i69Iu"
      },
      "source": [
        "criterion = nn.MSELoss()\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A49fvwOf69Iv",
        "outputId": "7fd06e84-e2c0-43b6-8bb1-df5bcc5adf6c"
      },
      "source": [
        "training(num_epochs, model, train_loader, criterion, optimizer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch : 1/200, recon loss = 0.01978195\n",
            "epoch : 2/200, recon loss = 0.01637038\n",
            "epoch : 3/200, recon loss = 0.01545645\n",
            "epoch : 4/200, recon loss = 0.01315527\n",
            "epoch : 5/200, recon loss = 0.01141412\n",
            "epoch : 6/200, recon loss = 0.01078181\n",
            "epoch : 7/200, recon loss = 0.01042307\n",
            "epoch : 8/200, recon loss = 0.01011216\n",
            "epoch : 9/200, recon loss = 0.00977376\n",
            "epoch : 10/200, recon loss = 0.00956904\n",
            "epoch : 11/200, recon loss = 0.00940188\n",
            "epoch : 12/200, recon loss = 0.00927159\n",
            "epoch : 13/200, recon loss = 0.00917862\n",
            "epoch : 14/200, recon loss = 0.00906874\n",
            "epoch : 15/200, recon loss = 0.00896763\n",
            "epoch : 16/200, recon loss = 0.00881151\n",
            "epoch : 17/200, recon loss = 0.00871635\n",
            "epoch : 18/200, recon loss = 0.00862613\n",
            "epoch : 19/200, recon loss = 0.00843092\n",
            "epoch : 20/200, recon loss = 0.00834418\n",
            "epoch : 21/200, recon loss = 0.00828966\n",
            "epoch : 22/200, recon loss = 0.00818139\n",
            "epoch : 23/200, recon loss = 0.00812029\n",
            "epoch : 24/200, recon loss = 0.00805194\n",
            "epoch : 25/200, recon loss = 0.00792620\n",
            "epoch : 26/200, recon loss = 0.00791222\n",
            "epoch : 27/200, recon loss = 0.00786054\n",
            "epoch : 28/200, recon loss = 0.00781320\n",
            "epoch : 29/200, recon loss = 0.00774394\n",
            "epoch : 30/200, recon loss = 0.00771718\n",
            "epoch : 31/200, recon loss = 0.00771939\n",
            "epoch : 32/200, recon loss = 0.00767354\n",
            "epoch : 33/200, recon loss = 0.00759013\n",
            "epoch : 34/200, recon loss = 0.00758815\n",
            "epoch : 35/200, recon loss = 0.00753511\n",
            "epoch : 36/200, recon loss = 0.00752906\n",
            "epoch : 37/200, recon loss = 0.00745884\n",
            "epoch : 38/200, recon loss = 0.00740293\n",
            "epoch : 39/200, recon loss = 0.00743460\n",
            "epoch : 40/200, recon loss = 0.00739333\n",
            "epoch : 41/200, recon loss = 0.00736783\n",
            "epoch : 42/200, recon loss = 0.00726228\n",
            "epoch : 43/200, recon loss = 0.00731579\n",
            "epoch : 44/200, recon loss = 0.00723514\n",
            "epoch : 45/200, recon loss = 0.00722050\n",
            "epoch : 46/200, recon loss = 0.00722291\n",
            "epoch : 47/200, recon loss = 0.00721840\n",
            "epoch : 48/200, recon loss = 0.00711424\n",
            "epoch : 49/200, recon loss = 0.00711491\n",
            "epoch : 50/200, recon loss = 0.00710530\n",
            "epoch : 51/200, recon loss = 0.00709059\n",
            "epoch : 52/200, recon loss = 0.00703435\n",
            "epoch : 53/200, recon loss = 0.00706007\n",
            "epoch : 54/200, recon loss = 0.00697933\n",
            "epoch : 55/200, recon loss = 0.00703026\n",
            "epoch : 56/200, recon loss = 0.00698744\n",
            "epoch : 57/200, recon loss = 0.00695554\n",
            "epoch : 58/200, recon loss = 0.00691496\n",
            "epoch : 59/200, recon loss = 0.00688156\n",
            "epoch : 60/200, recon loss = 0.00686701\n",
            "epoch : 61/200, recon loss = 0.00689652\n",
            "epoch : 62/200, recon loss = 0.00687320\n",
            "epoch : 63/200, recon loss = 0.00684962\n",
            "epoch : 64/200, recon loss = 0.00681850\n",
            "epoch : 65/200, recon loss = 0.00682454\n",
            "epoch : 66/200, recon loss = 0.00680648\n",
            "epoch : 67/200, recon loss = 0.00675620\n",
            "epoch : 68/200, recon loss = 0.00677429\n",
            "epoch : 69/200, recon loss = 0.00677083\n",
            "epoch : 70/200, recon loss = 0.00672364\n",
            "epoch : 71/200, recon loss = 0.00672108\n",
            "epoch : 72/200, recon loss = 0.00669841\n",
            "epoch : 73/200, recon loss = 0.00665109\n",
            "epoch : 74/200, recon loss = 0.00661894\n",
            "epoch : 75/200, recon loss = 0.00662215\n",
            "epoch : 76/200, recon loss = 0.00671951\n",
            "epoch : 77/200, recon loss = 0.00661887\n",
            "epoch : 78/200, recon loss = 0.00668472\n",
            "epoch : 79/200, recon loss = 0.00664257\n",
            "epoch : 80/200, recon loss = 0.00663448\n",
            "epoch : 81/200, recon loss = 0.00656530\n",
            "epoch : 82/200, recon loss = 0.00663584\n",
            "epoch : 83/200, recon loss = 0.00666261\n",
            "epoch : 84/200, recon loss = 0.00661878\n",
            "epoch : 85/200, recon loss = 0.00659794\n",
            "epoch : 86/200, recon loss = 0.00656293\n",
            "epoch : 87/200, recon loss = 0.00662533\n",
            "epoch : 88/200, recon loss = 0.00659977\n",
            "epoch : 89/200, recon loss = 0.00661662\n",
            "epoch : 90/200, recon loss = 0.00667054\n",
            "epoch : 91/200, recon loss = 0.00660096\n",
            "epoch : 92/200, recon loss = 0.00661037\n",
            "epoch : 93/200, recon loss = 0.00659027\n",
            "epoch : 94/200, recon loss = 0.00658983\n",
            "epoch : 95/200, recon loss = 0.00657544\n",
            "epoch : 96/200, recon loss = 0.00654163\n",
            "epoch : 97/200, recon loss = 0.00653676\n",
            "epoch : 98/200, recon loss = 0.00652096\n",
            "epoch : 99/200, recon loss = 0.00653952\n",
            "epoch : 100/200, recon loss = 0.00646997\n",
            "epoch : 101/200, recon loss = 0.00647230\n",
            "epoch : 102/200, recon loss = 0.00656901\n",
            "epoch : 103/200, recon loss = 0.00656959\n",
            "epoch : 104/200, recon loss = 0.00648029\n",
            "epoch : 105/200, recon loss = 0.00650217\n",
            "epoch : 106/200, recon loss = 0.00646666\n",
            "epoch : 107/200, recon loss = 0.00653846\n",
            "epoch : 108/200, recon loss = 0.00649144\n",
            "epoch : 109/200, recon loss = 0.00649428\n",
            "epoch : 110/200, recon loss = 0.00647131\n",
            "epoch : 111/200, recon loss = 0.00646688\n",
            "epoch : 112/200, recon loss = 0.00647041\n",
            "epoch : 113/200, recon loss = 0.00645643\n",
            "epoch : 114/200, recon loss = 0.00649082\n",
            "epoch : 115/200, recon loss = 0.00645286\n",
            "epoch : 116/200, recon loss = 0.00646502\n",
            "epoch : 117/200, recon loss = 0.00646321\n",
            "epoch : 118/200, recon loss = 0.00643695\n",
            "epoch : 119/200, recon loss = 0.00646354\n",
            "epoch : 120/200, recon loss = 0.00642500\n",
            "epoch : 121/200, recon loss = 0.00647961\n",
            "epoch : 122/200, recon loss = 0.00643369\n",
            "epoch : 123/200, recon loss = 0.00637127\n",
            "epoch : 124/200, recon loss = 0.00647271\n",
            "epoch : 125/200, recon loss = 0.00642465\n",
            "epoch : 126/200, recon loss = 0.00639638\n",
            "epoch : 127/200, recon loss = 0.00644472\n",
            "epoch : 128/200, recon loss = 0.00643800\n",
            "epoch : 129/200, recon loss = 0.00644880\n",
            "epoch : 130/200, recon loss = 0.00642084\n",
            "epoch : 131/200, recon loss = 0.00636592\n",
            "epoch : 132/200, recon loss = 0.00638052\n",
            "epoch : 133/200, recon loss = 0.00633196\n",
            "epoch : 134/200, recon loss = 0.00631745\n",
            "epoch : 135/200, recon loss = 0.00633981\n",
            "epoch : 136/200, recon loss = 0.00633600\n",
            "epoch : 137/200, recon loss = 0.00632778\n",
            "epoch : 138/200, recon loss = 0.00635127\n",
            "epoch : 139/200, recon loss = 0.00630581\n",
            "epoch : 140/200, recon loss = 0.00631231\n",
            "epoch : 141/200, recon loss = 0.00623927\n",
            "epoch : 142/200, recon loss = 0.00631404\n",
            "epoch : 143/200, recon loss = 0.00629958\n",
            "epoch : 144/200, recon loss = 0.00627260\n",
            "epoch : 145/200, recon loss = 0.00631844\n",
            "epoch : 146/200, recon loss = 0.00625122\n",
            "epoch : 147/200, recon loss = 0.00622320\n",
            "epoch : 148/200, recon loss = 0.00627441\n",
            "epoch : 149/200, recon loss = 0.00618243\n",
            "epoch : 150/200, recon loss = 0.00620558\n",
            "epoch : 151/200, recon loss = 0.00622988\n",
            "epoch : 152/200, recon loss = 0.00616556\n",
            "epoch : 153/200, recon loss = 0.00616216\n",
            "epoch : 154/200, recon loss = 0.00617639\n",
            "epoch : 155/200, recon loss = 0.00620730\n",
            "epoch : 156/200, recon loss = 0.00615912\n",
            "epoch : 157/200, recon loss = 0.00613153\n",
            "epoch : 158/200, recon loss = 0.00618047\n",
            "epoch : 159/200, recon loss = 0.00609884\n",
            "epoch : 160/200, recon loss = 0.00613583\n",
            "epoch : 161/200, recon loss = 0.00616214\n",
            "epoch : 162/200, recon loss = 0.00607519\n",
            "epoch : 163/200, recon loss = 0.00613414\n",
            "epoch : 164/200, recon loss = 0.00607586\n",
            "epoch : 165/200, recon loss = 0.00611671\n",
            "epoch : 166/200, recon loss = 0.00606044\n",
            "epoch : 167/200, recon loss = 0.00607764\n",
            "epoch : 168/200, recon loss = 0.00601871\n",
            "epoch : 169/200, recon loss = 0.00604600\n",
            "epoch : 170/200, recon loss = 0.00605397\n",
            "epoch : 171/200, recon loss = 0.00603424\n",
            "epoch : 172/200, recon loss = 0.00597248\n",
            "epoch : 173/200, recon loss = 0.00599669\n",
            "epoch : 174/200, recon loss = 0.00599050\n",
            "epoch : 175/200, recon loss = 0.00597840\n",
            "epoch : 176/200, recon loss = 0.00600487\n",
            "epoch : 177/200, recon loss = 0.00598422\n",
            "epoch : 178/200, recon loss = 0.00596858\n",
            "epoch : 179/200, recon loss = 0.00594966\n",
            "epoch : 180/200, recon loss = 0.00597810\n",
            "epoch : 181/200, recon loss = 0.00598774\n",
            "epoch : 182/200, recon loss = 0.00600627\n",
            "epoch : 183/200, recon loss = 0.00595902\n",
            "epoch : 184/200, recon loss = 0.00593871\n",
            "epoch : 185/200, recon loss = 0.00596482\n",
            "epoch : 186/200, recon loss = 0.00593230\n",
            "epoch : 187/200, recon loss = 0.00592971\n",
            "epoch : 188/200, recon loss = 0.00592927\n",
            "epoch : 189/200, recon loss = 0.00593963\n",
            "epoch : 190/200, recon loss = 0.00590107\n",
            "epoch : 191/200, recon loss = 0.00588698\n",
            "epoch : 192/200, recon loss = 0.00590771\n",
            "epoch : 193/200, recon loss = 0.00589433\n",
            "epoch : 194/200, recon loss = 0.00581131\n",
            "epoch : 195/200, recon loss = 0.00585281\n",
            "epoch : 196/200, recon loss = 0.00587212\n",
            "epoch : 197/200, recon loss = 0.00585617\n",
            "epoch : 198/200, recon loss = 0.00583888\n",
            "epoch : 199/200, recon loss = 0.00585105\n",
            "epoch : 200/200, recon loss = 0.00586537\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y09GlBWH69Iv",
        "outputId": "f8918299-108a-4cf6-b683-a2ebe5b700b6"
      },
      "source": [
        "model.eval()\n",
        "rmse_sum = 0\n",
        "\n",
        "filled_data = model(missed_data.to(device))\n",
        "filled_data = filled_data.cpu().detach().numpy()\n",
        "\n",
        "rmse_sum = rmse_error(test_data, filled_data, cols, mask)\n",
        "\n",
        "print(rmse_sum)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1.8551008859606322\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IWEfF5D669Iw"
      },
      "source": [
        "## Mean and Median Imputation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GEi4k8y969Iw"
      },
      "source": [
        "### By category"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dGTUUSEr69Ix",
        "outputId": "5108f083-3ffd-4ede-bdd5-b850983c4abc"
      },
      "source": [
        "imputed = X_test_mean\n",
        "error = rmse_error(y_test, imputed, cols-num_embeddings, mask)\n",
        "print(error)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2.531217748333226\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dXDGepth69Ix",
        "outputId": "c38cdbfe-cc41-4b79-8a80-b43760d50b0d"
      },
      "source": [
        "imputed = X_test_median\n",
        "error = rmse_error(y_test, imputed, cols-num_embeddings, mask)\n",
        "print(error)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2.680654619816815\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t8zwNZ5i69Ix"
      },
      "source": [
        "### Across all foods"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0ooJ9hHi69Iy",
        "outputId": "2875fef8-c54e-48c8-b206-26743aba7d4f"
      },
      "source": [
        "imputed = X_test_mean_all\n",
        "error = rmse_error(y_test, imputed, cols-num_embeddings, mask)\n",
        "print(error)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "3.350671086407284\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-QPzRD3A69Iy",
        "outputId": "87a51a64-a130-41d8-add5-c889226daf5b"
      },
      "source": [
        "imputed = X_test_median_all\n",
        "error = rmse_error(y_test, imputed, cols-num_embeddings, mask)\n",
        "print(error)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "3.4719214819705706\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lnyedPPI69Iz"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KyU-WnUC7GWD"
      },
      "source": [
        "# 80 Columns"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "39AnJ1ma7GWE"
      },
      "source": [
        "data_path = 'datasets/avarage datasets/byColumnsWithEmbeddings/80_columns_embeddings.csv'\n",
        "num_embeddings = 20\n",
        "num_columns = 80"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 350
        },
        "id": "Yo5snldk7GWF",
        "outputId": "40b40a89-817a-4461-d622-f2d0370b2c2a"
      },
      "source": [
        "dataset = pd.read_csv(data_path, index_col=0)\n",
        "dataset[dataset.notna().all(axis=1)].describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>fdc_id</th>\n",
              "      <th>1004</th>\n",
              "      <th>1087</th>\n",
              "      <th>1090</th>\n",
              "      <th>1092</th>\n",
              "      <th>1095</th>\n",
              "      <th>1091</th>\n",
              "      <th>1089</th>\n",
              "      <th>1098</th>\n",
              "      <th>1101</th>\n",
              "      <th>1051</th>\n",
              "      <th>1007</th>\n",
              "      <th>1002</th>\n",
              "      <th>1093</th>\n",
              "      <th>1079</th>\n",
              "      <th>1003</th>\n",
              "      <th>1009</th>\n",
              "      <th>1102</th>\n",
              "      <th>1265</th>\n",
              "      <th>1266</th>\n",
              "      <th>1264</th>\n",
              "      <th>1315</th>\n",
              "      <th>1316</th>\n",
              "      <th>1314</th>\n",
              "      <th>1094</th>\n",
              "      <th>1097</th>\n",
              "      <th>1137</th>\n",
              "      <th>1146</th>\n",
              "      <th>1300</th>\n",
              "      <th>1299</th>\n",
              "      <th>1404</th>\n",
              "      <th>1267</th>\n",
              "      <th>1273</th>\n",
              "      <th>1323</th>\n",
              "      <th>1167</th>\n",
              "      <th>1271</th>\n",
              "      <th>1304</th>\n",
              "      <th>1166</th>\n",
              "      <th>1175</th>\n",
              "      <th>1263</th>\n",
              "      <th>...</th>\n",
              "      <th>1277</th>\n",
              "      <th>1292</th>\n",
              "      <th>1258</th>\n",
              "      <th>1293</th>\n",
              "      <th>1405</th>\n",
              "      <th>1010</th>\n",
              "      <th>1170</th>\n",
              "      <th>1109</th>\n",
              "      <th>1177</th>\n",
              "      <th>1178</th>\n",
              "      <th>1312</th>\n",
              "      <th>2009</th>\n",
              "      <th>1334</th>\n",
              "      <th>1257</th>\n",
              "      <th>1100</th>\n",
              "      <th>1107</th>\n",
              "      <th>1223</th>\n",
              "      <th>1224</th>\n",
              "      <th>1210</th>\n",
              "      <th>1211</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>1.310500e+04</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.00000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>4.807517e+05</td>\n",
              "      <td>1.824697</td>\n",
              "      <td>27.659061</td>\n",
              "      <td>14.595597</td>\n",
              "      <td>125.003892</td>\n",
              "      <td>0.384163</td>\n",
              "      <td>55.746662</td>\n",
              "      <td>0.476038</td>\n",
              "      <td>0.077504</td>\n",
              "      <td>0.149979</td>\n",
              "      <td>8.349769</td>\n",
              "      <td>0.472742</td>\n",
              "      <td>0.243665</td>\n",
              "      <td>133.394887</td>\n",
              "      <td>0.350355</td>\n",
              "      <td>1.942243</td>\n",
              "      <td>2.561062</td>\n",
              "      <td>42.944411</td>\n",
              "      <td>0.308332</td>\n",
              "      <td>0.116491</td>\n",
              "      <td>0.065003</td>\n",
              "      <td>0.605730</td>\n",
              "      <td>0.211003</td>\n",
              "      <td>0.023788</td>\n",
              "      <td>14.730179</td>\n",
              "      <td>2.983579</td>\n",
              "      <td>78.72934</td>\n",
              "      <td>12.225029</td>\n",
              "      <td>0.005267</td>\n",
              "      <td>0.005886</td>\n",
              "      <td>0.016719</td>\n",
              "      <td>0.004499</td>\n",
              "      <td>0.003870</td>\n",
              "      <td>0.002480</td>\n",
              "      <td>0.118839</td>\n",
              "      <td>0.002989</td>\n",
              "      <td>0.015767</td>\n",
              "      <td>0.012258</td>\n",
              "      <td>0.008632</td>\n",
              "      <td>0.038133</td>\n",
              "      <td>...</td>\n",
              "      <td>0.002201</td>\n",
              "      <td>0.434562</td>\n",
              "      <td>0.319591</td>\n",
              "      <td>0.150495</td>\n",
              "      <td>0.000279</td>\n",
              "      <td>0.147034</td>\n",
              "      <td>0.011803</td>\n",
              "      <td>0.073772</td>\n",
              "      <td>1.055017</td>\n",
              "      <td>0.027458</td>\n",
              "      <td>0.000093</td>\n",
              "      <td>0.002787</td>\n",
              "      <td>0.000065</td>\n",
              "      <td>0.008114</td>\n",
              "      <td>12.187135</td>\n",
              "      <td>8.910568</td>\n",
              "      <td>0.016636</td>\n",
              "      <td>0.027917</td>\n",
              "      <td>0.002017</td>\n",
              "      <td>0.007339</td>\n",
              "      <td>21488.189968</td>\n",
              "      <td>176.402583</td>\n",
              "      <td>0.119704</td>\n",
              "      <td>-0.093912</td>\n",
              "      <td>-0.048494</td>\n",
              "      <td>-0.007831</td>\n",
              "      <td>0.051473</td>\n",
              "      <td>0.023113</td>\n",
              "      <td>0.063163</td>\n",
              "      <td>0.043670</td>\n",
              "      <td>-0.005261</td>\n",
              "      <td>-0.053342</td>\n",
              "      <td>0.032862</td>\n",
              "      <td>0.017149</td>\n",
              "      <td>0.053913</td>\n",
              "      <td>-0.050025</td>\n",
              "      <td>-0.006816</td>\n",
              "      <td>-0.001294</td>\n",
              "      <td>0.045430</td>\n",
              "      <td>-0.002778</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.544018e+05</td>\n",
              "      <td>7.838466</td>\n",
              "      <td>99.924214</td>\n",
              "      <td>45.754175</td>\n",
              "      <td>382.322293</td>\n",
              "      <td>1.055340</td>\n",
              "      <td>153.432660</td>\n",
              "      <td>1.483360</td>\n",
              "      <td>0.264892</td>\n",
              "      <td>0.515558</td>\n",
              "      <td>23.549114</td>\n",
              "      <td>4.416751</td>\n",
              "      <td>0.999601</td>\n",
              "      <td>1736.846193</td>\n",
              "      <td>1.186155</td>\n",
              "      <td>6.583455</td>\n",
              "      <td>9.446549</td>\n",
              "      <td>187.200801</td>\n",
              "      <td>1.605577</td>\n",
              "      <td>0.597369</td>\n",
              "      <td>0.559044</td>\n",
              "      <td>4.440086</td>\n",
              "      <td>1.941342</td>\n",
              "      <td>0.148635</td>\n",
              "      <td>56.459805</td>\n",
              "      <td>11.848737</td>\n",
              "      <td>302.68343</td>\n",
              "      <td>49.040700</td>\n",
              "      <td>0.030796</td>\n",
              "      <td>0.043100</td>\n",
              "      <td>0.223785</td>\n",
              "      <td>0.039142</td>\n",
              "      <td>0.065227</td>\n",
              "      <td>0.015745</td>\n",
              "      <td>0.828604</td>\n",
              "      <td>0.017050</td>\n",
              "      <td>0.106190</td>\n",
              "      <td>0.071481</td>\n",
              "      <td>0.054902</td>\n",
              "      <td>0.974266</td>\n",
              "      <td>...</td>\n",
              "      <td>0.025626</td>\n",
              "      <td>4.338678</td>\n",
              "      <td>2.546196</td>\n",
              "      <td>1.960438</td>\n",
              "      <td>0.002974</td>\n",
              "      <td>2.828219</td>\n",
              "      <td>0.085062</td>\n",
              "      <td>1.032461</td>\n",
              "      <td>10.123472</td>\n",
              "      <td>0.206489</td>\n",
              "      <td>0.001497</td>\n",
              "      <td>0.029047</td>\n",
              "      <td>0.000516</td>\n",
              "      <td>0.083699</td>\n",
              "      <td>237.511065</td>\n",
              "      <td>175.426202</td>\n",
              "      <td>0.181747</td>\n",
              "      <td>0.318716</td>\n",
              "      <td>0.022631</td>\n",
              "      <td>0.078590</td>\n",
              "      <td>254447.573253</td>\n",
              "      <td>3958.010969</td>\n",
              "      <td>1.112342</td>\n",
              "      <td>0.694392</td>\n",
              "      <td>0.598293</td>\n",
              "      <td>0.511537</td>\n",
              "      <td>0.453723</td>\n",
              "      <td>0.451007</td>\n",
              "      <td>0.417454</td>\n",
              "      <td>0.426071</td>\n",
              "      <td>0.400175</td>\n",
              "      <td>0.345475</td>\n",
              "      <td>0.349559</td>\n",
              "      <td>0.332055</td>\n",
              "      <td>0.321275</td>\n",
              "      <td>0.292891</td>\n",
              "      <td>0.270026</td>\n",
              "      <td>0.284946</td>\n",
              "      <td>0.261710</td>\n",
              "      <td>0.269164</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>3.198770e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-139555.939272</td>\n",
              "      <td>-7413.329674</td>\n",
              "      <td>-3.349638</td>\n",
              "      <td>-1.585647</td>\n",
              "      <td>-1.527263</td>\n",
              "      <td>-1.442526</td>\n",
              "      <td>-1.189856</td>\n",
              "      <td>-1.064453</td>\n",
              "      <td>-1.077035</td>\n",
              "      <td>-0.829624</td>\n",
              "      <td>-0.933852</td>\n",
              "      <td>-0.969973</td>\n",
              "      <td>-1.097073</td>\n",
              "      <td>-0.920129</td>\n",
              "      <td>-1.006213</td>\n",
              "      <td>-1.000462</td>\n",
              "      <td>-0.798443</td>\n",
              "      <td>-0.685798</td>\n",
              "      <td>-0.669546</td>\n",
              "      <td>-0.809746</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>3.256480e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-133692.806088</td>\n",
              "      <td>-2328.507401</td>\n",
              "      <td>-0.796369</td>\n",
              "      <td>-0.667107</td>\n",
              "      <td>-0.477073</td>\n",
              "      <td>-0.343775</td>\n",
              "      <td>-0.236112</td>\n",
              "      <td>-0.286794</td>\n",
              "      <td>-0.177182</td>\n",
              "      <td>-0.292727</td>\n",
              "      <td>-0.303204</td>\n",
              "      <td>-0.332177</td>\n",
              "      <td>-0.196668</td>\n",
              "      <td>-0.219358</td>\n",
              "      <td>-0.160230</td>\n",
              "      <td>-0.217182</td>\n",
              "      <td>-0.194962</td>\n",
              "      <td>-0.175535</td>\n",
              "      <td>-0.131800</td>\n",
              "      <td>-0.159046</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>3.314100e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-127823.125712</td>\n",
              "      <td>-153.274226</td>\n",
              "      <td>-0.002647</td>\n",
              "      <td>-0.011945</td>\n",
              "      <td>-0.071964</td>\n",
              "      <td>-0.022449</td>\n",
              "      <td>0.012034</td>\n",
              "      <td>0.040864</td>\n",
              "      <td>0.040584</td>\n",
              "      <td>-0.032775</td>\n",
              "      <td>-0.006456</td>\n",
              "      <td>0.045001</td>\n",
              "      <td>0.064457</td>\n",
              "      <td>-0.007091</td>\n",
              "      <td>0.051132</td>\n",
              "      <td>-0.054635</td>\n",
              "      <td>-0.020075</td>\n",
              "      <td>-0.031549</td>\n",
              "      <td>0.015217</td>\n",
              "      <td>-0.036377</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>7.474680e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>288262.097477</td>\n",
              "      <td>3032.870912</td>\n",
              "      <td>0.983044</td>\n",
              "      <td>0.441140</td>\n",
              "      <td>0.323271</td>\n",
              "      <td>0.289042</td>\n",
              "      <td>0.257716</td>\n",
              "      <td>0.227569</td>\n",
              "      <td>0.309666</td>\n",
              "      <td>0.274537</td>\n",
              "      <td>0.267120</td>\n",
              "      <td>0.166330</td>\n",
              "      <td>0.248007</td>\n",
              "      <td>0.188549</td>\n",
              "      <td>0.251005</td>\n",
              "      <td>0.108285</td>\n",
              "      <td>0.190913</td>\n",
              "      <td>0.178225</td>\n",
              "      <td>0.197493</td>\n",
              "      <td>0.175122</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.105897e+06</td>\n",
              "      <td>99.400000</td>\n",
              "      <td>1390.000000</td>\n",
              "      <td>386.000000</td>\n",
              "      <td>2520.000000</td>\n",
              "      <td>7.660000</td>\n",
              "      <td>997.000000</td>\n",
              "      <td>12.700000</td>\n",
              "      <td>1.920000</td>\n",
              "      <td>7.870000</td>\n",
              "      <td>96.700000</td>\n",
              "      <td>99.600000</td>\n",
              "      <td>13.200000</td>\n",
              "      <td>40700.000000</td>\n",
              "      <td>11.500000</td>\n",
              "      <td>79.900000</td>\n",
              "      <td>43.900000</td>\n",
              "      <td>1790.000000</td>\n",
              "      <td>22.800000</td>\n",
              "      <td>8.610000</td>\n",
              "      <td>17.000000</td>\n",
              "      <td>73.200000</td>\n",
              "      <td>52.600000</td>\n",
              "      <td>3.400000</td>\n",
              "      <td>311.000000</td>\n",
              "      <td>93.000000</td>\n",
              "      <td>1710.00000</td>\n",
              "      <td>449.000000</td>\n",
              "      <td>0.485000</td>\n",
              "      <td>0.850000</td>\n",
              "      <td>7.950000</td>\n",
              "      <td>0.774000</td>\n",
              "      <td>1.830000</td>\n",
              "      <td>0.303000</td>\n",
              "      <td>14.000000</td>\n",
              "      <td>0.222000</td>\n",
              "      <td>1.930000</td>\n",
              "      <td>1.860000</td>\n",
              "      <td>0.989000</td>\n",
              "      <td>42.400000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.612000</td>\n",
              "      <td>74.200000</td>\n",
              "      <td>82.500000</td>\n",
              "      <td>58.100000</td>\n",
              "      <td>0.171000</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>1.640000</td>\n",
              "      <td>28.300000</td>\n",
              "      <td>348.000000</td>\n",
              "      <td>4.400000</td>\n",
              "      <td>0.086000</td>\n",
              "      <td>0.712000</td>\n",
              "      <td>0.022000</td>\n",
              "      <td>1.590000</td>\n",
              "      <td>7430.000000</td>\n",
              "      <td>8750.000000</td>\n",
              "      <td>5.320000</td>\n",
              "      <td>9.700000</td>\n",
              "      <td>0.630000</td>\n",
              "      <td>2.100000</td>\n",
              "      <td>646711.616853</td>\n",
              "      <td>7635.496843</td>\n",
              "      <td>2.647689</td>\n",
              "      <td>1.869347</td>\n",
              "      <td>1.616849</td>\n",
              "      <td>2.041568</td>\n",
              "      <td>1.700615</td>\n",
              "      <td>1.764944</td>\n",
              "      <td>1.547783</td>\n",
              "      <td>1.348230</td>\n",
              "      <td>1.122909</td>\n",
              "      <td>1.235692</td>\n",
              "      <td>1.230902</td>\n",
              "      <td>1.301706</td>\n",
              "      <td>1.225573</td>\n",
              "      <td>1.032070</td>\n",
              "      <td>0.770407</td>\n",
              "      <td>0.843408</td>\n",
              "      <td>1.053437</td>\n",
              "      <td>0.875425</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8 rows × 101 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "             fdc_id          1004  ...            18            19\n",
              "count  1.310500e+04  13105.000000  ...  13105.000000  13105.000000\n",
              "mean   4.807517e+05      1.824697  ...      0.045430     -0.002778\n",
              "std    2.544018e+05      7.838466  ...      0.261710      0.269164\n",
              "min    3.198770e+05      0.000000  ...     -0.669546     -0.809746\n",
              "25%    3.256480e+05      0.000000  ...     -0.131800     -0.159046\n",
              "50%    3.314100e+05      0.000000  ...      0.015217     -0.036377\n",
              "75%    7.474680e+05      0.000000  ...      0.197493      0.175122\n",
              "max    1.105897e+06     99.400000  ...      1.053437      0.875425\n",
              "\n",
              "[8 rows x 101 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aycGkaGn7GWF"
      },
      "source": [
        "data = dataset\n",
        "train_data, test_data = train_test_split(data, test_size=test_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 350
        },
        "id": "O1YHHbh77GWG",
        "outputId": "04812156-b587-42f2-c340-54b42022f540"
      },
      "source": [
        "train_data.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>fdc_id</th>\n",
              "      <th>1004</th>\n",
              "      <th>1087</th>\n",
              "      <th>1090</th>\n",
              "      <th>1092</th>\n",
              "      <th>1095</th>\n",
              "      <th>1091</th>\n",
              "      <th>1089</th>\n",
              "      <th>1098</th>\n",
              "      <th>1101</th>\n",
              "      <th>1051</th>\n",
              "      <th>1007</th>\n",
              "      <th>1002</th>\n",
              "      <th>1093</th>\n",
              "      <th>1079</th>\n",
              "      <th>1003</th>\n",
              "      <th>1009</th>\n",
              "      <th>1102</th>\n",
              "      <th>1265</th>\n",
              "      <th>1266</th>\n",
              "      <th>1264</th>\n",
              "      <th>1315</th>\n",
              "      <th>1316</th>\n",
              "      <th>1314</th>\n",
              "      <th>1094</th>\n",
              "      <th>1097</th>\n",
              "      <th>1137</th>\n",
              "      <th>1146</th>\n",
              "      <th>1300</th>\n",
              "      <th>1299</th>\n",
              "      <th>1404</th>\n",
              "      <th>1267</th>\n",
              "      <th>1273</th>\n",
              "      <th>1323</th>\n",
              "      <th>1167</th>\n",
              "      <th>1271</th>\n",
              "      <th>1304</th>\n",
              "      <th>1166</th>\n",
              "      <th>1175</th>\n",
              "      <th>1263</th>\n",
              "      <th>...</th>\n",
              "      <th>1277</th>\n",
              "      <th>1292</th>\n",
              "      <th>1258</th>\n",
              "      <th>1293</th>\n",
              "      <th>1405</th>\n",
              "      <th>1010</th>\n",
              "      <th>1170</th>\n",
              "      <th>1109</th>\n",
              "      <th>1177</th>\n",
              "      <th>1178</th>\n",
              "      <th>1312</th>\n",
              "      <th>2009</th>\n",
              "      <th>1334</th>\n",
              "      <th>1257</th>\n",
              "      <th>1100</th>\n",
              "      <th>1107</th>\n",
              "      <th>1223</th>\n",
              "      <th>1224</th>\n",
              "      <th>1210</th>\n",
              "      <th>1211</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>1.048400e+04</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>4.808406e+05</td>\n",
              "      <td>1.788952</td>\n",
              "      <td>27.818199</td>\n",
              "      <td>14.610349</td>\n",
              "      <td>125.796356</td>\n",
              "      <td>0.381077</td>\n",
              "      <td>55.589088</td>\n",
              "      <td>0.473794</td>\n",
              "      <td>0.077637</td>\n",
              "      <td>0.150204</td>\n",
              "      <td>8.280277</td>\n",
              "      <td>0.466951</td>\n",
              "      <td>0.235234</td>\n",
              "      <td>130.758585</td>\n",
              "      <td>0.355857</td>\n",
              "      <td>1.938891</td>\n",
              "      <td>2.572338</td>\n",
              "      <td>42.944601</td>\n",
              "      <td>0.312186</td>\n",
              "      <td>0.116781</td>\n",
              "      <td>0.064787</td>\n",
              "      <td>0.635888</td>\n",
              "      <td>0.215604</td>\n",
              "      <td>0.024394</td>\n",
              "      <td>14.746471</td>\n",
              "      <td>2.996175</td>\n",
              "      <td>78.773655</td>\n",
              "      <td>12.361551</td>\n",
              "      <td>0.005233</td>\n",
              "      <td>0.005803</td>\n",
              "      <td>0.015959</td>\n",
              "      <td>0.004814</td>\n",
              "      <td>0.004390</td>\n",
              "      <td>0.002434</td>\n",
              "      <td>0.120633</td>\n",
              "      <td>0.002984</td>\n",
              "      <td>0.015861</td>\n",
              "      <td>0.012295</td>\n",
              "      <td>0.008884</td>\n",
              "      <td>0.039484</td>\n",
              "      <td>...</td>\n",
              "      <td>0.002451</td>\n",
              "      <td>0.420075</td>\n",
              "      <td>0.306654</td>\n",
              "      <td>0.147405</td>\n",
              "      <td>0.000279</td>\n",
              "      <td>0.165155</td>\n",
              "      <td>0.011789</td>\n",
              "      <td>0.078353</td>\n",
              "      <td>0.988936</td>\n",
              "      <td>0.027461</td>\n",
              "      <td>0.000099</td>\n",
              "      <td>0.002724</td>\n",
              "      <td>0.000066</td>\n",
              "      <td>0.008259</td>\n",
              "      <td>11.711017</td>\n",
              "      <td>8.692865</td>\n",
              "      <td>0.016277</td>\n",
              "      <td>0.027779</td>\n",
              "      <td>0.001996</td>\n",
              "      <td>0.007305</td>\n",
              "      <td>21577.236256</td>\n",
              "      <td>166.880759</td>\n",
              "      <td>0.116692</td>\n",
              "      <td>-0.094338</td>\n",
              "      <td>-0.047668</td>\n",
              "      <td>-0.012898</td>\n",
              "      <td>0.056047</td>\n",
              "      <td>0.022257</td>\n",
              "      <td>0.061427</td>\n",
              "      <td>0.043563</td>\n",
              "      <td>-0.005482</td>\n",
              "      <td>-0.054468</td>\n",
              "      <td>0.031648</td>\n",
              "      <td>0.018106</td>\n",
              "      <td>0.054145</td>\n",
              "      <td>-0.051153</td>\n",
              "      <td>-0.005925</td>\n",
              "      <td>0.000058</td>\n",
              "      <td>0.046189</td>\n",
              "      <td>-0.002405</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.545874e+05</td>\n",
              "      <td>7.742546</td>\n",
              "      <td>100.368856</td>\n",
              "      <td>45.772164</td>\n",
              "      <td>383.926355</td>\n",
              "      <td>1.048002</td>\n",
              "      <td>153.150946</td>\n",
              "      <td>1.474350</td>\n",
              "      <td>0.264757</td>\n",
              "      <td>0.516615</td>\n",
              "      <td>23.518922</td>\n",
              "      <td>4.413584</td>\n",
              "      <td>0.981380</td>\n",
              "      <td>1700.110237</td>\n",
              "      <td>1.198870</td>\n",
              "      <td>6.577799</td>\n",
              "      <td>9.472052</td>\n",
              "      <td>187.192828</td>\n",
              "      <td>1.609271</td>\n",
              "      <td>0.593455</td>\n",
              "      <td>0.563034</td>\n",
              "      <td>4.639686</td>\n",
              "      <td>1.932338</td>\n",
              "      <td>0.155189</td>\n",
              "      <td>56.453297</td>\n",
              "      <td>11.869546</td>\n",
              "      <td>302.284160</td>\n",
              "      <td>49.438662</td>\n",
              "      <td>0.030464</td>\n",
              "      <td>0.042131</td>\n",
              "      <td>0.208596</td>\n",
              "      <td>0.041514</td>\n",
              "      <td>0.071241</td>\n",
              "      <td>0.015491</td>\n",
              "      <td>0.851130</td>\n",
              "      <td>0.017098</td>\n",
              "      <td>0.106631</td>\n",
              "      <td>0.070982</td>\n",
              "      <td>0.055063</td>\n",
              "      <td>1.007735</td>\n",
              "      <td>...</td>\n",
              "      <td>0.027812</td>\n",
              "      <td>4.227437</td>\n",
              "      <td>2.382598</td>\n",
              "      <td>1.910046</td>\n",
              "      <td>0.003049</td>\n",
              "      <td>3.143208</td>\n",
              "      <td>0.085169</td>\n",
              "      <td>1.058159</td>\n",
              "      <td>9.861488</td>\n",
              "      <td>0.208564</td>\n",
              "      <td>0.001648</td>\n",
              "      <td>0.028177</td>\n",
              "      <td>0.000529</td>\n",
              "      <td>0.084488</td>\n",
              "      <td>230.035809</td>\n",
              "      <td>177.214654</td>\n",
              "      <td>0.177415</td>\n",
              "      <td>0.311426</td>\n",
              "      <td>0.022198</td>\n",
              "      <td>0.077407</td>\n",
              "      <td>254633.088377</td>\n",
              "      <td>3965.821318</td>\n",
              "      <td>1.112802</td>\n",
              "      <td>0.695036</td>\n",
              "      <td>0.597310</td>\n",
              "      <td>0.509655</td>\n",
              "      <td>0.454499</td>\n",
              "      <td>0.449308</td>\n",
              "      <td>0.418755</td>\n",
              "      <td>0.426148</td>\n",
              "      <td>0.401053</td>\n",
              "      <td>0.343192</td>\n",
              "      <td>0.350124</td>\n",
              "      <td>0.332365</td>\n",
              "      <td>0.321075</td>\n",
              "      <td>0.293762</td>\n",
              "      <td>0.269050</td>\n",
              "      <td>0.286023</td>\n",
              "      <td>0.261000</td>\n",
              "      <td>0.270059</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>3.198770e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-139555.939272</td>\n",
              "      <td>-7413.329674</td>\n",
              "      <td>-3.349638</td>\n",
              "      <td>-1.585647</td>\n",
              "      <td>-1.527263</td>\n",
              "      <td>-1.442526</td>\n",
              "      <td>-1.189856</td>\n",
              "      <td>-1.064453</td>\n",
              "      <td>-1.077035</td>\n",
              "      <td>-0.829624</td>\n",
              "      <td>-0.933852</td>\n",
              "      <td>-0.969973</td>\n",
              "      <td>-1.097073</td>\n",
              "      <td>-0.920129</td>\n",
              "      <td>-1.006163</td>\n",
              "      <td>-1.000462</td>\n",
              "      <td>-0.798443</td>\n",
              "      <td>-0.685798</td>\n",
              "      <td>-0.669546</td>\n",
              "      <td>-0.809746</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>3.256138e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-133727.696151</td>\n",
              "      <td>-2357.200364</td>\n",
              "      <td>-0.795388</td>\n",
              "      <td>-0.666598</td>\n",
              "      <td>-0.477031</td>\n",
              "      <td>-0.343848</td>\n",
              "      <td>-0.235564</td>\n",
              "      <td>-0.286745</td>\n",
              "      <td>-0.177279</td>\n",
              "      <td>-0.293252</td>\n",
              "      <td>-0.303703</td>\n",
              "      <td>-0.332336</td>\n",
              "      <td>-0.199342</td>\n",
              "      <td>-0.221195</td>\n",
              "      <td>-0.159572</td>\n",
              "      <td>-0.217351</td>\n",
              "      <td>-0.192035</td>\n",
              "      <td>-0.174844</td>\n",
              "      <td>-0.132597</td>\n",
              "      <td>-0.158719</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>3.314425e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-127790.018350</td>\n",
              "      <td>-144.692521</td>\n",
              "      <td>-0.006754</td>\n",
              "      <td>-0.011181</td>\n",
              "      <td>-0.071705</td>\n",
              "      <td>-0.026870</td>\n",
              "      <td>0.013067</td>\n",
              "      <td>0.040808</td>\n",
              "      <td>0.038015</td>\n",
              "      <td>-0.032916</td>\n",
              "      <td>-0.006045</td>\n",
              "      <td>0.043369</td>\n",
              "      <td>0.064454</td>\n",
              "      <td>-0.007210</td>\n",
              "      <td>0.050987</td>\n",
              "      <td>-0.055414</td>\n",
              "      <td>-0.019909</td>\n",
              "      <td>-0.031226</td>\n",
              "      <td>0.015510</td>\n",
              "      <td>-0.034718</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>7.474640e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>288258.022725</td>\n",
              "      <td>3048.413899</td>\n",
              "      <td>0.982095</td>\n",
              "      <td>0.441213</td>\n",
              "      <td>0.323066</td>\n",
              "      <td>0.282833</td>\n",
              "      <td>0.264060</td>\n",
              "      <td>0.227521</td>\n",
              "      <td>0.305305</td>\n",
              "      <td>0.274315</td>\n",
              "      <td>0.267013</td>\n",
              "      <td>0.165323</td>\n",
              "      <td>0.246141</td>\n",
              "      <td>0.187702</td>\n",
              "      <td>0.251093</td>\n",
              "      <td>0.108230</td>\n",
              "      <td>0.191220</td>\n",
              "      <td>0.178754</td>\n",
              "      <td>0.197991</td>\n",
              "      <td>0.176335</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.105897e+06</td>\n",
              "      <td>99.400000</td>\n",
              "      <td>1390.000000</td>\n",
              "      <td>369.000000</td>\n",
              "      <td>2520.000000</td>\n",
              "      <td>7.660000</td>\n",
              "      <td>997.000000</td>\n",
              "      <td>12.700000</td>\n",
              "      <td>1.850000</td>\n",
              "      <td>7.870000</td>\n",
              "      <td>96.700000</td>\n",
              "      <td>99.600000</td>\n",
              "      <td>13.200000</td>\n",
              "      <td>40300.000000</td>\n",
              "      <td>11.500000</td>\n",
              "      <td>79.900000</td>\n",
              "      <td>43.400000</td>\n",
              "      <td>1650.000000</td>\n",
              "      <td>22.800000</td>\n",
              "      <td>8.610000</td>\n",
              "      <td>17.000000</td>\n",
              "      <td>73.200000</td>\n",
              "      <td>52.600000</td>\n",
              "      <td>3.400000</td>\n",
              "      <td>311.000000</td>\n",
              "      <td>91.000000</td>\n",
              "      <td>1710.000000</td>\n",
              "      <td>410.000000</td>\n",
              "      <td>0.474000</td>\n",
              "      <td>0.806000</td>\n",
              "      <td>7.950000</td>\n",
              "      <td>0.774000</td>\n",
              "      <td>1.830000</td>\n",
              "      <td>0.303000</td>\n",
              "      <td>14.000000</td>\n",
              "      <td>0.222000</td>\n",
              "      <td>1.770000</td>\n",
              "      <td>1.860000</td>\n",
              "      <td>0.989000</td>\n",
              "      <td>42.400000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.612000</td>\n",
              "      <td>73.200000</td>\n",
              "      <td>48.400000</td>\n",
              "      <td>57.600000</td>\n",
              "      <td>0.171000</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>1.640000</td>\n",
              "      <td>28.300000</td>\n",
              "      <td>348.000000</td>\n",
              "      <td>4.400000</td>\n",
              "      <td>0.086000</td>\n",
              "      <td>0.712000</td>\n",
              "      <td>0.022000</td>\n",
              "      <td>1.590000</td>\n",
              "      <td>7260.000000</td>\n",
              "      <td>8750.000000</td>\n",
              "      <td>5.020000</td>\n",
              "      <td>9.220000</td>\n",
              "      <td>0.630000</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>646711.616853</td>\n",
              "      <td>7635.496843</td>\n",
              "      <td>2.647689</td>\n",
              "      <td>1.869347</td>\n",
              "      <td>1.616849</td>\n",
              "      <td>2.041568</td>\n",
              "      <td>1.700615</td>\n",
              "      <td>1.764944</td>\n",
              "      <td>1.547783</td>\n",
              "      <td>1.348230</td>\n",
              "      <td>1.122909</td>\n",
              "      <td>1.167115</td>\n",
              "      <td>1.222555</td>\n",
              "      <td>1.289735</td>\n",
              "      <td>1.225573</td>\n",
              "      <td>1.032070</td>\n",
              "      <td>0.770407</td>\n",
              "      <td>0.843408</td>\n",
              "      <td>1.053437</td>\n",
              "      <td>0.875414</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8 rows × 101 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "             fdc_id          1004  ...            18            19\n",
              "count  1.048400e+04  10484.000000  ...  10484.000000  10484.000000\n",
              "mean   4.808406e+05      1.788952  ...      0.046189     -0.002405\n",
              "std    2.545874e+05      7.742546  ...      0.261000      0.270059\n",
              "min    3.198770e+05      0.000000  ...     -0.669546     -0.809746\n",
              "25%    3.256138e+05      0.000000  ...     -0.132597     -0.158719\n",
              "50%    3.314425e+05      0.000000  ...      0.015510     -0.034718\n",
              "75%    7.474640e+05      0.000000  ...      0.197991      0.176335\n",
              "max    1.105897e+06     99.400000  ...      1.053437      0.875414\n",
              "\n",
              "[8 rows x 101 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 74
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 350
        },
        "id": "__1kIK6D7GWH",
        "outputId": "b4d35094-7f08-4339-b052-51dff3001203"
      },
      "source": [
        "test_data.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>fdc_id</th>\n",
              "      <th>1004</th>\n",
              "      <th>1087</th>\n",
              "      <th>1090</th>\n",
              "      <th>1092</th>\n",
              "      <th>1095</th>\n",
              "      <th>1091</th>\n",
              "      <th>1089</th>\n",
              "      <th>1098</th>\n",
              "      <th>1101</th>\n",
              "      <th>1051</th>\n",
              "      <th>1007</th>\n",
              "      <th>1002</th>\n",
              "      <th>1093</th>\n",
              "      <th>1079</th>\n",
              "      <th>1003</th>\n",
              "      <th>1009</th>\n",
              "      <th>1102</th>\n",
              "      <th>1265</th>\n",
              "      <th>1266</th>\n",
              "      <th>1264</th>\n",
              "      <th>1315</th>\n",
              "      <th>1316</th>\n",
              "      <th>1314</th>\n",
              "      <th>1094</th>\n",
              "      <th>1097</th>\n",
              "      <th>1137</th>\n",
              "      <th>1146</th>\n",
              "      <th>1300</th>\n",
              "      <th>1299</th>\n",
              "      <th>1404</th>\n",
              "      <th>1267</th>\n",
              "      <th>1273</th>\n",
              "      <th>1323</th>\n",
              "      <th>1167</th>\n",
              "      <th>1271</th>\n",
              "      <th>1304</th>\n",
              "      <th>1166</th>\n",
              "      <th>1175</th>\n",
              "      <th>1263</th>\n",
              "      <th>...</th>\n",
              "      <th>1277</th>\n",
              "      <th>1292</th>\n",
              "      <th>1258</th>\n",
              "      <th>1293</th>\n",
              "      <th>1405</th>\n",
              "      <th>1010</th>\n",
              "      <th>1170</th>\n",
              "      <th>1109</th>\n",
              "      <th>1177</th>\n",
              "      <th>1178</th>\n",
              "      <th>1312</th>\n",
              "      <th>2009</th>\n",
              "      <th>1334</th>\n",
              "      <th>1257</th>\n",
              "      <th>1100</th>\n",
              "      <th>1107</th>\n",
              "      <th>1223</th>\n",
              "      <th>1224</th>\n",
              "      <th>1210</th>\n",
              "      <th>1211</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>2.621000e+03</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.00000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>4.803963e+05</td>\n",
              "      <td>1.967676</td>\n",
              "      <td>27.022510</td>\n",
              "      <td>14.536589</td>\n",
              "      <td>121.834033</td>\n",
              "      <td>0.396509</td>\n",
              "      <td>56.376955</td>\n",
              "      <td>0.485013</td>\n",
              "      <td>0.076968</td>\n",
              "      <td>0.149080</td>\n",
              "      <td>8.627738</td>\n",
              "      <td>0.495906</td>\n",
              "      <td>0.277390</td>\n",
              "      <td>143.940099</td>\n",
              "      <td>0.328348</td>\n",
              "      <td>1.955654</td>\n",
              "      <td>2.515960</td>\n",
              "      <td>42.943647</td>\n",
              "      <td>0.292919</td>\n",
              "      <td>0.115330</td>\n",
              "      <td>0.065867</td>\n",
              "      <td>0.485100</td>\n",
              "      <td>0.192602</td>\n",
              "      <td>0.021364</td>\n",
              "      <td>14.665013</td>\n",
              "      <td>2.933193</td>\n",
              "      <td>78.552079</td>\n",
              "      <td>11.678939</td>\n",
              "      <td>0.005403</td>\n",
              "      <td>0.006216</td>\n",
              "      <td>0.019760</td>\n",
              "      <td>0.003240</td>\n",
              "      <td>0.001788</td>\n",
              "      <td>0.002662</td>\n",
              "      <td>0.111663</td>\n",
              "      <td>0.003010</td>\n",
              "      <td>0.015391</td>\n",
              "      <td>0.012110</td>\n",
              "      <td>0.007625</td>\n",
              "      <td>0.032726</td>\n",
              "      <td>...</td>\n",
              "      <td>0.001202</td>\n",
              "      <td>0.492513</td>\n",
              "      <td>0.37134</td>\n",
              "      <td>0.162857</td>\n",
              "      <td>0.000279</td>\n",
              "      <td>0.074552</td>\n",
              "      <td>0.011859</td>\n",
              "      <td>0.055448</td>\n",
              "      <td>1.319344</td>\n",
              "      <td>0.027448</td>\n",
              "      <td>0.000068</td>\n",
              "      <td>0.003040</td>\n",
              "      <td>0.000063</td>\n",
              "      <td>0.007536</td>\n",
              "      <td>14.091606</td>\n",
              "      <td>9.781381</td>\n",
              "      <td>0.018074</td>\n",
              "      <td>0.028465</td>\n",
              "      <td>0.002101</td>\n",
              "      <td>0.007475</td>\n",
              "      <td>21132.004815</td>\n",
              "      <td>214.489880</td>\n",
              "      <td>0.131754</td>\n",
              "      <td>-0.092210</td>\n",
              "      <td>-0.051795</td>\n",
              "      <td>0.012438</td>\n",
              "      <td>0.033177</td>\n",
              "      <td>0.026538</td>\n",
              "      <td>0.070110</td>\n",
              "      <td>0.044096</td>\n",
              "      <td>-0.004377</td>\n",
              "      <td>-0.048841</td>\n",
              "      <td>0.037719</td>\n",
              "      <td>0.013325</td>\n",
              "      <td>0.052985</td>\n",
              "      <td>-0.045513</td>\n",
              "      <td>-0.010380</td>\n",
              "      <td>-0.006703</td>\n",
              "      <td>0.042393</td>\n",
              "      <td>-0.004267</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.537059e+05</td>\n",
              "      <td>8.210923</td>\n",
              "      <td>98.141815</td>\n",
              "      <td>45.690826</td>\n",
              "      <td>375.893218</td>\n",
              "      <td>1.084308</td>\n",
              "      <td>154.582160</td>\n",
              "      <td>1.519119</td>\n",
              "      <td>0.265481</td>\n",
              "      <td>0.511403</td>\n",
              "      <td>23.671963</td>\n",
              "      <td>4.430164</td>\n",
              "      <td>1.068915</td>\n",
              "      <td>1876.920647</td>\n",
              "      <td>1.133821</td>\n",
              "      <td>6.607275</td>\n",
              "      <td>9.345496</td>\n",
              "      <td>187.268411</td>\n",
              "      <td>1.590927</td>\n",
              "      <td>0.612892</td>\n",
              "      <td>0.542897</td>\n",
              "      <td>3.528785</td>\n",
              "      <td>1.977214</td>\n",
              "      <td>0.118853</td>\n",
              "      <td>56.496558</td>\n",
              "      <td>11.767255</td>\n",
              "      <td>304.333119</td>\n",
              "      <td>47.420738</td>\n",
              "      <td>0.032094</td>\n",
              "      <td>0.046780</td>\n",
              "      <td>0.276343</td>\n",
              "      <td>0.027657</td>\n",
              "      <td>0.031095</td>\n",
              "      <td>0.016725</td>\n",
              "      <td>0.731701</td>\n",
              "      <td>0.016862</td>\n",
              "      <td>0.104423</td>\n",
              "      <td>0.073456</td>\n",
              "      <td>0.054255</td>\n",
              "      <td>0.827105</td>\n",
              "      <td>...</td>\n",
              "      <td>0.013715</td>\n",
              "      <td>4.758129</td>\n",
              "      <td>3.11585</td>\n",
              "      <td>2.150577</td>\n",
              "      <td>0.002654</td>\n",
              "      <td>0.685144</td>\n",
              "      <td>0.084649</td>\n",
              "      <td>0.922500</td>\n",
              "      <td>11.107897</td>\n",
              "      <td>0.198013</td>\n",
              "      <td>0.000584</td>\n",
              "      <td>0.032298</td>\n",
              "      <td>0.000462</td>\n",
              "      <td>0.080481</td>\n",
              "      <td>265.353791</td>\n",
              "      <td>168.112176</td>\n",
              "      <td>0.198158</td>\n",
              "      <td>0.346407</td>\n",
              "      <td>0.024288</td>\n",
              "      <td>0.083169</td>\n",
              "      <td>253752.326858</td>\n",
              "      <td>3927.135566</td>\n",
              "      <td>1.110629</td>\n",
              "      <td>0.691941</td>\n",
              "      <td>0.602314</td>\n",
              "      <td>0.518598</td>\n",
              "      <td>0.450228</td>\n",
              "      <td>0.457808</td>\n",
              "      <td>0.412216</td>\n",
              "      <td>0.425847</td>\n",
              "      <td>0.396720</td>\n",
              "      <td>0.354490</td>\n",
              "      <td>0.347314</td>\n",
              "      <td>0.330849</td>\n",
              "      <td>0.322132</td>\n",
              "      <td>0.289392</td>\n",
              "      <td>0.273918</td>\n",
              "      <td>0.280586</td>\n",
              "      <td>0.264559</td>\n",
              "      <td>0.265600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>3.198780e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-139554.920584</td>\n",
              "      <td>-7410.386832</td>\n",
              "      <td>-2.489668</td>\n",
              "      <td>-1.577342</td>\n",
              "      <td>-1.433074</td>\n",
              "      <td>-1.442400</td>\n",
              "      <td>-1.189727</td>\n",
              "      <td>-0.922758</td>\n",
              "      <td>-1.021707</td>\n",
              "      <td>-0.808912</td>\n",
              "      <td>-0.933827</td>\n",
              "      <td>-0.946624</td>\n",
              "      <td>-0.961907</td>\n",
              "      <td>-0.890940</td>\n",
              "      <td>-1.006213</td>\n",
              "      <td>-0.894695</td>\n",
              "      <td>-0.797834</td>\n",
              "      <td>-0.601732</td>\n",
              "      <td>-0.616257</td>\n",
              "      <td>-0.809725</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>3.257840e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-133554.264517</td>\n",
              "      <td>-2265.726243</td>\n",
              "      <td>-0.797956</td>\n",
              "      <td>-0.669478</td>\n",
              "      <td>-0.477206</td>\n",
              "      <td>-0.343567</td>\n",
              "      <td>-0.248446</td>\n",
              "      <td>-0.286810</td>\n",
              "      <td>-0.176744</td>\n",
              "      <td>-0.286960</td>\n",
              "      <td>-0.293944</td>\n",
              "      <td>-0.331787</td>\n",
              "      <td>-0.178625</td>\n",
              "      <td>-0.218948</td>\n",
              "      <td>-0.163446</td>\n",
              "      <td>-0.216294</td>\n",
              "      <td>-0.199150</td>\n",
              "      <td>-0.181633</td>\n",
              "      <td>-0.126746</td>\n",
              "      <td>-0.159625</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>3.311770e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-128060.480019</td>\n",
              "      <td>-165.539435</td>\n",
              "      <td>0.024403</td>\n",
              "      <td>-0.016796</td>\n",
              "      <td>-0.072881</td>\n",
              "      <td>-0.007731</td>\n",
              "      <td>-0.004707</td>\n",
              "      <td>0.041421</td>\n",
              "      <td>0.041124</td>\n",
              "      <td>-0.031619</td>\n",
              "      <td>-0.011872</td>\n",
              "      <td>0.047214</td>\n",
              "      <td>0.064470</td>\n",
              "      <td>-0.006434</td>\n",
              "      <td>0.051343</td>\n",
              "      <td>-0.050258</td>\n",
              "      <td>-0.021204</td>\n",
              "      <td>-0.041344</td>\n",
              "      <td>0.011042</td>\n",
              "      <td>-0.038586</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>7.474740e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>288268.209605</td>\n",
              "      <td>2978.847360</td>\n",
              "      <td>0.989184</td>\n",
              "      <td>0.440794</td>\n",
              "      <td>0.338293</td>\n",
              "      <td>0.321224</td>\n",
              "      <td>0.237860</td>\n",
              "      <td>0.227693</td>\n",
              "      <td>0.328289</td>\n",
              "      <td>0.275653</td>\n",
              "      <td>0.267565</td>\n",
              "      <td>0.169586</td>\n",
              "      <td>0.254462</td>\n",
              "      <td>0.189796</td>\n",
              "      <td>0.250228</td>\n",
              "      <td>0.110422</td>\n",
              "      <td>0.190131</td>\n",
              "      <td>0.174171</td>\n",
              "      <td>0.193967</td>\n",
              "      <td>0.170483</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.105891e+06</td>\n",
              "      <td>99.100000</td>\n",
              "      <td>1070.000000</td>\n",
              "      <td>386.000000</td>\n",
              "      <td>1800.000000</td>\n",
              "      <td>7.440000</td>\n",
              "      <td>997.000000</td>\n",
              "      <td>12.300000</td>\n",
              "      <td>1.920000</td>\n",
              "      <td>3.870000</td>\n",
              "      <td>95.200000</td>\n",
              "      <td>99.300000</td>\n",
              "      <td>13.200000</td>\n",
              "      <td>40700.000000</td>\n",
              "      <td>11.400000</td>\n",
              "      <td>34.200000</td>\n",
              "      <td>43.900000</td>\n",
              "      <td>1790.000000</td>\n",
              "      <td>22.200000</td>\n",
              "      <td>7.470000</td>\n",
              "      <td>16.700000</td>\n",
              "      <td>70.500000</td>\n",
              "      <td>52.300000</td>\n",
              "      <td>1.450000</td>\n",
              "      <td>284.000000</td>\n",
              "      <td>93.000000</td>\n",
              "      <td>1650.000000</td>\n",
              "      <td>449.000000</td>\n",
              "      <td>0.485000</td>\n",
              "      <td>0.850000</td>\n",
              "      <td>7.720000</td>\n",
              "      <td>0.663000</td>\n",
              "      <td>1.460000</td>\n",
              "      <td>0.285000</td>\n",
              "      <td>13.700000</td>\n",
              "      <td>0.192000</td>\n",
              "      <td>1.930000</td>\n",
              "      <td>1.690000</td>\n",
              "      <td>0.870000</td>\n",
              "      <td>41.800000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.576000</td>\n",
              "      <td>74.200000</td>\n",
              "      <td>82.50000</td>\n",
              "      <td>58.100000</td>\n",
              "      <td>0.043000</td>\n",
              "      <td>18.800000</td>\n",
              "      <td>1.300000</td>\n",
              "      <td>27.600000</td>\n",
              "      <td>172.000000</td>\n",
              "      <td>3.900000</td>\n",
              "      <td>0.011000</td>\n",
              "      <td>0.687000</td>\n",
              "      <td>0.008000</td>\n",
              "      <td>1.180000</td>\n",
              "      <td>7430.000000</td>\n",
              "      <td>5260.000000</td>\n",
              "      <td>5.320000</td>\n",
              "      <td>9.700000</td>\n",
              "      <td>0.620000</td>\n",
              "      <td>2.100000</td>\n",
              "      <td>646705.504729</td>\n",
              "      <td>7634.515887</td>\n",
              "      <td>2.647480</td>\n",
              "      <td>1.869342</td>\n",
              "      <td>1.560331</td>\n",
              "      <td>1.990330</td>\n",
              "      <td>1.592393</td>\n",
              "      <td>1.764828</td>\n",
              "      <td>1.361376</td>\n",
              "      <td>1.224556</td>\n",
              "      <td>1.068824</td>\n",
              "      <td>1.235692</td>\n",
              "      <td>1.230902</td>\n",
              "      <td>1.301706</td>\n",
              "      <td>1.182738</td>\n",
              "      <td>1.032062</td>\n",
              "      <td>0.746230</td>\n",
              "      <td>0.702585</td>\n",
              "      <td>1.053217</td>\n",
              "      <td>0.875425</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8 rows × 101 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "             fdc_id         1004  ...           18           19\n",
              "count  2.621000e+03  2621.000000  ...  2621.000000  2621.000000\n",
              "mean   4.803963e+05     1.967676  ...     0.042393    -0.004267\n",
              "std    2.537059e+05     8.210923  ...     0.264559     0.265600\n",
              "min    3.198780e+05     0.000000  ...    -0.616257    -0.809725\n",
              "25%    3.257840e+05     0.000000  ...    -0.126746    -0.159625\n",
              "50%    3.311770e+05     0.000000  ...     0.011042    -0.038586\n",
              "75%    7.474740e+05     0.000000  ...     0.193967     0.170483\n",
              "max    1.105891e+06    99.100000  ...     1.053217     0.875425\n",
              "\n",
              "[8 rows x 101 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 75
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "riwskT9I7GWH",
        "outputId": "ef6ed999-e70c-431c-963f-01f603ed1b60"
      },
      "source": [
        "indices = test_data['fdc_id']\n",
        "indices"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "9313      336041\n",
              "2440      324243\n",
              "2257      323937\n",
              "1041      322215\n",
              "12468    1105140\n",
              "          ...   \n",
              "11687     790512\n",
              "3464      325917\n",
              "11720     790553\n",
              "2688      324702\n",
              "5900      329898\n",
              "Name: fdc_id, Length: 2621, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 76
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R8opNyxm7GWI"
      },
      "source": [
        "del train_data['fdc_id']\n",
        "del test_data['fdc_id']\n",
        "columns = train_data.columns"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pUfqcrq87GWJ",
        "outputId": "c7ebda6f-03e8-43a7-937e-48536968e961"
      },
      "source": [
        "train_data = train_data.to_numpy()\n",
        "test_data.fillna(0, inplace=True)\n",
        "test_data = test_data.to_numpy()\n",
        "data = dataset.values\n",
        "rows, cols = data.shape\n",
        "cols -= 1\n",
        "\n",
        "y_test = test_data.copy()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pandas/core/frame.py:4327: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  downcast=downcast,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 350
        },
        "id": "RPUmJJgf7GWJ",
        "outputId": "c55ec02a-87d6-48bd-f504-657236e80422"
      },
      "source": [
        "missed_data, missed_data_train, mask = missing_method(test_data, train_data, num_embeddings)\n",
        "pd.DataFrame(missed_data).describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "      <th>30</th>\n",
              "      <th>31</th>\n",
              "      <th>32</th>\n",
              "      <th>33</th>\n",
              "      <th>34</th>\n",
              "      <th>35</th>\n",
              "      <th>36</th>\n",
              "      <th>37</th>\n",
              "      <th>38</th>\n",
              "      <th>39</th>\n",
              "      <th>...</th>\n",
              "      <th>60</th>\n",
              "      <th>61</th>\n",
              "      <th>62</th>\n",
              "      <th>63</th>\n",
              "      <th>64</th>\n",
              "      <th>65</th>\n",
              "      <th>66</th>\n",
              "      <th>67</th>\n",
              "      <th>68</th>\n",
              "      <th>69</th>\n",
              "      <th>70</th>\n",
              "      <th>71</th>\n",
              "      <th>72</th>\n",
              "      <th>73</th>\n",
              "      <th>74</th>\n",
              "      <th>75</th>\n",
              "      <th>76</th>\n",
              "      <th>77</th>\n",
              "      <th>78</th>\n",
              "      <th>79</th>\n",
              "      <th>80</th>\n",
              "      <th>81</th>\n",
              "      <th>82</th>\n",
              "      <th>83</th>\n",
              "      <th>84</th>\n",
              "      <th>85</th>\n",
              "      <th>86</th>\n",
              "      <th>87</th>\n",
              "      <th>88</th>\n",
              "      <th>89</th>\n",
              "      <th>90</th>\n",
              "      <th>91</th>\n",
              "      <th>92</th>\n",
              "      <th>93</th>\n",
              "      <th>94</th>\n",
              "      <th>95</th>\n",
              "      <th>96</th>\n",
              "      <th>97</th>\n",
              "      <th>98</th>\n",
              "      <th>99</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>2109.000000</td>\n",
              "      <td>2078.000000</td>\n",
              "      <td>2092.00000</td>\n",
              "      <td>2100.000000</td>\n",
              "      <td>2123.00000</td>\n",
              "      <td>2085.000000</td>\n",
              "      <td>2137.000000</td>\n",
              "      <td>2119.000000</td>\n",
              "      <td>2110.000000</td>\n",
              "      <td>2086.000000</td>\n",
              "      <td>2088.000000</td>\n",
              "      <td>2079.000000</td>\n",
              "      <td>2117.000000</td>\n",
              "      <td>2126.000000</td>\n",
              "      <td>2125.000000</td>\n",
              "      <td>2110.000000</td>\n",
              "      <td>2120.000000</td>\n",
              "      <td>2138.000000</td>\n",
              "      <td>2114.000000</td>\n",
              "      <td>2104.000000</td>\n",
              "      <td>2088.000000</td>\n",
              "      <td>2118.000000</td>\n",
              "      <td>2109.000000</td>\n",
              "      <td>2123.00000</td>\n",
              "      <td>2083.000000</td>\n",
              "      <td>2091.000000</td>\n",
              "      <td>2100.000000</td>\n",
              "      <td>2140.000000</td>\n",
              "      <td>2087.000000</td>\n",
              "      <td>2087.000000</td>\n",
              "      <td>2097.000000</td>\n",
              "      <td>2130.000000</td>\n",
              "      <td>2107.000000</td>\n",
              "      <td>2058.000000</td>\n",
              "      <td>2122.000000</td>\n",
              "      <td>2120.000000</td>\n",
              "      <td>2065.000000</td>\n",
              "      <td>2095.000000</td>\n",
              "      <td>2091.000000</td>\n",
              "      <td>2093.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>2076.000000</td>\n",
              "      <td>2107.000000</td>\n",
              "      <td>2101.000000</td>\n",
              "      <td>2092.000000</td>\n",
              "      <td>2125.000000</td>\n",
              "      <td>2102.000000</td>\n",
              "      <td>2086.000000</td>\n",
              "      <td>2085.000000</td>\n",
              "      <td>2105.000000</td>\n",
              "      <td>2094.000000</td>\n",
              "      <td>2108.00000</td>\n",
              "      <td>2133.000000</td>\n",
              "      <td>2068.000000</td>\n",
              "      <td>2096.000000</td>\n",
              "      <td>2046.000000</td>\n",
              "      <td>2067.000000</td>\n",
              "      <td>2111.000000</td>\n",
              "      <td>2099.000000</td>\n",
              "      <td>2094.000000</td>\n",
              "      <td>2092.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>1.967435</td>\n",
              "      <td>26.703080</td>\n",
              "      <td>13.71305</td>\n",
              "      <td>120.066667</td>\n",
              "      <td>0.39764</td>\n",
              "      <td>58.003357</td>\n",
              "      <td>0.493776</td>\n",
              "      <td>0.080866</td>\n",
              "      <td>0.152113</td>\n",
              "      <td>8.641716</td>\n",
              "      <td>0.500747</td>\n",
              "      <td>0.254367</td>\n",
              "      <td>161.042985</td>\n",
              "      <td>0.323612</td>\n",
              "      <td>1.978193</td>\n",
              "      <td>2.486318</td>\n",
              "      <td>43.015943</td>\n",
              "      <td>0.297366</td>\n",
              "      <td>0.109253</td>\n",
              "      <td>0.058020</td>\n",
              "      <td>0.437340</td>\n",
              "      <td>0.183017</td>\n",
              "      <td>0.021095</td>\n",
              "      <td>14.16910</td>\n",
              "      <td>3.165098</td>\n",
              "      <td>73.178384</td>\n",
              "      <td>11.415000</td>\n",
              "      <td>0.005707</td>\n",
              "      <td>0.005720</td>\n",
              "      <td>0.021977</td>\n",
              "      <td>0.003471</td>\n",
              "      <td>0.002084</td>\n",
              "      <td>0.002667</td>\n",
              "      <td>0.103992</td>\n",
              "      <td>0.003213</td>\n",
              "      <td>0.014764</td>\n",
              "      <td>0.011882</td>\n",
              "      <td>0.008267</td>\n",
              "      <td>0.037143</td>\n",
              "      <td>0.006607</td>\n",
              "      <td>...</td>\n",
              "      <td>0.001267</td>\n",
              "      <td>0.508391</td>\n",
              "      <td>0.367177</td>\n",
              "      <td>0.180243</td>\n",
              "      <td>0.000321</td>\n",
              "      <td>0.071613</td>\n",
              "      <td>0.011859</td>\n",
              "      <td>0.054974</td>\n",
              "      <td>1.316390</td>\n",
              "      <td>0.025907</td>\n",
              "      <td>0.00008</td>\n",
              "      <td>0.003531</td>\n",
              "      <td>0.000056</td>\n",
              "      <td>0.009091</td>\n",
              "      <td>15.702639</td>\n",
              "      <td>7.275278</td>\n",
              "      <td>0.017606</td>\n",
              "      <td>0.033414</td>\n",
              "      <td>0.001969</td>\n",
              "      <td>0.007199</td>\n",
              "      <td>21132.004815</td>\n",
              "      <td>214.489880</td>\n",
              "      <td>0.131754</td>\n",
              "      <td>-0.092210</td>\n",
              "      <td>-0.051795</td>\n",
              "      <td>0.012438</td>\n",
              "      <td>0.033177</td>\n",
              "      <td>0.026538</td>\n",
              "      <td>0.070110</td>\n",
              "      <td>0.044096</td>\n",
              "      <td>-0.004377</td>\n",
              "      <td>-0.048841</td>\n",
              "      <td>0.037719</td>\n",
              "      <td>0.013325</td>\n",
              "      <td>0.052985</td>\n",
              "      <td>-0.045513</td>\n",
              "      <td>-0.010380</td>\n",
              "      <td>-0.006703</td>\n",
              "      <td>0.042393</td>\n",
              "      <td>-0.004267</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>8.333341</td>\n",
              "      <td>97.186923</td>\n",
              "      <td>43.91800</td>\n",
              "      <td>372.767341</td>\n",
              "      <td>1.09067</td>\n",
              "      <td>155.870192</td>\n",
              "      <td>1.527220</td>\n",
              "      <td>0.272309</td>\n",
              "      <td>0.514505</td>\n",
              "      <td>23.644255</td>\n",
              "      <td>4.442314</td>\n",
              "      <td>0.983221</td>\n",
              "      <td>2083.637258</td>\n",
              "      <td>1.131511</td>\n",
              "      <td>6.652319</td>\n",
              "      <td>9.294418</td>\n",
              "      <td>185.051257</td>\n",
              "      <td>1.647440</td>\n",
              "      <td>0.594519</td>\n",
              "      <td>0.426314</td>\n",
              "      <td>3.169038</td>\n",
              "      <td>1.860204</td>\n",
              "      <td>0.115439</td>\n",
              "      <td>55.54696</td>\n",
              "      <td>12.216444</td>\n",
              "      <td>294.649725</td>\n",
              "      <td>45.966284</td>\n",
              "      <td>0.032902</td>\n",
              "      <td>0.045211</td>\n",
              "      <td>0.307572</td>\n",
              "      <td>0.029392</td>\n",
              "      <td>0.034465</td>\n",
              "      <td>0.016328</td>\n",
              "      <td>0.673840</td>\n",
              "      <td>0.017801</td>\n",
              "      <td>0.097919</td>\n",
              "      <td>0.069045</td>\n",
              "      <td>0.057652</td>\n",
              "      <td>0.923418</td>\n",
              "      <td>0.058450</td>\n",
              "      <td>...</td>\n",
              "      <td>0.015056</td>\n",
              "      <td>4.994738</td>\n",
              "      <td>3.059325</td>\n",
              "      <td>2.373638</td>\n",
              "      <td>0.002881</td>\n",
              "      <td>0.651726</td>\n",
              "      <td>0.083499</td>\n",
              "      <td>0.908443</td>\n",
              "      <td>11.203809</td>\n",
              "      <td>0.186068</td>\n",
              "      <td>0.00064</td>\n",
              "      <td>0.035284</td>\n",
              "      <td>0.000442</td>\n",
              "      <td>0.089476</td>\n",
              "      <td>285.237165</td>\n",
              "      <td>158.288625</td>\n",
              "      <td>0.207159</td>\n",
              "      <td>0.383715</td>\n",
              "      <td>0.021935</td>\n",
              "      <td>0.081271</td>\n",
              "      <td>253752.326858</td>\n",
              "      <td>3927.135566</td>\n",
              "      <td>1.110629</td>\n",
              "      <td>0.691941</td>\n",
              "      <td>0.602314</td>\n",
              "      <td>0.518598</td>\n",
              "      <td>0.450228</td>\n",
              "      <td>0.457808</td>\n",
              "      <td>0.412216</td>\n",
              "      <td>0.425847</td>\n",
              "      <td>0.396720</td>\n",
              "      <td>0.354490</td>\n",
              "      <td>0.347314</td>\n",
              "      <td>0.330849</td>\n",
              "      <td>0.322132</td>\n",
              "      <td>0.289392</td>\n",
              "      <td>0.273918</td>\n",
              "      <td>0.280586</td>\n",
              "      <td>0.264559</td>\n",
              "      <td>0.265600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-139554.920584</td>\n",
              "      <td>-7410.386832</td>\n",
              "      <td>-2.489668</td>\n",
              "      <td>-1.577342</td>\n",
              "      <td>-1.433074</td>\n",
              "      <td>-1.442400</td>\n",
              "      <td>-1.189727</td>\n",
              "      <td>-0.922758</td>\n",
              "      <td>-1.021707</td>\n",
              "      <td>-0.808912</td>\n",
              "      <td>-0.933827</td>\n",
              "      <td>-0.946624</td>\n",
              "      <td>-0.961907</td>\n",
              "      <td>-0.890940</td>\n",
              "      <td>-1.006213</td>\n",
              "      <td>-0.894695</td>\n",
              "      <td>-0.797834</td>\n",
              "      <td>-0.601732</td>\n",
              "      <td>-0.616257</td>\n",
              "      <td>-0.809725</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-133554.264517</td>\n",
              "      <td>-2265.726243</td>\n",
              "      <td>-0.797956</td>\n",
              "      <td>-0.669478</td>\n",
              "      <td>-0.477206</td>\n",
              "      <td>-0.343567</td>\n",
              "      <td>-0.248446</td>\n",
              "      <td>-0.286810</td>\n",
              "      <td>-0.176744</td>\n",
              "      <td>-0.286960</td>\n",
              "      <td>-0.293944</td>\n",
              "      <td>-0.331787</td>\n",
              "      <td>-0.178625</td>\n",
              "      <td>-0.218948</td>\n",
              "      <td>-0.163446</td>\n",
              "      <td>-0.216294</td>\n",
              "      <td>-0.199150</td>\n",
              "      <td>-0.181633</td>\n",
              "      <td>-0.126746</td>\n",
              "      <td>-0.159625</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-128060.480019</td>\n",
              "      <td>-165.539435</td>\n",
              "      <td>0.024403</td>\n",
              "      <td>-0.016796</td>\n",
              "      <td>-0.072881</td>\n",
              "      <td>-0.007731</td>\n",
              "      <td>-0.004707</td>\n",
              "      <td>0.041421</td>\n",
              "      <td>0.041124</td>\n",
              "      <td>-0.031619</td>\n",
              "      <td>-0.011872</td>\n",
              "      <td>0.047214</td>\n",
              "      <td>0.064470</td>\n",
              "      <td>-0.006434</td>\n",
              "      <td>0.051343</td>\n",
              "      <td>-0.050258</td>\n",
              "      <td>-0.021204</td>\n",
              "      <td>-0.041344</td>\n",
              "      <td>0.011042</td>\n",
              "      <td>-0.038586</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>288268.209605</td>\n",
              "      <td>2978.847360</td>\n",
              "      <td>0.989184</td>\n",
              "      <td>0.440794</td>\n",
              "      <td>0.338293</td>\n",
              "      <td>0.321224</td>\n",
              "      <td>0.237860</td>\n",
              "      <td>0.227693</td>\n",
              "      <td>0.328289</td>\n",
              "      <td>0.275653</td>\n",
              "      <td>0.267565</td>\n",
              "      <td>0.169586</td>\n",
              "      <td>0.254462</td>\n",
              "      <td>0.189796</td>\n",
              "      <td>0.250228</td>\n",
              "      <td>0.110422</td>\n",
              "      <td>0.190131</td>\n",
              "      <td>0.174171</td>\n",
              "      <td>0.193967</td>\n",
              "      <td>0.170483</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>99.100000</td>\n",
              "      <td>1070.000000</td>\n",
              "      <td>350.00000</td>\n",
              "      <td>1800.000000</td>\n",
              "      <td>7.44000</td>\n",
              "      <td>997.000000</td>\n",
              "      <td>12.300000</td>\n",
              "      <td>1.920000</td>\n",
              "      <td>3.870000</td>\n",
              "      <td>94.700000</td>\n",
              "      <td>99.300000</td>\n",
              "      <td>13.200000</td>\n",
              "      <td>40700.000000</td>\n",
              "      <td>11.400000</td>\n",
              "      <td>34.200000</td>\n",
              "      <td>43.900000</td>\n",
              "      <td>1790.000000</td>\n",
              "      <td>22.200000</td>\n",
              "      <td>7.380000</td>\n",
              "      <td>7.740000</td>\n",
              "      <td>67.800000</td>\n",
              "      <td>52.300000</td>\n",
              "      <td>1.340000</td>\n",
              "      <td>284.00000</td>\n",
              "      <td>93.000000</td>\n",
              "      <td>1650.000000</td>\n",
              "      <td>294.000000</td>\n",
              "      <td>0.485000</td>\n",
              "      <td>0.850000</td>\n",
              "      <td>7.720000</td>\n",
              "      <td>0.663000</td>\n",
              "      <td>1.460000</td>\n",
              "      <td>0.285000</td>\n",
              "      <td>7.500000</td>\n",
              "      <td>0.192000</td>\n",
              "      <td>1.390000</td>\n",
              "      <td>1.690000</td>\n",
              "      <td>0.870000</td>\n",
              "      <td>41.800000</td>\n",
              "      <td>1.090000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.576000</td>\n",
              "      <td>74.200000</td>\n",
              "      <td>82.500000</td>\n",
              "      <td>58.100000</td>\n",
              "      <td>0.043000</td>\n",
              "      <td>18.800000</td>\n",
              "      <td>1.300000</td>\n",
              "      <td>27.600000</td>\n",
              "      <td>172.000000</td>\n",
              "      <td>3.900000</td>\n",
              "      <td>0.01100</td>\n",
              "      <td>0.687000</td>\n",
              "      <td>0.008000</td>\n",
              "      <td>1.180000</td>\n",
              "      <td>7430.000000</td>\n",
              "      <td>5260.000000</td>\n",
              "      <td>5.320000</td>\n",
              "      <td>9.700000</td>\n",
              "      <td>0.600000</td>\n",
              "      <td>2.100000</td>\n",
              "      <td>646705.504729</td>\n",
              "      <td>7634.515887</td>\n",
              "      <td>2.647480</td>\n",
              "      <td>1.869342</td>\n",
              "      <td>1.560331</td>\n",
              "      <td>1.990330</td>\n",
              "      <td>1.592393</td>\n",
              "      <td>1.764828</td>\n",
              "      <td>1.361376</td>\n",
              "      <td>1.224556</td>\n",
              "      <td>1.068824</td>\n",
              "      <td>1.235692</td>\n",
              "      <td>1.230902</td>\n",
              "      <td>1.301706</td>\n",
              "      <td>1.182738</td>\n",
              "      <td>1.032062</td>\n",
              "      <td>0.746230</td>\n",
              "      <td>0.702585</td>\n",
              "      <td>1.053217</td>\n",
              "      <td>0.875425</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8 rows × 100 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                0            1   ...           98           99\n",
              "count  2109.000000  2078.000000  ...  2621.000000  2621.000000\n",
              "mean      1.967435    26.703080  ...     0.042393    -0.004267\n",
              "std       8.333341    97.186923  ...     0.264559     0.265600\n",
              "min       0.000000     0.000000  ...    -0.616257    -0.809725\n",
              "25%       0.000000     0.000000  ...    -0.126746    -0.159625\n",
              "50%       0.000000     0.000000  ...     0.011042    -0.038586\n",
              "75%       0.000000     0.000000  ...     0.193967     0.170483\n",
              "max      99.100000  1070.000000  ...     1.053217     0.875425\n",
              "\n",
              "[8 rows x 100 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 79
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5DrSWUEb7GWK"
      },
      "source": [
        "# traditional methods test sets - impute across all foods\n",
        "X_train = pd.DataFrame(missed_data_train, columns = columns)\n",
        "X_test_mean_all = pd.DataFrame(missed_data, columns = columns)\n",
        "X_test_median_all = pd.DataFrame(missed_data, columns = columns)\n",
        "\n",
        "X_test_mean_all = impute_traditional_all_foods(X_train, X_test_mean_all, 'mean')\n",
        "X_test_median_all = impute_traditional_all_foods(X_train, X_test_median_all, 'median')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jI2Zj87o7GWK"
      },
      "source": [
        "# Traditional methods test sets - impute by food category\n",
        "X_test_mean = pd.DataFrame(missed_data, columns = columns, index=indices)\n",
        "X_test_median = X_test_mean.copy()\n",
        "df_mean_cat = df_mean[df_mean['fdc_id'].isin(indices)]\n",
        "X_test_mean = impute_traditional_by_category(X_test_mean, df_mean_cat, indices)\n",
        "df_median_cat = df_median[df_median['fdc_id'].isin(indices)]\n",
        "X_test_median = impute_traditional_by_category(X_test_median, df_median_cat, indices)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EeHLEnz-7GWL"
      },
      "source": [
        "scaler = MinMaxScaler()\n",
        "scaler.fit(train_data)\n",
        "train_data = scaler.transform(train_data)\n",
        "test_data = scaler.transform(test_data)\n",
        "X_test_mean = scaler.transform(X_test_mean)\n",
        "X_test_median = scaler.transform(X_test_median)\n",
        "X_test_mean_all = scaler.transform(X_test_mean_all)\n",
        "X_test_median_all = scaler.transform(X_test_median_all)\n",
        "\n",
        "y_test = scaler.transform(y_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gsrv2M2A7GWL"
      },
      "source": [
        "# datasets without embeddings for autoencoder\n",
        "train_data_noEmbeddings = pd.DataFrame(train_data, columns = columns)\n",
        "train_data_noEmbeddings = np.array(train_data_noEmbeddings.iloc[:, :-num_embeddings]) # df = df.iloc[: , :-1]\n",
        "test_mean_noEmbeddings = np.array(pd.DataFrame(X_test_mean, columns = columns).iloc[:, :-num_embeddings])\n",
        "test_median_noEmbeddings = np.array(pd.DataFrame(X_test_median, columns = columns).iloc[:, :-num_embeddings])\n",
        "test_mean_noEmbeddings_all = np.array(pd.DataFrame(X_test_mean_all, columns = columns).iloc[:, :-num_embeddings])\n",
        "test_median_noEmbeddings_all = np.array(pd.DataFrame(X_test_median_all, columns = columns).iloc[:, :-num_embeddings])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KGg9vpv17GWM"
      },
      "source": [
        "missed_data = X_test_mean_all\n",
        "missed_data = torch.from_numpy(missed_data).float()\n",
        "\n",
        "train_data = torch.from_numpy(train_data).float()\n",
        "\n",
        "train_loader = torch.utils.data.DataLoader(dataset=train_data,\n",
        "                                           batch_size=batch_size,\n",
        "                                           shuffle=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XuTlxeIi7GWN"
      },
      "source": [
        "missed_data_noEm = test_mean_noEmbeddings_all\n",
        "missed_data_noEm = torch.from_numpy(missed_data_noEm).float()\n",
        "\n",
        "train_data_noEmbeddings = torch.from_numpy(train_data_noEmbeddings).float()\n",
        "\n",
        "train_loader_noEm = torch.utils.data.DataLoader(dataset=train_data_noEmbeddings,\n",
        "                                           batch_size=batch_size,\n",
        "                                           shuffle=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RJCuBLa67GWO"
      },
      "source": [
        "## Autoencoder - without embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A5jIEm4u7GWO"
      },
      "source": [
        "model = Autoencoder(dim=cols-num_embeddings).to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D7-qienM7GWP"
      },
      "source": [
        "criterion = nn.MSELoss()\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UxZuClsg7GWP",
        "outputId": "60e9488b-0a85-47ad-b3cc-b2b03e11dd98"
      },
      "source": [
        "training(num_epochs, model, train_loader_noEm, criterion, optimizer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch : 1/200, recon loss = 0.00723645\n",
            "epoch : 2/200, recon loss = 0.00623200\n",
            "epoch : 3/200, recon loss = 0.00608631\n",
            "epoch : 4/200, recon loss = 0.00575263\n",
            "epoch : 5/200, recon loss = 0.00497927\n",
            "epoch : 6/200, recon loss = 0.00397156\n",
            "epoch : 7/200, recon loss = 0.00338542\n",
            "epoch : 8/200, recon loss = 0.00317286\n",
            "epoch : 9/200, recon loss = 0.00310097\n",
            "epoch : 10/200, recon loss = 0.00307003\n",
            "epoch : 11/200, recon loss = 0.00305066\n",
            "epoch : 12/200, recon loss = 0.00301768\n",
            "epoch : 13/200, recon loss = 0.00299938\n",
            "epoch : 14/200, recon loss = 0.00296950\n",
            "epoch : 15/200, recon loss = 0.00292809\n",
            "epoch : 16/200, recon loss = 0.00288915\n",
            "epoch : 17/200, recon loss = 0.00283167\n",
            "epoch : 18/200, recon loss = 0.00278632\n",
            "epoch : 19/200, recon loss = 0.00272362\n",
            "epoch : 20/200, recon loss = 0.00268940\n",
            "epoch : 21/200, recon loss = 0.00263934\n",
            "epoch : 22/200, recon loss = 0.00258300\n",
            "epoch : 23/200, recon loss = 0.00256760\n",
            "epoch : 24/200, recon loss = 0.00254809\n",
            "epoch : 25/200, recon loss = 0.00252104\n",
            "epoch : 26/200, recon loss = 0.00247968\n",
            "epoch : 27/200, recon loss = 0.00246973\n",
            "epoch : 28/200, recon loss = 0.00244818\n",
            "epoch : 29/200, recon loss = 0.00242328\n",
            "epoch : 30/200, recon loss = 0.00241681\n",
            "epoch : 31/200, recon loss = 0.00237925\n",
            "epoch : 32/200, recon loss = 0.00235772\n",
            "epoch : 33/200, recon loss = 0.00233196\n",
            "epoch : 34/200, recon loss = 0.00230074\n",
            "epoch : 35/200, recon loss = 0.00224741\n",
            "epoch : 36/200, recon loss = 0.00220567\n",
            "epoch : 37/200, recon loss = 0.00216504\n",
            "epoch : 38/200, recon loss = 0.00212909\n",
            "epoch : 39/200, recon loss = 0.00208599\n",
            "epoch : 40/200, recon loss = 0.00204494\n",
            "epoch : 41/200, recon loss = 0.00200802\n",
            "epoch : 42/200, recon loss = 0.00198360\n",
            "epoch : 43/200, recon loss = 0.00196986\n",
            "epoch : 44/200, recon loss = 0.00195587\n",
            "epoch : 45/200, recon loss = 0.00192038\n",
            "epoch : 46/200, recon loss = 0.00192065\n",
            "epoch : 47/200, recon loss = 0.00190671\n",
            "epoch : 48/200, recon loss = 0.00190018\n",
            "epoch : 49/200, recon loss = 0.00189126\n",
            "epoch : 50/200, recon loss = 0.00190825\n",
            "epoch : 51/200, recon loss = 0.00188646\n",
            "epoch : 52/200, recon loss = 0.00189605\n",
            "epoch : 53/200, recon loss = 0.00186747\n",
            "epoch : 54/200, recon loss = 0.00186126\n",
            "epoch : 55/200, recon loss = 0.00186056\n",
            "epoch : 56/200, recon loss = 0.00187486\n",
            "epoch : 57/200, recon loss = 0.00186098\n",
            "epoch : 58/200, recon loss = 0.00186109\n",
            "epoch : 59/200, recon loss = 0.00186338\n",
            "epoch : 60/200, recon loss = 0.00187644\n",
            "epoch : 61/200, recon loss = 0.00184205\n",
            "epoch : 62/200, recon loss = 0.00183937\n",
            "epoch : 63/200, recon loss = 0.00188108\n",
            "epoch : 64/200, recon loss = 0.00186292\n",
            "epoch : 65/200, recon loss = 0.00185807\n",
            "epoch : 66/200, recon loss = 0.00184938\n",
            "epoch : 67/200, recon loss = 0.00185676\n",
            "epoch : 68/200, recon loss = 0.00185299\n",
            "epoch : 69/200, recon loss = 0.00186105\n",
            "epoch : 70/200, recon loss = 0.00183166\n",
            "epoch : 71/200, recon loss = 0.00183572\n",
            "epoch : 72/200, recon loss = 0.00185786\n",
            "epoch : 73/200, recon loss = 0.00182483\n",
            "epoch : 74/200, recon loss = 0.00185675\n",
            "epoch : 75/200, recon loss = 0.00182751\n",
            "epoch : 76/200, recon loss = 0.00183547\n",
            "epoch : 77/200, recon loss = 0.00182906\n",
            "epoch : 78/200, recon loss = 0.00183675\n",
            "epoch : 79/200, recon loss = 0.00184419\n",
            "epoch : 80/200, recon loss = 0.00183510\n",
            "epoch : 81/200, recon loss = 0.00182104\n",
            "epoch : 82/200, recon loss = 0.00183690\n",
            "epoch : 83/200, recon loss = 0.00184765\n",
            "epoch : 84/200, recon loss = 0.00184642\n",
            "epoch : 85/200, recon loss = 0.00183876\n",
            "epoch : 86/200, recon loss = 0.00182842\n",
            "epoch : 87/200, recon loss = 0.00182940\n",
            "epoch : 88/200, recon loss = 0.00181363\n",
            "epoch : 89/200, recon loss = 0.00181099\n",
            "epoch : 90/200, recon loss = 0.00180973\n",
            "epoch : 91/200, recon loss = 0.00181110\n",
            "epoch : 92/200, recon loss = 0.00180957\n",
            "epoch : 93/200, recon loss = 0.00180544\n",
            "epoch : 94/200, recon loss = 0.00182497\n",
            "epoch : 95/200, recon loss = 0.00182177\n",
            "epoch : 96/200, recon loss = 0.00180528\n",
            "epoch : 97/200, recon loss = 0.00180315\n",
            "epoch : 98/200, recon loss = 0.00179617\n",
            "epoch : 99/200, recon loss = 0.00180536\n",
            "epoch : 100/200, recon loss = 0.00179808\n",
            "epoch : 101/200, recon loss = 0.00180109\n",
            "epoch : 102/200, recon loss = 0.00178796\n",
            "epoch : 103/200, recon loss = 0.00180230\n",
            "epoch : 104/200, recon loss = 0.00179792\n",
            "epoch : 105/200, recon loss = 0.00179125\n",
            "epoch : 106/200, recon loss = 0.00178091\n",
            "epoch : 107/200, recon loss = 0.00176455\n",
            "epoch : 108/200, recon loss = 0.00177532\n",
            "epoch : 109/200, recon loss = 0.00178327\n",
            "epoch : 110/200, recon loss = 0.00178397\n",
            "epoch : 111/200, recon loss = 0.00177148\n",
            "epoch : 112/200, recon loss = 0.00176829\n",
            "epoch : 113/200, recon loss = 0.00177024\n",
            "epoch : 114/200, recon loss = 0.00178309\n",
            "epoch : 115/200, recon loss = 0.00174765\n",
            "epoch : 116/200, recon loss = 0.00174921\n",
            "epoch : 117/200, recon loss = 0.00174532\n",
            "epoch : 118/200, recon loss = 0.00174384\n",
            "epoch : 119/200, recon loss = 0.00175785\n",
            "epoch : 120/200, recon loss = 0.00173572\n",
            "epoch : 121/200, recon loss = 0.00173396\n",
            "epoch : 122/200, recon loss = 0.00170617\n",
            "epoch : 123/200, recon loss = 0.00174318\n",
            "epoch : 124/200, recon loss = 0.00172062\n",
            "epoch : 125/200, recon loss = 0.00171285\n",
            "epoch : 126/200, recon loss = 0.00171034\n",
            "epoch : 127/200, recon loss = 0.00170215\n",
            "epoch : 128/200, recon loss = 0.00169096\n",
            "epoch : 129/200, recon loss = 0.00170776\n",
            "epoch : 130/200, recon loss = 0.00168135\n",
            "epoch : 131/200, recon loss = 0.00168559\n",
            "epoch : 132/200, recon loss = 0.00167319\n",
            "epoch : 133/200, recon loss = 0.00165881\n",
            "epoch : 134/200, recon loss = 0.00166068\n",
            "epoch : 135/200, recon loss = 0.00165581\n",
            "epoch : 136/200, recon loss = 0.00165603\n",
            "epoch : 137/200, recon loss = 0.00163922\n",
            "epoch : 138/200, recon loss = 0.00165844\n",
            "epoch : 139/200, recon loss = 0.00162769\n",
            "epoch : 140/200, recon loss = 0.00162963\n",
            "epoch : 141/200, recon loss = 0.00164324\n",
            "epoch : 142/200, recon loss = 0.00162952\n",
            "epoch : 143/200, recon loss = 0.00161351\n",
            "epoch : 144/200, recon loss = 0.00161767\n",
            "epoch : 145/200, recon loss = 0.00162506\n",
            "epoch : 146/200, recon loss = 0.00159639\n",
            "epoch : 147/200, recon loss = 0.00160419\n",
            "epoch : 148/200, recon loss = 0.00160436\n",
            "epoch : 149/200, recon loss = 0.00159325\n",
            "epoch : 150/200, recon loss = 0.00161209\n",
            "epoch : 151/200, recon loss = 0.00158283\n",
            "epoch : 152/200, recon loss = 0.00159162\n",
            "epoch : 153/200, recon loss = 0.00158156\n",
            "epoch : 154/200, recon loss = 0.00157226\n",
            "epoch : 155/200, recon loss = 0.00157494\n",
            "epoch : 156/200, recon loss = 0.00155901\n",
            "epoch : 157/200, recon loss = 0.00157940\n",
            "epoch : 158/200, recon loss = 0.00156612\n",
            "epoch : 159/200, recon loss = 0.00157606\n",
            "epoch : 160/200, recon loss = 0.00156186\n",
            "epoch : 161/200, recon loss = 0.00154729\n",
            "epoch : 162/200, recon loss = 0.00155769\n",
            "epoch : 163/200, recon loss = 0.00155262\n",
            "epoch : 164/200, recon loss = 0.00155334\n",
            "epoch : 165/200, recon loss = 0.00153121\n",
            "epoch : 166/200, recon loss = 0.00153181\n",
            "epoch : 167/200, recon loss = 0.00152951\n",
            "epoch : 168/200, recon loss = 0.00151840\n",
            "epoch : 169/200, recon loss = 0.00153604\n",
            "epoch : 170/200, recon loss = 0.00152209\n",
            "epoch : 171/200, recon loss = 0.00151762\n",
            "epoch : 172/200, recon loss = 0.00152200\n",
            "epoch : 173/200, recon loss = 0.00149509\n",
            "epoch : 174/200, recon loss = 0.00148770\n",
            "epoch : 175/200, recon loss = 0.00150177\n",
            "epoch : 176/200, recon loss = 0.00150579\n",
            "epoch : 177/200, recon loss = 0.00149195\n",
            "epoch : 178/200, recon loss = 0.00149757\n",
            "epoch : 179/200, recon loss = 0.00149421\n",
            "epoch : 180/200, recon loss = 0.00147834\n",
            "epoch : 181/200, recon loss = 0.00147764\n",
            "epoch : 182/200, recon loss = 0.00147020\n",
            "epoch : 183/200, recon loss = 0.00149420\n",
            "epoch : 184/200, recon loss = 0.00144807\n",
            "epoch : 185/200, recon loss = 0.00144082\n",
            "epoch : 186/200, recon loss = 0.00147132\n",
            "epoch : 187/200, recon loss = 0.00147896\n",
            "epoch : 188/200, recon loss = 0.00146926\n",
            "epoch : 189/200, recon loss = 0.00147516\n",
            "epoch : 190/200, recon loss = 0.00145522\n",
            "epoch : 191/200, recon loss = 0.00145488\n",
            "epoch : 192/200, recon loss = 0.00144129\n",
            "epoch : 193/200, recon loss = 0.00144756\n",
            "epoch : 194/200, recon loss = 0.00143312\n",
            "epoch : 195/200, recon loss = 0.00144265\n",
            "epoch : 196/200, recon loss = 0.00143325\n",
            "epoch : 197/200, recon loss = 0.00141967\n",
            "epoch : 198/200, recon loss = 0.00144865\n",
            "epoch : 199/200, recon loss = 0.00142578\n",
            "epoch : 200/200, recon loss = 0.00143751\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HWs070Pw7GWQ",
        "outputId": "e5eec14c-a0b4-4801-e64e-f5d83515c721"
      },
      "source": [
        "model.eval()\n",
        "rmse_sum = 0\n",
        "\n",
        "filled_data = model(missed_data_noEm.to(device))\n",
        "filled_data = filled_data.cpu().detach().numpy()\n",
        "\n",
        "rmse_sum = rmse_error(test_data, filled_data, cols-num_embeddings, mask)\n",
        "\n",
        "print(rmse_sum)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "3.1464000386280997\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0gGWYlp67GWR"
      },
      "source": [
        "## Autoencoder - with embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TIDyzg2R7GWR"
      },
      "source": [
        "model = Autoencoder(dim=cols).to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hdg36wfV7GWS"
      },
      "source": [
        "criterion = nn.MSELoss()\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KJRVBvPY7GWS",
        "outputId": "96157a40-6c7d-4fa6-83ae-5240f4a44203"
      },
      "source": [
        "training(num_epochs, model, train_loader, criterion, optimizer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch : 1/200, recon loss = 0.01493986\n",
            "epoch : 2/200, recon loss = 0.01184460\n",
            "epoch : 3/200, recon loss = 0.01175992\n",
            "epoch : 4/200, recon loss = 0.01161368\n",
            "epoch : 5/200, recon loss = 0.01129655\n",
            "epoch : 6/200, recon loss = 0.01056569\n",
            "epoch : 7/200, recon loss = 0.00956796\n",
            "epoch : 8/200, recon loss = 0.00903610\n",
            "epoch : 9/200, recon loss = 0.00883364\n",
            "epoch : 10/200, recon loss = 0.00874422\n",
            "epoch : 11/200, recon loss = 0.00869116\n",
            "epoch : 12/200, recon loss = 0.00863454\n",
            "epoch : 13/200, recon loss = 0.00858350\n",
            "epoch : 14/200, recon loss = 0.00852487\n",
            "epoch : 15/200, recon loss = 0.00844616\n",
            "epoch : 16/200, recon loss = 0.00835073\n",
            "epoch : 17/200, recon loss = 0.00825278\n",
            "epoch : 18/200, recon loss = 0.00815671\n",
            "epoch : 19/200, recon loss = 0.00804786\n",
            "epoch : 20/200, recon loss = 0.00797374\n",
            "epoch : 21/200, recon loss = 0.00788826\n",
            "epoch : 22/200, recon loss = 0.00784343\n",
            "epoch : 23/200, recon loss = 0.00779354\n",
            "epoch : 24/200, recon loss = 0.00773754\n",
            "epoch : 25/200, recon loss = 0.00769605\n",
            "epoch : 26/200, recon loss = 0.00765566\n",
            "epoch : 27/200, recon loss = 0.00760844\n",
            "epoch : 28/200, recon loss = 0.00753953\n",
            "epoch : 29/200, recon loss = 0.00747908\n",
            "epoch : 30/200, recon loss = 0.00743078\n",
            "epoch : 31/200, recon loss = 0.00735289\n",
            "epoch : 32/200, recon loss = 0.00730681\n",
            "epoch : 33/200, recon loss = 0.00724723\n",
            "epoch : 34/200, recon loss = 0.00716677\n",
            "epoch : 35/200, recon loss = 0.00713059\n",
            "epoch : 36/200, recon loss = 0.00705319\n",
            "epoch : 37/200, recon loss = 0.00697987\n",
            "epoch : 38/200, recon loss = 0.00691349\n",
            "epoch : 39/200, recon loss = 0.00690362\n",
            "epoch : 40/200, recon loss = 0.00682418\n",
            "epoch : 41/200, recon loss = 0.00680596\n",
            "epoch : 42/200, recon loss = 0.00675900\n",
            "epoch : 43/200, recon loss = 0.00670796\n",
            "epoch : 44/200, recon loss = 0.00665836\n",
            "epoch : 45/200, recon loss = 0.00665851\n",
            "epoch : 46/200, recon loss = 0.00661743\n",
            "epoch : 47/200, recon loss = 0.00660870\n",
            "epoch : 48/200, recon loss = 0.00656261\n",
            "epoch : 49/200, recon loss = 0.00655451\n",
            "epoch : 50/200, recon loss = 0.00650813\n",
            "epoch : 51/200, recon loss = 0.00652391\n",
            "epoch : 52/200, recon loss = 0.00646897\n",
            "epoch : 53/200, recon loss = 0.00644691\n",
            "epoch : 54/200, recon loss = 0.00645969\n",
            "epoch : 55/200, recon loss = 0.00642074\n",
            "epoch : 56/200, recon loss = 0.00638904\n",
            "epoch : 57/200, recon loss = 0.00639415\n",
            "epoch : 58/200, recon loss = 0.00637240\n",
            "epoch : 59/200, recon loss = 0.00635742\n",
            "epoch : 60/200, recon loss = 0.00633366\n",
            "epoch : 61/200, recon loss = 0.00632450\n",
            "epoch : 62/200, recon loss = 0.00628581\n",
            "epoch : 63/200, recon loss = 0.00627952\n",
            "epoch : 64/200, recon loss = 0.00627992\n",
            "epoch : 65/200, recon loss = 0.00628762\n",
            "epoch : 66/200, recon loss = 0.00625092\n",
            "epoch : 67/200, recon loss = 0.00622432\n",
            "epoch : 68/200, recon loss = 0.00620524\n",
            "epoch : 69/200, recon loss = 0.00618653\n",
            "epoch : 70/200, recon loss = 0.00619257\n",
            "epoch : 71/200, recon loss = 0.00615843\n",
            "epoch : 72/200, recon loss = 0.00612591\n",
            "epoch : 73/200, recon loss = 0.00615028\n",
            "epoch : 74/200, recon loss = 0.00611881\n",
            "epoch : 75/200, recon loss = 0.00613107\n",
            "epoch : 76/200, recon loss = 0.00613329\n",
            "epoch : 77/200, recon loss = 0.00608837\n",
            "epoch : 78/200, recon loss = 0.00609169\n",
            "epoch : 79/200, recon loss = 0.00608397\n",
            "epoch : 80/200, recon loss = 0.00603733\n",
            "epoch : 81/200, recon loss = 0.00604243\n",
            "epoch : 82/200, recon loss = 0.00602525\n",
            "epoch : 83/200, recon loss = 0.00601107\n",
            "epoch : 84/200, recon loss = 0.00599180\n",
            "epoch : 85/200, recon loss = 0.00599268\n",
            "epoch : 86/200, recon loss = 0.00597281\n",
            "epoch : 87/200, recon loss = 0.00597364\n",
            "epoch : 88/200, recon loss = 0.00595701\n",
            "epoch : 89/200, recon loss = 0.00595430\n",
            "epoch : 90/200, recon loss = 0.00593952\n",
            "epoch : 91/200, recon loss = 0.00592894\n",
            "epoch : 92/200, recon loss = 0.00593945\n",
            "epoch : 93/200, recon loss = 0.00592054\n",
            "epoch : 94/200, recon loss = 0.00590354\n",
            "epoch : 95/200, recon loss = 0.00587785\n",
            "epoch : 96/200, recon loss = 0.00589960\n",
            "epoch : 97/200, recon loss = 0.00586389\n",
            "epoch : 98/200, recon loss = 0.00586239\n",
            "epoch : 99/200, recon loss = 0.00586535\n",
            "epoch : 100/200, recon loss = 0.00580854\n",
            "epoch : 101/200, recon loss = 0.00579152\n",
            "epoch : 102/200, recon loss = 0.00576778\n",
            "epoch : 103/200, recon loss = 0.00580616\n",
            "epoch : 104/200, recon loss = 0.00578801\n",
            "epoch : 105/200, recon loss = 0.00577334\n",
            "epoch : 106/200, recon loss = 0.00576103\n",
            "epoch : 107/200, recon loss = 0.00574360\n",
            "epoch : 108/200, recon loss = 0.00574834\n",
            "epoch : 109/200, recon loss = 0.00575436\n",
            "epoch : 110/200, recon loss = 0.00571486\n",
            "epoch : 111/200, recon loss = 0.00572030\n",
            "epoch : 112/200, recon loss = 0.00569421\n",
            "epoch : 113/200, recon loss = 0.00569184\n",
            "epoch : 114/200, recon loss = 0.00573562\n",
            "epoch : 115/200, recon loss = 0.00570613\n",
            "epoch : 116/200, recon loss = 0.00567897\n",
            "epoch : 117/200, recon loss = 0.00566256\n",
            "epoch : 118/200, recon loss = 0.00566671\n",
            "epoch : 119/200, recon loss = 0.00565211\n",
            "epoch : 120/200, recon loss = 0.00569357\n",
            "epoch : 121/200, recon loss = 0.00562647\n",
            "epoch : 122/200, recon loss = 0.00564916\n",
            "epoch : 123/200, recon loss = 0.00558050\n",
            "epoch : 124/200, recon loss = 0.00561506\n",
            "epoch : 125/200, recon loss = 0.00564308\n",
            "epoch : 126/200, recon loss = 0.00556880\n",
            "epoch : 127/200, recon loss = 0.00557498\n",
            "epoch : 128/200, recon loss = 0.00557876\n",
            "epoch : 129/200, recon loss = 0.00559238\n",
            "epoch : 130/200, recon loss = 0.00558861\n",
            "epoch : 131/200, recon loss = 0.00557130\n",
            "epoch : 132/200, recon loss = 0.00557059\n",
            "epoch : 133/200, recon loss = 0.00556697\n",
            "epoch : 134/200, recon loss = 0.00559344\n",
            "epoch : 135/200, recon loss = 0.00557196\n",
            "epoch : 136/200, recon loss = 0.00552676\n",
            "epoch : 137/200, recon loss = 0.00550982\n",
            "epoch : 138/200, recon loss = 0.00552366\n",
            "epoch : 139/200, recon loss = 0.00552771\n",
            "epoch : 140/200, recon loss = 0.00548069\n",
            "epoch : 141/200, recon loss = 0.00551362\n",
            "epoch : 142/200, recon loss = 0.00551510\n",
            "epoch : 143/200, recon loss = 0.00549275\n",
            "epoch : 144/200, recon loss = 0.00547726\n",
            "epoch : 145/200, recon loss = 0.00547176\n",
            "epoch : 146/200, recon loss = 0.00549751\n",
            "epoch : 147/200, recon loss = 0.00547309\n",
            "epoch : 148/200, recon loss = 0.00549410\n",
            "epoch : 149/200, recon loss = 0.00549005\n",
            "epoch : 150/200, recon loss = 0.00545908\n",
            "epoch : 151/200, recon loss = 0.00549218\n",
            "epoch : 152/200, recon loss = 0.00544357\n",
            "epoch : 153/200, recon loss = 0.00542468\n",
            "epoch : 154/200, recon loss = 0.00542159\n",
            "epoch : 155/200, recon loss = 0.00542137\n",
            "epoch : 156/200, recon loss = 0.00544141\n",
            "epoch : 157/200, recon loss = 0.00544190\n",
            "epoch : 158/200, recon loss = 0.00542098\n",
            "epoch : 159/200, recon loss = 0.00541473\n",
            "epoch : 160/200, recon loss = 0.00540144\n",
            "epoch : 161/200, recon loss = 0.00538232\n",
            "epoch : 162/200, recon loss = 0.00540156\n",
            "epoch : 163/200, recon loss = 0.00541582\n",
            "epoch : 164/200, recon loss = 0.00537936\n",
            "epoch : 165/200, recon loss = 0.00537611\n",
            "epoch : 166/200, recon loss = 0.00538170\n",
            "epoch : 167/200, recon loss = 0.00536953\n",
            "epoch : 168/200, recon loss = 0.00540306\n",
            "epoch : 169/200, recon loss = 0.00538371\n",
            "epoch : 170/200, recon loss = 0.00537456\n",
            "epoch : 171/200, recon loss = 0.00535500\n",
            "epoch : 172/200, recon loss = 0.00535657\n",
            "epoch : 173/200, recon loss = 0.00533241\n",
            "epoch : 174/200, recon loss = 0.00533198\n",
            "epoch : 175/200, recon loss = 0.00532854\n",
            "epoch : 176/200, recon loss = 0.00531316\n",
            "epoch : 177/200, recon loss = 0.00536189\n",
            "epoch : 178/200, recon loss = 0.00531386\n",
            "epoch : 179/200, recon loss = 0.00530701\n",
            "epoch : 180/200, recon loss = 0.00529350\n",
            "epoch : 181/200, recon loss = 0.00530522\n",
            "epoch : 182/200, recon loss = 0.00530638\n",
            "epoch : 183/200, recon loss = 0.00532572\n",
            "epoch : 184/200, recon loss = 0.00531917\n",
            "epoch : 185/200, recon loss = 0.00527608\n",
            "epoch : 186/200, recon loss = 0.00530887\n",
            "epoch : 187/200, recon loss = 0.00529899\n",
            "epoch : 188/200, recon loss = 0.00527678\n",
            "epoch : 189/200, recon loss = 0.00528508\n",
            "epoch : 190/200, recon loss = 0.00527911\n",
            "epoch : 191/200, recon loss = 0.00526465\n",
            "epoch : 192/200, recon loss = 0.00527176\n",
            "epoch : 193/200, recon loss = 0.00527497\n",
            "epoch : 194/200, recon loss = 0.00524657\n",
            "epoch : 195/200, recon loss = 0.00522595\n",
            "epoch : 196/200, recon loss = 0.00525761\n",
            "epoch : 197/200, recon loss = 0.00523570\n",
            "epoch : 198/200, recon loss = 0.00520434\n",
            "epoch : 199/200, recon loss = 0.00523674\n",
            "epoch : 200/200, recon loss = 0.00522757\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NxmY61KZ7GWT",
        "outputId": "d57d65e2-93d5-4281-eed0-4e2a8d55ae86"
      },
      "source": [
        "model.eval()\n",
        "rmse_sum = 0\n",
        "\n",
        "filled_data = model(missed_data.to(device))\n",
        "filled_data = filled_data.cpu().detach().numpy()\n",
        "\n",
        "rmse_sum = rmse_error(test_data, filled_data, cols, mask)\n",
        "\n",
        "print(rmse_sum)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "3.1930650168502\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OLDXzazW7GWT"
      },
      "source": [
        "## Mean and Median Imputation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UayXjiTM7GWU"
      },
      "source": [
        "### By category"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nmWEoNUW7GWU",
        "outputId": "66c5c799-4b51-4e5a-826d-c2565df6a2c5"
      },
      "source": [
        "imputed = X_test_mean\n",
        "error = rmse_error(y_test, imputed, cols-num_embeddings, mask)\n",
        "print(error)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "4.240093997588478\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wJt_az0w7GWV",
        "outputId": "32daf404-d64c-4893-ac95-d4f7248b02e7"
      },
      "source": [
        "imputed = X_test_median\n",
        "error = rmse_error(y_test, imputed, cols-num_embeddings, mask)\n",
        "print(error)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "4.456391764474138\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qq0lUGgE7GWV"
      },
      "source": [
        "### Across all foods"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jhm_9MjL7GWW",
        "outputId": "0fdf6055-e2ac-4159-bbc0-f7e741a9646d"
      },
      "source": [
        "imputed = X_test_mean_all\n",
        "error = rmse_error(y_test, imputed, cols-num_embeddings, mask)\n",
        "print(error)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "5.005264371707832\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CZuYKdMN7GWW",
        "outputId": "20046967-134c-4a04-a8fa-3e8a7fadf5e5"
      },
      "source": [
        "imputed = X_test_median_all\n",
        "error = rmse_error(y_test, imputed, cols-num_embeddings, mask)\n",
        "print(error)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "5.1325066538984725\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OntKaWBd7GWW"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j-EImi317Q0x"
      },
      "source": [
        "# 120 Columns"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IqA3Ql367Q1L"
      },
      "source": [
        "data_path = 'datasets/average datasets/byColumnsWithEmbeddings/120_columns_embeddings.csv'\n",
        "num_embeddings = 30\n",
        "num_columns = 120"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 350
        },
        "id": "jpKR8bt-7Q1M",
        "outputId": "4f93c55f-11d9-40df-8b4c-1769ddc09c40"
      },
      "source": [
        "dataset = pd.read_csv(data_path, index_col=0)\n",
        "dataset[dataset.notna().all(axis=1)].describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>fdc_id</th>\n",
              "      <th>1004</th>\n",
              "      <th>1087</th>\n",
              "      <th>1090</th>\n",
              "      <th>1092</th>\n",
              "      <th>1095</th>\n",
              "      <th>1091</th>\n",
              "      <th>1089</th>\n",
              "      <th>1098</th>\n",
              "      <th>1101</th>\n",
              "      <th>1051</th>\n",
              "      <th>1007</th>\n",
              "      <th>1002</th>\n",
              "      <th>1093</th>\n",
              "      <th>1079</th>\n",
              "      <th>1003</th>\n",
              "      <th>1009</th>\n",
              "      <th>1102</th>\n",
              "      <th>1265</th>\n",
              "      <th>1266</th>\n",
              "      <th>1264</th>\n",
              "      <th>1315</th>\n",
              "      <th>1316</th>\n",
              "      <th>1314</th>\n",
              "      <th>1094</th>\n",
              "      <th>1097</th>\n",
              "      <th>1137</th>\n",
              "      <th>1146</th>\n",
              "      <th>1300</th>\n",
              "      <th>1299</th>\n",
              "      <th>1404</th>\n",
              "      <th>1267</th>\n",
              "      <th>1273</th>\n",
              "      <th>1323</th>\n",
              "      <th>1167</th>\n",
              "      <th>1271</th>\n",
              "      <th>1304</th>\n",
              "      <th>1166</th>\n",
              "      <th>1175</th>\n",
              "      <th>1263</th>\n",
              "      <th>...</th>\n",
              "      <th>1180</th>\n",
              "      <th>1194</th>\n",
              "      <th>1128</th>\n",
              "      <th>1195</th>\n",
              "      <th>1005</th>\n",
              "      <th>1075</th>\n",
              "      <th>1196</th>\n",
              "      <th>1197</th>\n",
              "      <th>1198</th>\n",
              "      <th>1084</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>1.310500e+04</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.00000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>4.807517e+05</td>\n",
              "      <td>1.824697</td>\n",
              "      <td>27.659061</td>\n",
              "      <td>14.595597</td>\n",
              "      <td>125.003892</td>\n",
              "      <td>0.384163</td>\n",
              "      <td>55.746662</td>\n",
              "      <td>0.476038</td>\n",
              "      <td>0.077504</td>\n",
              "      <td>0.149979</td>\n",
              "      <td>8.349769</td>\n",
              "      <td>0.472742</td>\n",
              "      <td>0.243665</td>\n",
              "      <td>133.394887</td>\n",
              "      <td>0.350355</td>\n",
              "      <td>1.942243</td>\n",
              "      <td>2.561062</td>\n",
              "      <td>42.944411</td>\n",
              "      <td>0.308332</td>\n",
              "      <td>0.116491</td>\n",
              "      <td>0.065003</td>\n",
              "      <td>0.605730</td>\n",
              "      <td>0.211003</td>\n",
              "      <td>0.023788</td>\n",
              "      <td>14.730179</td>\n",
              "      <td>2.983579</td>\n",
              "      <td>78.72934</td>\n",
              "      <td>12.225029</td>\n",
              "      <td>0.005267</td>\n",
              "      <td>0.005886</td>\n",
              "      <td>0.016719</td>\n",
              "      <td>0.004499</td>\n",
              "      <td>0.003870</td>\n",
              "      <td>0.002480</td>\n",
              "      <td>0.118839</td>\n",
              "      <td>0.002989</td>\n",
              "      <td>0.015767</td>\n",
              "      <td>0.012258</td>\n",
              "      <td>0.008632</td>\n",
              "      <td>0.038133</td>\n",
              "      <td>...</td>\n",
              "      <td>0.928287</td>\n",
              "      <td>0.039206</td>\n",
              "      <td>0.002494</td>\n",
              "      <td>0.726326</td>\n",
              "      <td>0.164999</td>\n",
              "      <td>0.002459</td>\n",
              "      <td>0.045410</td>\n",
              "      <td>0.042976</td>\n",
              "      <td>0.046929</td>\n",
              "      <td>0.021892</td>\n",
              "      <td>21488.189968</td>\n",
              "      <td>176.402583</td>\n",
              "      <td>0.119704</td>\n",
              "      <td>-0.093912</td>\n",
              "      <td>-0.048494</td>\n",
              "      <td>-0.007831</td>\n",
              "      <td>0.051473</td>\n",
              "      <td>0.023113</td>\n",
              "      <td>0.063163</td>\n",
              "      <td>0.043670</td>\n",
              "      <td>-0.005261</td>\n",
              "      <td>-0.053342</td>\n",
              "      <td>0.032862</td>\n",
              "      <td>0.017148</td>\n",
              "      <td>0.053914</td>\n",
              "      <td>-0.050016</td>\n",
              "      <td>-0.006804</td>\n",
              "      <td>-0.001309</td>\n",
              "      <td>0.045397</td>\n",
              "      <td>-0.002817</td>\n",
              "      <td>-0.004870</td>\n",
              "      <td>0.015158</td>\n",
              "      <td>-0.002832</td>\n",
              "      <td>-0.008018</td>\n",
              "      <td>0.007639</td>\n",
              "      <td>-0.005292</td>\n",
              "      <td>0.013278</td>\n",
              "      <td>0.026410</td>\n",
              "      <td>-0.008193</td>\n",
              "      <td>-0.003744</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.544018e+05</td>\n",
              "      <td>7.838466</td>\n",
              "      <td>99.924214</td>\n",
              "      <td>45.754175</td>\n",
              "      <td>382.322293</td>\n",
              "      <td>1.055340</td>\n",
              "      <td>153.432660</td>\n",
              "      <td>1.483360</td>\n",
              "      <td>0.264892</td>\n",
              "      <td>0.515558</td>\n",
              "      <td>23.549114</td>\n",
              "      <td>4.416751</td>\n",
              "      <td>0.999601</td>\n",
              "      <td>1736.846193</td>\n",
              "      <td>1.186155</td>\n",
              "      <td>6.583455</td>\n",
              "      <td>9.446549</td>\n",
              "      <td>187.200801</td>\n",
              "      <td>1.605577</td>\n",
              "      <td>0.597369</td>\n",
              "      <td>0.559044</td>\n",
              "      <td>4.440086</td>\n",
              "      <td>1.941342</td>\n",
              "      <td>0.148635</td>\n",
              "      <td>56.459805</td>\n",
              "      <td>11.848737</td>\n",
              "      <td>302.68343</td>\n",
              "      <td>49.040700</td>\n",
              "      <td>0.030796</td>\n",
              "      <td>0.043100</td>\n",
              "      <td>0.223785</td>\n",
              "      <td>0.039142</td>\n",
              "      <td>0.065227</td>\n",
              "      <td>0.015745</td>\n",
              "      <td>0.828604</td>\n",
              "      <td>0.017050</td>\n",
              "      <td>0.106190</td>\n",
              "      <td>0.071481</td>\n",
              "      <td>0.054902</td>\n",
              "      <td>0.974266</td>\n",
              "      <td>...</td>\n",
              "      <td>15.719211</td>\n",
              "      <td>0.824407</td>\n",
              "      <td>0.055645</td>\n",
              "      <td>14.277867</td>\n",
              "      <td>2.905073</td>\n",
              "      <td>0.034625</td>\n",
              "      <td>1.333663</td>\n",
              "      <td>0.624933</td>\n",
              "      <td>1.671006</td>\n",
              "      <td>0.319893</td>\n",
              "      <td>254447.573253</td>\n",
              "      <td>3958.010969</td>\n",
              "      <td>1.112342</td>\n",
              "      <td>0.694392</td>\n",
              "      <td>0.598293</td>\n",
              "      <td>0.511537</td>\n",
              "      <td>0.453723</td>\n",
              "      <td>0.451007</td>\n",
              "      <td>0.417454</td>\n",
              "      <td>0.426072</td>\n",
              "      <td>0.400175</td>\n",
              "      <td>0.345475</td>\n",
              "      <td>0.349559</td>\n",
              "      <td>0.332055</td>\n",
              "      <td>0.321282</td>\n",
              "      <td>0.292925</td>\n",
              "      <td>0.270047</td>\n",
              "      <td>0.284938</td>\n",
              "      <td>0.261697</td>\n",
              "      <td>0.269206</td>\n",
              "      <td>0.240664</td>\n",
              "      <td>0.243863</td>\n",
              "      <td>0.245151</td>\n",
              "      <td>0.247991</td>\n",
              "      <td>0.235566</td>\n",
              "      <td>0.205593</td>\n",
              "      <td>0.224944</td>\n",
              "      <td>0.212094</td>\n",
              "      <td>0.209662</td>\n",
              "      <td>0.206100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>3.198770e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-139555.939272</td>\n",
              "      <td>-7413.329674</td>\n",
              "      <td>-3.349638</td>\n",
              "      <td>-1.585647</td>\n",
              "      <td>-1.527263</td>\n",
              "      <td>-1.442526</td>\n",
              "      <td>-1.189856</td>\n",
              "      <td>-1.064453</td>\n",
              "      <td>-1.077035</td>\n",
              "      <td>-0.829626</td>\n",
              "      <td>-0.933856</td>\n",
              "      <td>-0.969972</td>\n",
              "      <td>-1.097135</td>\n",
              "      <td>-0.920029</td>\n",
              "      <td>-1.006142</td>\n",
              "      <td>-1.000203</td>\n",
              "      <td>-0.798836</td>\n",
              "      <td>-0.687075</td>\n",
              "      <td>-0.668804</td>\n",
              "      <td>-0.801703</td>\n",
              "      <td>-0.726147</td>\n",
              "      <td>-0.698734</td>\n",
              "      <td>-0.733370</td>\n",
              "      <td>-0.848113</td>\n",
              "      <td>-0.673804</td>\n",
              "      <td>-0.609317</td>\n",
              "      <td>-0.622568</td>\n",
              "      <td>-0.618201</td>\n",
              "      <td>-0.605566</td>\n",
              "      <td>-0.802281</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>3.256480e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-133692.806088</td>\n",
              "      <td>-2328.507401</td>\n",
              "      <td>-0.796369</td>\n",
              "      <td>-0.667107</td>\n",
              "      <td>-0.477073</td>\n",
              "      <td>-0.343775</td>\n",
              "      <td>-0.236112</td>\n",
              "      <td>-0.286794</td>\n",
              "      <td>-0.177179</td>\n",
              "      <td>-0.292722</td>\n",
              "      <td>-0.303209</td>\n",
              "      <td>-0.332181</td>\n",
              "      <td>-0.196565</td>\n",
              "      <td>-0.219471</td>\n",
              "      <td>-0.160211</td>\n",
              "      <td>-0.217184</td>\n",
              "      <td>-0.194592</td>\n",
              "      <td>-0.175290</td>\n",
              "      <td>-0.131532</td>\n",
              "      <td>-0.159000</td>\n",
              "      <td>-0.129686</td>\n",
              "      <td>-0.142597</td>\n",
              "      <td>-0.128993</td>\n",
              "      <td>-0.177006</td>\n",
              "      <td>-0.164276</td>\n",
              "      <td>-0.149660</td>\n",
              "      <td>-0.136069</td>\n",
              "      <td>-0.112281</td>\n",
              "      <td>-0.131025</td>\n",
              "      <td>-0.103372</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>3.314100e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-127823.125712</td>\n",
              "      <td>-153.274226</td>\n",
              "      <td>-0.002647</td>\n",
              "      <td>-0.011945</td>\n",
              "      <td>-0.071964</td>\n",
              "      <td>-0.022449</td>\n",
              "      <td>0.012034</td>\n",
              "      <td>0.040864</td>\n",
              "      <td>0.040584</td>\n",
              "      <td>-0.032776</td>\n",
              "      <td>-0.006464</td>\n",
              "      <td>0.045016</td>\n",
              "      <td>0.064438</td>\n",
              "      <td>-0.007061</td>\n",
              "      <td>0.051129</td>\n",
              "      <td>-0.054894</td>\n",
              "      <td>-0.020014</td>\n",
              "      <td>-0.030517</td>\n",
              "      <td>0.015432</td>\n",
              "      <td>-0.034379</td>\n",
              "      <td>-0.022279</td>\n",
              "      <td>0.000418</td>\n",
              "      <td>-0.023101</td>\n",
              "      <td>0.012883</td>\n",
              "      <td>-0.022581</td>\n",
              "      <td>-0.015932</td>\n",
              "      <td>0.015818</td>\n",
              "      <td>0.012556</td>\n",
              "      <td>-0.039363</td>\n",
              "      <td>0.003180</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>7.474680e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>288262.097477</td>\n",
              "      <td>3032.870912</td>\n",
              "      <td>0.983044</td>\n",
              "      <td>0.441140</td>\n",
              "      <td>0.323271</td>\n",
              "      <td>0.289041</td>\n",
              "      <td>0.257716</td>\n",
              "      <td>0.227569</td>\n",
              "      <td>0.309666</td>\n",
              "      <td>0.274538</td>\n",
              "      <td>0.267124</td>\n",
              "      <td>0.166322</td>\n",
              "      <td>0.247990</td>\n",
              "      <td>0.188549</td>\n",
              "      <td>0.251011</td>\n",
              "      <td>0.109037</td>\n",
              "      <td>0.190487</td>\n",
              "      <td>0.178190</td>\n",
              "      <td>0.198249</td>\n",
              "      <td>0.177465</td>\n",
              "      <td>0.098791</td>\n",
              "      <td>0.180564</td>\n",
              "      <td>0.128409</td>\n",
              "      <td>0.157268</td>\n",
              "      <td>0.136539</td>\n",
              "      <td>0.119725</td>\n",
              "      <td>0.154657</td>\n",
              "      <td>0.164012</td>\n",
              "      <td>0.105063</td>\n",
              "      <td>0.111023</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.105897e+06</td>\n",
              "      <td>99.400000</td>\n",
              "      <td>1390.000000</td>\n",
              "      <td>386.000000</td>\n",
              "      <td>2520.000000</td>\n",
              "      <td>7.660000</td>\n",
              "      <td>997.000000</td>\n",
              "      <td>12.700000</td>\n",
              "      <td>1.920000</td>\n",
              "      <td>7.870000</td>\n",
              "      <td>96.700000</td>\n",
              "      <td>99.600000</td>\n",
              "      <td>13.200000</td>\n",
              "      <td>40700.000000</td>\n",
              "      <td>11.500000</td>\n",
              "      <td>79.900000</td>\n",
              "      <td>43.900000</td>\n",
              "      <td>1790.000000</td>\n",
              "      <td>22.800000</td>\n",
              "      <td>8.610000</td>\n",
              "      <td>17.000000</td>\n",
              "      <td>73.200000</td>\n",
              "      <td>52.600000</td>\n",
              "      <td>3.400000</td>\n",
              "      <td>311.000000</td>\n",
              "      <td>93.000000</td>\n",
              "      <td>1710.00000</td>\n",
              "      <td>449.000000</td>\n",
              "      <td>0.485000</td>\n",
              "      <td>0.850000</td>\n",
              "      <td>7.950000</td>\n",
              "      <td>0.774000</td>\n",
              "      <td>1.830000</td>\n",
              "      <td>0.303000</td>\n",
              "      <td>14.000000</td>\n",
              "      <td>0.222000</td>\n",
              "      <td>1.930000</td>\n",
              "      <td>1.860000</td>\n",
              "      <td>0.989000</td>\n",
              "      <td>42.400000</td>\n",
              "      <td>...</td>\n",
              "      <td>404.000000</td>\n",
              "      <td>42.700000</td>\n",
              "      <td>2.510000</td>\n",
              "      <td>376.000000</td>\n",
              "      <td>99.600000</td>\n",
              "      <td>1.100000</td>\n",
              "      <td>65.500000</td>\n",
              "      <td>13.100000</td>\n",
              "      <td>126.000000</td>\n",
              "      <td>9.200000</td>\n",
              "      <td>646711.616853</td>\n",
              "      <td>7635.496843</td>\n",
              "      <td>2.647689</td>\n",
              "      <td>1.869347</td>\n",
              "      <td>1.616849</td>\n",
              "      <td>2.041568</td>\n",
              "      <td>1.700616</td>\n",
              "      <td>1.764945</td>\n",
              "      <td>1.547788</td>\n",
              "      <td>1.348236</td>\n",
              "      <td>1.122911</td>\n",
              "      <td>1.235694</td>\n",
              "      <td>1.230920</td>\n",
              "      <td>1.301510</td>\n",
              "      <td>1.225628</td>\n",
              "      <td>1.031464</td>\n",
              "      <td>0.770631</td>\n",
              "      <td>0.843875</td>\n",
              "      <td>1.049396</td>\n",
              "      <td>0.867503</td>\n",
              "      <td>1.478596</td>\n",
              "      <td>0.921429</td>\n",
              "      <td>0.756656</td>\n",
              "      <td>0.908209</td>\n",
              "      <td>0.921199</td>\n",
              "      <td>0.965138</td>\n",
              "      <td>0.774711</td>\n",
              "      <td>0.770132</td>\n",
              "      <td>0.968808</td>\n",
              "      <td>0.881655</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8 rows × 151 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "             fdc_id          1004  ...            28            29\n",
              "count  1.310500e+04  13105.000000  ...  13105.000000  13105.000000\n",
              "mean   4.807517e+05      1.824697  ...     -0.008193     -0.003744\n",
              "std    2.544018e+05      7.838466  ...      0.209662      0.206100\n",
              "min    3.198770e+05      0.000000  ...     -0.605566     -0.802281\n",
              "25%    3.256480e+05      0.000000  ...     -0.131025     -0.103372\n",
              "50%    3.314100e+05      0.000000  ...     -0.039363      0.003180\n",
              "75%    7.474680e+05      0.000000  ...      0.105063      0.111023\n",
              "max    1.105897e+06     99.400000  ...      0.968808      0.881655\n",
              "\n",
              "[8 rows x 151 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JeyX3krD7Q1N"
      },
      "source": [
        "data = dataset\n",
        "train_data, test_data = train_test_split(data, test_size=test_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 350
        },
        "id": "2LkT1Lfv7Q1N",
        "outputId": "bea6a8d6-e68b-46ff-84bb-8ab1a0794c5f"
      },
      "source": [
        "train_data.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>fdc_id</th>\n",
              "      <th>1004</th>\n",
              "      <th>1087</th>\n",
              "      <th>1090</th>\n",
              "      <th>1092</th>\n",
              "      <th>1095</th>\n",
              "      <th>1091</th>\n",
              "      <th>1089</th>\n",
              "      <th>1098</th>\n",
              "      <th>1101</th>\n",
              "      <th>1051</th>\n",
              "      <th>1007</th>\n",
              "      <th>1002</th>\n",
              "      <th>1093</th>\n",
              "      <th>1079</th>\n",
              "      <th>1003</th>\n",
              "      <th>1009</th>\n",
              "      <th>1102</th>\n",
              "      <th>1265</th>\n",
              "      <th>1266</th>\n",
              "      <th>1264</th>\n",
              "      <th>1315</th>\n",
              "      <th>1316</th>\n",
              "      <th>1314</th>\n",
              "      <th>1094</th>\n",
              "      <th>1097</th>\n",
              "      <th>1137</th>\n",
              "      <th>1146</th>\n",
              "      <th>1300</th>\n",
              "      <th>1299</th>\n",
              "      <th>1404</th>\n",
              "      <th>1267</th>\n",
              "      <th>1273</th>\n",
              "      <th>1323</th>\n",
              "      <th>1167</th>\n",
              "      <th>1271</th>\n",
              "      <th>1304</th>\n",
              "      <th>1166</th>\n",
              "      <th>1175</th>\n",
              "      <th>1263</th>\n",
              "      <th>...</th>\n",
              "      <th>1180</th>\n",
              "      <th>1194</th>\n",
              "      <th>1128</th>\n",
              "      <th>1195</th>\n",
              "      <th>1005</th>\n",
              "      <th>1075</th>\n",
              "      <th>1196</th>\n",
              "      <th>1197</th>\n",
              "      <th>1198</th>\n",
              "      <th>1084</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>1.048400e+04</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>4.807307e+05</td>\n",
              "      <td>1.838933</td>\n",
              "      <td>27.191816</td>\n",
              "      <td>14.418333</td>\n",
              "      <td>124.497043</td>\n",
              "      <td>0.377420</td>\n",
              "      <td>55.049886</td>\n",
              "      <td>0.472891</td>\n",
              "      <td>0.076689</td>\n",
              "      <td>0.147316</td>\n",
              "      <td>8.377778</td>\n",
              "      <td>0.465995</td>\n",
              "      <td>0.247112</td>\n",
              "      <td>126.185712</td>\n",
              "      <td>0.347301</td>\n",
              "      <td>1.912456</td>\n",
              "      <td>2.568743</td>\n",
              "      <td>42.085340</td>\n",
              "      <td>0.315506</td>\n",
              "      <td>0.119174</td>\n",
              "      <td>0.067715</td>\n",
              "      <td>0.623412</td>\n",
              "      <td>0.223398</td>\n",
              "      <td>0.024106</td>\n",
              "      <td>14.656143</td>\n",
              "      <td>2.980733</td>\n",
              "      <td>78.750095</td>\n",
              "      <td>12.213277</td>\n",
              "      <td>0.005360</td>\n",
              "      <td>0.005941</td>\n",
              "      <td>0.017104</td>\n",
              "      <td>0.004821</td>\n",
              "      <td>0.004366</td>\n",
              "      <td>0.002521</td>\n",
              "      <td>0.119267</td>\n",
              "      <td>0.003027</td>\n",
              "      <td>0.015955</td>\n",
              "      <td>0.011941</td>\n",
              "      <td>0.008555</td>\n",
              "      <td>0.043831</td>\n",
              "      <td>...</td>\n",
              "      <td>0.865958</td>\n",
              "      <td>0.035702</td>\n",
              "      <td>0.002479</td>\n",
              "      <td>0.663859</td>\n",
              "      <td>0.168130</td>\n",
              "      <td>0.002428</td>\n",
              "      <td>0.054016</td>\n",
              "      <td>0.043018</td>\n",
              "      <td>0.052289</td>\n",
              "      <td>0.022568</td>\n",
              "      <td>21467.008441</td>\n",
              "      <td>182.062106</td>\n",
              "      <td>0.119423</td>\n",
              "      <td>-0.095347</td>\n",
              "      <td>-0.048735</td>\n",
              "      <td>-0.007134</td>\n",
              "      <td>0.054944</td>\n",
              "      <td>0.023179</td>\n",
              "      <td>0.063675</td>\n",
              "      <td>0.043259</td>\n",
              "      <td>-0.004838</td>\n",
              "      <td>-0.051300</td>\n",
              "      <td>0.030691</td>\n",
              "      <td>0.017925</td>\n",
              "      <td>0.052880</td>\n",
              "      <td>-0.050042</td>\n",
              "      <td>-0.004916</td>\n",
              "      <td>-0.001487</td>\n",
              "      <td>0.045954</td>\n",
              "      <td>-0.004037</td>\n",
              "      <td>-0.004376</td>\n",
              "      <td>0.015646</td>\n",
              "      <td>-0.001045</td>\n",
              "      <td>-0.007970</td>\n",
              "      <td>0.006989</td>\n",
              "      <td>-0.004153</td>\n",
              "      <td>0.012878</td>\n",
              "      <td>0.027332</td>\n",
              "      <td>-0.007871</td>\n",
              "      <td>-0.002815</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.543947e+05</td>\n",
              "      <td>7.873509</td>\n",
              "      <td>98.818185</td>\n",
              "      <td>45.429656</td>\n",
              "      <td>382.049756</td>\n",
              "      <td>1.047533</td>\n",
              "      <td>152.629159</td>\n",
              "      <td>1.481276</td>\n",
              "      <td>0.263418</td>\n",
              "      <td>0.507523</td>\n",
              "      <td>23.608917</td>\n",
              "      <td>4.309402</td>\n",
              "      <td>1.010590</td>\n",
              "      <td>1662.616839</td>\n",
              "      <td>1.178584</td>\n",
              "      <td>6.533157</td>\n",
              "      <td>9.457161</td>\n",
              "      <td>184.404872</td>\n",
              "      <td>1.624915</td>\n",
              "      <td>0.606916</td>\n",
              "      <td>0.593610</td>\n",
              "      <td>4.496240</td>\n",
              "      <td>2.050528</td>\n",
              "      <td>0.151031</td>\n",
              "      <td>56.189047</td>\n",
              "      <td>11.840536</td>\n",
              "      <td>302.770335</td>\n",
              "      <td>49.054157</td>\n",
              "      <td>0.031136</td>\n",
              "      <td>0.043461</td>\n",
              "      <td>0.226249</td>\n",
              "      <td>0.041357</td>\n",
              "      <td>0.070544</td>\n",
              "      <td>0.015894</td>\n",
              "      <td>0.836070</td>\n",
              "      <td>0.017135</td>\n",
              "      <td>0.106890</td>\n",
              "      <td>0.071769</td>\n",
              "      <td>0.054803</td>\n",
              "      <td>1.087551</td>\n",
              "      <td>...</td>\n",
              "      <td>15.102529</td>\n",
              "      <td>0.693033</td>\n",
              "      <td>0.056443</td>\n",
              "      <td>13.714635</td>\n",
              "      <td>2.924708</td>\n",
              "      <td>0.035324</td>\n",
              "      <td>1.488693</td>\n",
              "      <td>0.619556</td>\n",
              "      <td>1.854747</td>\n",
              "      <td>0.329757</td>\n",
              "      <td>254440.586435</td>\n",
              "      <td>3952.715216</td>\n",
              "      <td>1.109124</td>\n",
              "      <td>0.693789</td>\n",
              "      <td>0.599498</td>\n",
              "      <td>0.508734</td>\n",
              "      <td>0.453486</td>\n",
              "      <td>0.450050</td>\n",
              "      <td>0.417313</td>\n",
              "      <td>0.425748</td>\n",
              "      <td>0.400349</td>\n",
              "      <td>0.344332</td>\n",
              "      <td>0.351216</td>\n",
              "      <td>0.333003</td>\n",
              "      <td>0.321272</td>\n",
              "      <td>0.293894</td>\n",
              "      <td>0.270204</td>\n",
              "      <td>0.283863</td>\n",
              "      <td>0.263103</td>\n",
              "      <td>0.268481</td>\n",
              "      <td>0.241821</td>\n",
              "      <td>0.243146</td>\n",
              "      <td>0.245953</td>\n",
              "      <td>0.249616</td>\n",
              "      <td>0.235283</td>\n",
              "      <td>0.205396</td>\n",
              "      <td>0.225057</td>\n",
              "      <td>0.212131</td>\n",
              "      <td>0.210375</td>\n",
              "      <td>0.207533</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>3.198770e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-139555.939272</td>\n",
              "      <td>-7413.329674</td>\n",
              "      <td>-3.349638</td>\n",
              "      <td>-1.585647</td>\n",
              "      <td>-1.482007</td>\n",
              "      <td>-1.442526</td>\n",
              "      <td>-1.189856</td>\n",
              "      <td>-1.064453</td>\n",
              "      <td>-1.077035</td>\n",
              "      <td>-0.829626</td>\n",
              "      <td>-0.933843</td>\n",
              "      <td>-0.969972</td>\n",
              "      <td>-1.097135</td>\n",
              "      <td>-0.920029</td>\n",
              "      <td>-1.006142</td>\n",
              "      <td>-1.000139</td>\n",
              "      <td>-0.798836</td>\n",
              "      <td>-0.661197</td>\n",
              "      <td>-0.668804</td>\n",
              "      <td>-0.801681</td>\n",
              "      <td>-0.726147</td>\n",
              "      <td>-0.698734</td>\n",
              "      <td>-0.733370</td>\n",
              "      <td>-0.848113</td>\n",
              "      <td>-0.673804</td>\n",
              "      <td>-0.609317</td>\n",
              "      <td>-0.622568</td>\n",
              "      <td>-0.618201</td>\n",
              "      <td>-0.605566</td>\n",
              "      <td>-0.802281</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>3.256368e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-133704.266328</td>\n",
              "      <td>-2325.696385</td>\n",
              "      <td>-0.788362</td>\n",
              "      <td>-0.664546</td>\n",
              "      <td>-0.477113</td>\n",
              "      <td>-0.343361</td>\n",
              "      <td>-0.235777</td>\n",
              "      <td>-0.286330</td>\n",
              "      <td>-0.177309</td>\n",
              "      <td>-0.293599</td>\n",
              "      <td>-0.303225</td>\n",
              "      <td>-0.331717</td>\n",
              "      <td>-0.199220</td>\n",
              "      <td>-0.219429</td>\n",
              "      <td>-0.162129</td>\n",
              "      <td>-0.217265</td>\n",
              "      <td>-0.194685</td>\n",
              "      <td>-0.175304</td>\n",
              "      <td>-0.133387</td>\n",
              "      <td>-0.160270</td>\n",
              "      <td>-0.129654</td>\n",
              "      <td>-0.139515</td>\n",
              "      <td>-0.128500</td>\n",
              "      <td>-0.177134</td>\n",
              "      <td>-0.166202</td>\n",
              "      <td>-0.149504</td>\n",
              "      <td>-0.137358</td>\n",
              "      <td>-0.112711</td>\n",
              "      <td>-0.130938</td>\n",
              "      <td>-0.103349</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>3.313910e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-127842.480782</td>\n",
              "      <td>-115.754324</td>\n",
              "      <td>-0.006292</td>\n",
              "      <td>-0.018057</td>\n",
              "      <td>-0.071860</td>\n",
              "      <td>-0.021788</td>\n",
              "      <td>0.012815</td>\n",
              "      <td>0.040862</td>\n",
              "      <td>0.041067</td>\n",
              "      <td>-0.032651</td>\n",
              "      <td>-0.004219</td>\n",
              "      <td>0.045433</td>\n",
              "      <td>0.064433</td>\n",
              "      <td>-0.006994</td>\n",
              "      <td>0.050675</td>\n",
              "      <td>-0.055315</td>\n",
              "      <td>-0.016151</td>\n",
              "      <td>-0.030800</td>\n",
              "      <td>0.015893</td>\n",
              "      <td>-0.034500</td>\n",
              "      <td>-0.022281</td>\n",
              "      <td>0.005188</td>\n",
              "      <td>-0.019857</td>\n",
              "      <td>0.013883</td>\n",
              "      <td>-0.022115</td>\n",
              "      <td>-0.013533</td>\n",
              "      <td>0.015496</td>\n",
              "      <td>0.015006</td>\n",
              "      <td>-0.039344</td>\n",
              "      <td>0.003225</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>7.474842e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>288278.651157</td>\n",
              "      <td>3037.817807</td>\n",
              "      <td>0.982128</td>\n",
              "      <td>0.441242</td>\n",
              "      <td>0.324820</td>\n",
              "      <td>0.293505</td>\n",
              "      <td>0.268270</td>\n",
              "      <td>0.227682</td>\n",
              "      <td>0.314777</td>\n",
              "      <td>0.274099</td>\n",
              "      <td>0.267319</td>\n",
              "      <td>0.166430</td>\n",
              "      <td>0.242591</td>\n",
              "      <td>0.186729</td>\n",
              "      <td>0.250783</td>\n",
              "      <td>0.109022</td>\n",
              "      <td>0.190730</td>\n",
              "      <td>0.179107</td>\n",
              "      <td>0.199225</td>\n",
              "      <td>0.175066</td>\n",
              "      <td>0.098883</td>\n",
              "      <td>0.180564</td>\n",
              "      <td>0.129010</td>\n",
              "      <td>0.157526</td>\n",
              "      <td>0.135655</td>\n",
              "      <td>0.122485</td>\n",
              "      <td>0.154603</td>\n",
              "      <td>0.166079</td>\n",
              "      <td>0.105260</td>\n",
              "      <td>0.113410</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.105895e+06</td>\n",
              "      <td>99.400000</td>\n",
              "      <td>1380.000000</td>\n",
              "      <td>369.000000</td>\n",
              "      <td>2520.000000</td>\n",
              "      <td>7.460000</td>\n",
              "      <td>997.000000</td>\n",
              "      <td>12.700000</td>\n",
              "      <td>1.850000</td>\n",
              "      <td>4.170000</td>\n",
              "      <td>96.700000</td>\n",
              "      <td>99.600000</td>\n",
              "      <td>13.200000</td>\n",
              "      <td>40700.000000</td>\n",
              "      <td>11.500000</td>\n",
              "      <td>79.900000</td>\n",
              "      <td>43.900000</td>\n",
              "      <td>1790.000000</td>\n",
              "      <td>22.800000</td>\n",
              "      <td>8.610000</td>\n",
              "      <td>17.000000</td>\n",
              "      <td>73.200000</td>\n",
              "      <td>52.600000</td>\n",
              "      <td>3.400000</td>\n",
              "      <td>286.000000</td>\n",
              "      <td>93.000000</td>\n",
              "      <td>1650.000000</td>\n",
              "      <td>449.000000</td>\n",
              "      <td>0.485000</td>\n",
              "      <td>0.850000</td>\n",
              "      <td>7.720000</td>\n",
              "      <td>0.774000</td>\n",
              "      <td>1.830000</td>\n",
              "      <td>0.303000</td>\n",
              "      <td>14.000000</td>\n",
              "      <td>0.222000</td>\n",
              "      <td>1.930000</td>\n",
              "      <td>1.860000</td>\n",
              "      <td>0.989000</td>\n",
              "      <td>42.400000</td>\n",
              "      <td>...</td>\n",
              "      <td>404.000000</td>\n",
              "      <td>42.700000</td>\n",
              "      <td>2.510000</td>\n",
              "      <td>376.000000</td>\n",
              "      <td>99.600000</td>\n",
              "      <td>1.100000</td>\n",
              "      <td>65.500000</td>\n",
              "      <td>12.200000</td>\n",
              "      <td>126.000000</td>\n",
              "      <td>9.200000</td>\n",
              "      <td>646709.579482</td>\n",
              "      <td>7635.496843</td>\n",
              "      <td>2.647689</td>\n",
              "      <td>1.869347</td>\n",
              "      <td>1.616849</td>\n",
              "      <td>2.041568</td>\n",
              "      <td>1.700616</td>\n",
              "      <td>1.764945</td>\n",
              "      <td>1.547652</td>\n",
              "      <td>1.228567</td>\n",
              "      <td>1.122911</td>\n",
              "      <td>1.166864</td>\n",
              "      <td>1.230920</td>\n",
              "      <td>1.301510</td>\n",
              "      <td>1.225628</td>\n",
              "      <td>1.031456</td>\n",
              "      <td>0.770631</td>\n",
              "      <td>0.843875</td>\n",
              "      <td>1.048969</td>\n",
              "      <td>0.867503</td>\n",
              "      <td>1.478596</td>\n",
              "      <td>0.881955</td>\n",
              "      <td>0.756656</td>\n",
              "      <td>0.908209</td>\n",
              "      <td>0.921199</td>\n",
              "      <td>0.965138</td>\n",
              "      <td>0.774711</td>\n",
              "      <td>0.770132</td>\n",
              "      <td>0.968808</td>\n",
              "      <td>0.881655</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8 rows × 151 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "             fdc_id          1004  ...            28            29\n",
              "count  1.048400e+04  10484.000000  ...  10484.000000  10484.000000\n",
              "mean   4.807307e+05      1.838933  ...     -0.007871     -0.002815\n",
              "std    2.543947e+05      7.873509  ...      0.210375      0.207533\n",
              "min    3.198770e+05      0.000000  ...     -0.605566     -0.802281\n",
              "25%    3.256368e+05      0.000000  ...     -0.130938     -0.103349\n",
              "50%    3.313910e+05      0.000000  ...     -0.039344      0.003225\n",
              "75%    7.474842e+05      0.000000  ...      0.105260      0.113410\n",
              "max    1.105895e+06     99.400000  ...      0.968808      0.881655\n",
              "\n",
              "[8 rows x 151 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 350
        },
        "id": "RTVDsREy7Q1O",
        "outputId": "e2ccc328-2361-454a-f46b-10a04f6cca4c"
      },
      "source": [
        "test_data.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>fdc_id</th>\n",
              "      <th>1004</th>\n",
              "      <th>1087</th>\n",
              "      <th>1090</th>\n",
              "      <th>1092</th>\n",
              "      <th>1095</th>\n",
              "      <th>1091</th>\n",
              "      <th>1089</th>\n",
              "      <th>1098</th>\n",
              "      <th>1101</th>\n",
              "      <th>1051</th>\n",
              "      <th>1007</th>\n",
              "      <th>1002</th>\n",
              "      <th>1093</th>\n",
              "      <th>1079</th>\n",
              "      <th>1003</th>\n",
              "      <th>1009</th>\n",
              "      <th>1102</th>\n",
              "      <th>1265</th>\n",
              "      <th>1266</th>\n",
              "      <th>1264</th>\n",
              "      <th>1315</th>\n",
              "      <th>1316</th>\n",
              "      <th>1314</th>\n",
              "      <th>1094</th>\n",
              "      <th>1097</th>\n",
              "      <th>1137</th>\n",
              "      <th>1146</th>\n",
              "      <th>1300</th>\n",
              "      <th>1299</th>\n",
              "      <th>1404</th>\n",
              "      <th>1267</th>\n",
              "      <th>1273</th>\n",
              "      <th>1323</th>\n",
              "      <th>1167</th>\n",
              "      <th>1271</th>\n",
              "      <th>1304</th>\n",
              "      <th>1166</th>\n",
              "      <th>1175</th>\n",
              "      <th>1263</th>\n",
              "      <th>...</th>\n",
              "      <th>1180</th>\n",
              "      <th>1194</th>\n",
              "      <th>1128</th>\n",
              "      <th>1195</th>\n",
              "      <th>1005</th>\n",
              "      <th>1075</th>\n",
              "      <th>1196</th>\n",
              "      <th>1197</th>\n",
              "      <th>1198</th>\n",
              "      <th>1084</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>2.621000e+03</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>4.808360e+05</td>\n",
              "      <td>1.767753</td>\n",
              "      <td>29.528043</td>\n",
              "      <td>15.304655</td>\n",
              "      <td>127.031286</td>\n",
              "      <td>0.411137</td>\n",
              "      <td>58.533766</td>\n",
              "      <td>0.488626</td>\n",
              "      <td>0.080763</td>\n",
              "      <td>0.160633</td>\n",
              "      <td>8.237734</td>\n",
              "      <td>0.499729</td>\n",
              "      <td>0.229878</td>\n",
              "      <td>162.231591</td>\n",
              "      <td>0.362572</td>\n",
              "      <td>2.061393</td>\n",
              "      <td>2.530340</td>\n",
              "      <td>46.380694</td>\n",
              "      <td>0.279637</td>\n",
              "      <td>0.105758</td>\n",
              "      <td>0.054155</td>\n",
              "      <td>0.535003</td>\n",
              "      <td>0.161425</td>\n",
              "      <td>0.022514</td>\n",
              "      <td>15.026326</td>\n",
              "      <td>2.994964</td>\n",
              "      <td>78.646318</td>\n",
              "      <td>12.272034</td>\n",
              "      <td>0.004897</td>\n",
              "      <td>0.005666</td>\n",
              "      <td>0.015179</td>\n",
              "      <td>0.003214</td>\n",
              "      <td>0.001885</td>\n",
              "      <td>0.002314</td>\n",
              "      <td>0.117130</td>\n",
              "      <td>0.002836</td>\n",
              "      <td>0.015014</td>\n",
              "      <td>0.013524</td>\n",
              "      <td>0.008942</td>\n",
              "      <td>0.015341</td>\n",
              "      <td>...</td>\n",
              "      <td>1.177604</td>\n",
              "      <td>0.053224</td>\n",
              "      <td>0.002552</td>\n",
              "      <td>0.976192</td>\n",
              "      <td>0.152476</td>\n",
              "      <td>0.002579</td>\n",
              "      <td>0.010988</td>\n",
              "      <td>0.042808</td>\n",
              "      <td>0.025486</td>\n",
              "      <td>0.019191</td>\n",
              "      <td>21572.916075</td>\n",
              "      <td>153.764490</td>\n",
              "      <td>0.120828</td>\n",
              "      <td>-0.088173</td>\n",
              "      <td>-0.047529</td>\n",
              "      <td>-0.010620</td>\n",
              "      <td>0.037588</td>\n",
              "      <td>0.022852</td>\n",
              "      <td>0.061117</td>\n",
              "      <td>0.045312</td>\n",
              "      <td>-0.006952</td>\n",
              "      <td>-0.061510</td>\n",
              "      <td>0.041546</td>\n",
              "      <td>0.014043</td>\n",
              "      <td>0.058051</td>\n",
              "      <td>-0.049911</td>\n",
              "      <td>-0.014357</td>\n",
              "      <td>-0.000596</td>\n",
              "      <td>0.043168</td>\n",
              "      <td>0.002061</td>\n",
              "      <td>-0.006847</td>\n",
              "      <td>0.013206</td>\n",
              "      <td>-0.009980</td>\n",
              "      <td>-0.008207</td>\n",
              "      <td>0.010237</td>\n",
              "      <td>-0.009849</td>\n",
              "      <td>0.014877</td>\n",
              "      <td>0.022725</td>\n",
              "      <td>-0.009483</td>\n",
              "      <td>-0.007460</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.544785e+05</td>\n",
              "      <td>7.697916</td>\n",
              "      <td>104.229568</td>\n",
              "      <td>47.032037</td>\n",
              "      <td>383.476861</td>\n",
              "      <td>1.085789</td>\n",
              "      <td>156.604018</td>\n",
              "      <td>1.491882</td>\n",
              "      <td>0.270735</td>\n",
              "      <td>0.546488</td>\n",
              "      <td>23.312503</td>\n",
              "      <td>4.823097</td>\n",
              "      <td>0.954442</td>\n",
              "      <td>2006.587891</td>\n",
              "      <td>1.216119</td>\n",
              "      <td>6.780884</td>\n",
              "      <td>9.405717</td>\n",
              "      <td>197.989572</td>\n",
              "      <td>1.525738</td>\n",
              "      <td>0.557532</td>\n",
              "      <td>0.391267</td>\n",
              "      <td>4.208065</td>\n",
              "      <td>1.422409</td>\n",
              "      <td>0.138656</td>\n",
              "      <td>57.540020</td>\n",
              "      <td>11.883744</td>\n",
              "      <td>302.393274</td>\n",
              "      <td>48.996164</td>\n",
              "      <td>0.029397</td>\n",
              "      <td>0.041630</td>\n",
              "      <td>0.213682</td>\n",
              "      <td>0.028586</td>\n",
              "      <td>0.036916</td>\n",
              "      <td>0.015139</td>\n",
              "      <td>0.798193</td>\n",
              "      <td>0.016710</td>\n",
              "      <td>0.103358</td>\n",
              "      <td>0.070314</td>\n",
              "      <td>0.055309</td>\n",
              "      <td>0.119789</td>\n",
              "      <td>...</td>\n",
              "      <td>17.976672</td>\n",
              "      <td>1.215442</td>\n",
              "      <td>0.052341</td>\n",
              "      <td>16.338307</td>\n",
              "      <td>2.825680</td>\n",
              "      <td>0.031680</td>\n",
              "      <td>0.164785</td>\n",
              "      <td>0.646115</td>\n",
              "      <td>0.447997</td>\n",
              "      <td>0.276983</td>\n",
              "      <td>254524.057706</td>\n",
              "      <td>3979.800284</td>\n",
              "      <td>1.125335</td>\n",
              "      <td>0.696901</td>\n",
              "      <td>0.593561</td>\n",
              "      <td>0.522688</td>\n",
              "      <td>0.454492</td>\n",
              "      <td>0.454899</td>\n",
              "      <td>0.418094</td>\n",
              "      <td>0.427441</td>\n",
              "      <td>0.399548</td>\n",
              "      <td>0.349958</td>\n",
              "      <td>0.342780</td>\n",
              "      <td>0.328279</td>\n",
              "      <td>0.321352</td>\n",
              "      <td>0.289075</td>\n",
              "      <td>0.269336</td>\n",
              "      <td>0.289248</td>\n",
              "      <td>0.256030</td>\n",
              "      <td>0.272083</td>\n",
              "      <td>0.236014</td>\n",
              "      <td>0.246748</td>\n",
              "      <td>0.241832</td>\n",
              "      <td>0.241427</td>\n",
              "      <td>0.236721</td>\n",
              "      <td>0.206354</td>\n",
              "      <td>0.224524</td>\n",
              "      <td>0.211945</td>\n",
              "      <td>0.206822</td>\n",
              "      <td>0.200258</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>3.198780e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-139554.920584</td>\n",
              "      <td>-7401.558240</td>\n",
              "      <td>-2.962405</td>\n",
              "      <td>-1.577342</td>\n",
              "      <td>-1.527263</td>\n",
              "      <td>-1.442463</td>\n",
              "      <td>-1.189817</td>\n",
              "      <td>-1.006999</td>\n",
              "      <td>-0.950851</td>\n",
              "      <td>-0.808687</td>\n",
              "      <td>-0.933856</td>\n",
              "      <td>-0.969439</td>\n",
              "      <td>-0.961805</td>\n",
              "      <td>-0.889532</td>\n",
              "      <td>-1.006043</td>\n",
              "      <td>-1.000203</td>\n",
              "      <td>-0.798091</td>\n",
              "      <td>-0.687075</td>\n",
              "      <td>-0.613786</td>\n",
              "      <td>-0.801703</td>\n",
              "      <td>-0.678342</td>\n",
              "      <td>-0.648339</td>\n",
              "      <td>-0.733317</td>\n",
              "      <td>-0.847795</td>\n",
              "      <td>-0.570389</td>\n",
              "      <td>-0.609083</td>\n",
              "      <td>-0.548479</td>\n",
              "      <td>-0.617991</td>\n",
              "      <td>-0.605527</td>\n",
              "      <td>-0.723781</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>3.257300e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-133609.273672</td>\n",
              "      <td>-2363.368248</td>\n",
              "      <td>-0.820904</td>\n",
              "      <td>-0.671543</td>\n",
              "      <td>-0.476913</td>\n",
              "      <td>-0.345315</td>\n",
              "      <td>-0.246092</td>\n",
              "      <td>-0.288083</td>\n",
              "      <td>-0.170787</td>\n",
              "      <td>-0.283309</td>\n",
              "      <td>-0.302658</td>\n",
              "      <td>-0.334026</td>\n",
              "      <td>-0.184878</td>\n",
              "      <td>-0.222849</td>\n",
              "      <td>-0.152735</td>\n",
              "      <td>-0.216593</td>\n",
              "      <td>-0.193624</td>\n",
              "      <td>-0.174951</td>\n",
              "      <td>-0.125512</td>\n",
              "      <td>-0.157261</td>\n",
              "      <td>-0.129686</td>\n",
              "      <td>-0.151529</td>\n",
              "      <td>-0.136937</td>\n",
              "      <td>-0.176882</td>\n",
              "      <td>-0.154057</td>\n",
              "      <td>-0.150147</td>\n",
              "      <td>-0.133360</td>\n",
              "      <td>-0.110359</td>\n",
              "      <td>-0.132679</td>\n",
              "      <td>-0.104974</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>3.314830e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-127748.761484</td>\n",
              "      <td>-252.350686</td>\n",
              "      <td>0.026713</td>\n",
              "      <td>-0.010729</td>\n",
              "      <td>-0.072314</td>\n",
              "      <td>-0.025157</td>\n",
              "      <td>-0.004068</td>\n",
              "      <td>0.040868</td>\n",
              "      <td>0.028471</td>\n",
              "      <td>-0.033440</td>\n",
              "      <td>-0.013315</td>\n",
              "      <td>0.040263</td>\n",
              "      <td>0.064911</td>\n",
              "      <td>-0.008833</td>\n",
              "      <td>0.054967</td>\n",
              "      <td>-0.051972</td>\n",
              "      <td>-0.027597</td>\n",
              "      <td>-0.030280</td>\n",
              "      <td>0.013200</td>\n",
              "      <td>-0.034030</td>\n",
              "      <td>-0.022273</td>\n",
              "      <td>-0.013266</td>\n",
              "      <td>-0.032071</td>\n",
              "      <td>0.007956</td>\n",
              "      <td>-0.022689</td>\n",
              "      <td>-0.022648</td>\n",
              "      <td>0.016770</td>\n",
              "      <td>0.003926</td>\n",
              "      <td>-0.039613</td>\n",
              "      <td>-0.002187</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>7.474250e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>288218.293889</td>\n",
              "      <td>3014.232766</td>\n",
              "      <td>0.990554</td>\n",
              "      <td>0.434246</td>\n",
              "      <td>0.323139</td>\n",
              "      <td>0.277699</td>\n",
              "      <td>0.234861</td>\n",
              "      <td>0.226925</td>\n",
              "      <td>0.279577</td>\n",
              "      <td>0.287347</td>\n",
              "      <td>0.262978</td>\n",
              "      <td>0.165174</td>\n",
              "      <td>0.267027</td>\n",
              "      <td>0.190587</td>\n",
              "      <td>0.253358</td>\n",
              "      <td>0.109219</td>\n",
              "      <td>0.189096</td>\n",
              "      <td>0.171073</td>\n",
              "      <td>0.188646</td>\n",
              "      <td>0.185413</td>\n",
              "      <td>0.098563</td>\n",
              "      <td>0.180564</td>\n",
              "      <td>0.124176</td>\n",
              "      <td>0.156376</td>\n",
              "      <td>0.141215</td>\n",
              "      <td>0.110624</td>\n",
              "      <td>0.155096</td>\n",
              "      <td>0.154093</td>\n",
              "      <td>0.100122</td>\n",
              "      <td>0.109099</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.105897e+06</td>\n",
              "      <td>88.100000</td>\n",
              "      <td>1390.000000</td>\n",
              "      <td>386.000000</td>\n",
              "      <td>2480.000000</td>\n",
              "      <td>7.660000</td>\n",
              "      <td>995.000000</td>\n",
              "      <td>12.300000</td>\n",
              "      <td>1.920000</td>\n",
              "      <td>7.870000</td>\n",
              "      <td>95.400000</td>\n",
              "      <td>99.600000</td>\n",
              "      <td>12.900000</td>\n",
              "      <td>40100.000000</td>\n",
              "      <td>11.200000</td>\n",
              "      <td>51.100000</td>\n",
              "      <td>42.700000</td>\n",
              "      <td>1460.000000</td>\n",
              "      <td>22.600000</td>\n",
              "      <td>7.460000</td>\n",
              "      <td>7.780000</td>\n",
              "      <td>71.700000</td>\n",
              "      <td>51.300000</td>\n",
              "      <td>2.690000</td>\n",
              "      <td>311.000000</td>\n",
              "      <td>91.000000</td>\n",
              "      <td>1710.000000</td>\n",
              "      <td>338.000000</td>\n",
              "      <td>0.469000</td>\n",
              "      <td>0.806000</td>\n",
              "      <td>7.950000</td>\n",
              "      <td>0.721000</td>\n",
              "      <td>1.760000</td>\n",
              "      <td>0.264000</td>\n",
              "      <td>13.700000</td>\n",
              "      <td>0.196000</td>\n",
              "      <td>1.680000</td>\n",
              "      <td>1.570000</td>\n",
              "      <td>0.866000</td>\n",
              "      <td>2.370000</td>\n",
              "      <td>...</td>\n",
              "      <td>360.000000</td>\n",
              "      <td>42.700000</td>\n",
              "      <td>1.640000</td>\n",
              "      <td>334.000000</td>\n",
              "      <td>80.800000</td>\n",
              "      <td>0.730000</td>\n",
              "      <td>6.000000</td>\n",
              "      <td>13.100000</td>\n",
              "      <td>10.800000</td>\n",
              "      <td>8.100000</td>\n",
              "      <td>646711.616853</td>\n",
              "      <td>7634.515887</td>\n",
              "      <td>2.647585</td>\n",
              "      <td>1.869309</td>\n",
              "      <td>1.433469</td>\n",
              "      <td>2.028810</td>\n",
              "      <td>1.592535</td>\n",
              "      <td>1.758867</td>\n",
              "      <td>1.547788</td>\n",
              "      <td>1.348236</td>\n",
              "      <td>1.068685</td>\n",
              "      <td>1.235694</td>\n",
              "      <td>1.200106</td>\n",
              "      <td>1.246412</td>\n",
              "      <td>1.140985</td>\n",
              "      <td>1.031464</td>\n",
              "      <td>0.746734</td>\n",
              "      <td>0.806215</td>\n",
              "      <td>1.049396</td>\n",
              "      <td>0.867459</td>\n",
              "      <td>1.478362</td>\n",
              "      <td>0.921429</td>\n",
              "      <td>0.756642</td>\n",
              "      <td>0.764919</td>\n",
              "      <td>0.832335</td>\n",
              "      <td>0.949775</td>\n",
              "      <td>0.774605</td>\n",
              "      <td>0.769898</td>\n",
              "      <td>0.961489</td>\n",
              "      <td>0.790081</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8 rows × 151 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "             fdc_id         1004  ...           28           29\n",
              "count  2.621000e+03  2621.000000  ...  2621.000000  2621.000000\n",
              "mean   4.808360e+05     1.767753  ...    -0.009483    -0.007460\n",
              "std    2.544785e+05     7.697916  ...     0.206822     0.200258\n",
              "min    3.198780e+05     0.000000  ...    -0.605527    -0.723781\n",
              "25%    3.257300e+05     0.000000  ...    -0.132679    -0.104974\n",
              "50%    3.314830e+05     0.000000  ...    -0.039613    -0.002187\n",
              "75%    7.474250e+05     0.000000  ...     0.100122     0.109099\n",
              "max    1.105897e+06    88.100000  ...     0.961489     0.790081\n",
              "\n",
              "[8 rows x 151 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZqE6WJGI7Q1O",
        "outputId": "327cbf9c-a5d4-4b3d-f152-479174fabe7e"
      },
      "source": [
        "indices = test_data['fdc_id']\n",
        "indices"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10512     748595\n",
              "12618    1105328\n",
              "5891      329882\n",
              "9742      747334\n",
              "8399      335084\n",
              "          ...   \n",
              "12133    1104708\n",
              "7710      333356\n",
              "5455      329214\n",
              "6016      330136\n",
              "8229      334695\n",
              "Name: fdc_id, Length: 2621, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BCqed7lF7Q1P"
      },
      "source": [
        "del train_data['fdc_id']\n",
        "del test_data['fdc_id']\n",
        "columns = train_data.columns"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "voSpBHe-7Q1P",
        "outputId": "0b747e77-4e44-4cb2-9687-8a74a7122282"
      },
      "source": [
        "train_data = train_data.to_numpy()\n",
        "test_data.fillna(0, inplace=True)\n",
        "test_data = test_data.to_numpy()\n",
        "data = dataset.values\n",
        "rows, cols = data.shape\n",
        "cols -= 1\n",
        "\n",
        "y_test = test_data.copy()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pandas/core/frame.py:4327: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  downcast=downcast,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 350
        },
        "id": "6uqlo1Un7Q1Q",
        "outputId": "d76fc535-a53b-432d-89a1-451cfb2c2607"
      },
      "source": [
        "missed_data, missed_data_train, mask = missing_method(test_data, train_data, num_embeddings)\n",
        "pd.DataFrame(missed_data).describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "      <th>30</th>\n",
              "      <th>31</th>\n",
              "      <th>32</th>\n",
              "      <th>33</th>\n",
              "      <th>34</th>\n",
              "      <th>35</th>\n",
              "      <th>36</th>\n",
              "      <th>37</th>\n",
              "      <th>38</th>\n",
              "      <th>39</th>\n",
              "      <th>...</th>\n",
              "      <th>110</th>\n",
              "      <th>111</th>\n",
              "      <th>112</th>\n",
              "      <th>113</th>\n",
              "      <th>114</th>\n",
              "      <th>115</th>\n",
              "      <th>116</th>\n",
              "      <th>117</th>\n",
              "      <th>118</th>\n",
              "      <th>119</th>\n",
              "      <th>120</th>\n",
              "      <th>121</th>\n",
              "      <th>122</th>\n",
              "      <th>123</th>\n",
              "      <th>124</th>\n",
              "      <th>125</th>\n",
              "      <th>126</th>\n",
              "      <th>127</th>\n",
              "      <th>128</th>\n",
              "      <th>129</th>\n",
              "      <th>130</th>\n",
              "      <th>131</th>\n",
              "      <th>132</th>\n",
              "      <th>133</th>\n",
              "      <th>134</th>\n",
              "      <th>135</th>\n",
              "      <th>136</th>\n",
              "      <th>137</th>\n",
              "      <th>138</th>\n",
              "      <th>139</th>\n",
              "      <th>140</th>\n",
              "      <th>141</th>\n",
              "      <th>142</th>\n",
              "      <th>143</th>\n",
              "      <th>144</th>\n",
              "      <th>145</th>\n",
              "      <th>146</th>\n",
              "      <th>147</th>\n",
              "      <th>148</th>\n",
              "      <th>149</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>2079.000000</td>\n",
              "      <td>2131.000000</td>\n",
              "      <td>2122.000000</td>\n",
              "      <td>2116.000000</td>\n",
              "      <td>2114.000000</td>\n",
              "      <td>2112.000000</td>\n",
              "      <td>2101.000000</td>\n",
              "      <td>2086.000000</td>\n",
              "      <td>2104.000000</td>\n",
              "      <td>2071.000000</td>\n",
              "      <td>2086.000000</td>\n",
              "      <td>2115.000000</td>\n",
              "      <td>2104.000000</td>\n",
              "      <td>2096.000000</td>\n",
              "      <td>2098.000000</td>\n",
              "      <td>2087.000000</td>\n",
              "      <td>2097.000000</td>\n",
              "      <td>2102.000000</td>\n",
              "      <td>2109.000000</td>\n",
              "      <td>2098.000000</td>\n",
              "      <td>2086.000000</td>\n",
              "      <td>2053.000000</td>\n",
              "      <td>2091.000000</td>\n",
              "      <td>2081.000000</td>\n",
              "      <td>2129.000000</td>\n",
              "      <td>2076.000000</td>\n",
              "      <td>2088.000000</td>\n",
              "      <td>2074.000000</td>\n",
              "      <td>2119.000000</td>\n",
              "      <td>2121.000000</td>\n",
              "      <td>2093.000000</td>\n",
              "      <td>2129.000000</td>\n",
              "      <td>2108.000000</td>\n",
              "      <td>2122.000000</td>\n",
              "      <td>2104.000000</td>\n",
              "      <td>2108.000000</td>\n",
              "      <td>2109.000000</td>\n",
              "      <td>2072.000000</td>\n",
              "      <td>2064.000000</td>\n",
              "      <td>2115.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>2095.000000</td>\n",
              "      <td>2063.000000</td>\n",
              "      <td>2100.000000</td>\n",
              "      <td>2082.000000</td>\n",
              "      <td>2096.000000</td>\n",
              "      <td>2084.000000</td>\n",
              "      <td>2116.000000</td>\n",
              "      <td>2135.000000</td>\n",
              "      <td>2073.000000</td>\n",
              "      <td>2099.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>1.822968</td>\n",
              "      <td>29.281089</td>\n",
              "      <td>15.401979</td>\n",
              "      <td>127.583176</td>\n",
              "      <td>0.413018</td>\n",
              "      <td>58.667614</td>\n",
              "      <td>0.482922</td>\n",
              "      <td>0.082035</td>\n",
              "      <td>0.162047</td>\n",
              "      <td>8.100990</td>\n",
              "      <td>0.582287</td>\n",
              "      <td>0.235206</td>\n",
              "      <td>111.011882</td>\n",
              "      <td>0.379628</td>\n",
              "      <td>2.164971</td>\n",
              "      <td>2.341284</td>\n",
              "      <td>49.029375</td>\n",
              "      <td>0.243778</td>\n",
              "      <td>0.105295</td>\n",
              "      <td>0.046996</td>\n",
              "      <td>0.452872</td>\n",
              "      <td>0.150144</td>\n",
              "      <td>0.024538</td>\n",
              "      <td>13.952427</td>\n",
              "      <td>3.262001</td>\n",
              "      <td>75.279865</td>\n",
              "      <td>12.383621</td>\n",
              "      <td>0.005118</td>\n",
              "      <td>0.005440</td>\n",
              "      <td>0.013107</td>\n",
              "      <td>0.003535</td>\n",
              "      <td>0.001947</td>\n",
              "      <td>0.002442</td>\n",
              "      <td>0.121552</td>\n",
              "      <td>0.002490</td>\n",
              "      <td>0.015886</td>\n",
              "      <td>0.014331</td>\n",
              "      <td>0.008495</td>\n",
              "      <td>0.014514</td>\n",
              "      <td>0.007035</td>\n",
              "      <td>...</td>\n",
              "      <td>1.207351</td>\n",
              "      <td>0.056714</td>\n",
              "      <td>0.002152</td>\n",
              "      <td>0.855187</td>\n",
              "      <td>0.133941</td>\n",
              "      <td>0.002893</td>\n",
              "      <td>0.010491</td>\n",
              "      <td>0.040094</td>\n",
              "      <td>0.018186</td>\n",
              "      <td>0.016532</td>\n",
              "      <td>21572.916075</td>\n",
              "      <td>153.764490</td>\n",
              "      <td>0.120828</td>\n",
              "      <td>-0.088173</td>\n",
              "      <td>-0.047529</td>\n",
              "      <td>-0.010620</td>\n",
              "      <td>0.037588</td>\n",
              "      <td>0.022852</td>\n",
              "      <td>0.061117</td>\n",
              "      <td>0.045312</td>\n",
              "      <td>-0.006952</td>\n",
              "      <td>-0.061510</td>\n",
              "      <td>0.041546</td>\n",
              "      <td>0.014043</td>\n",
              "      <td>0.058051</td>\n",
              "      <td>-0.049911</td>\n",
              "      <td>-0.014357</td>\n",
              "      <td>-0.000596</td>\n",
              "      <td>0.043168</td>\n",
              "      <td>0.002061</td>\n",
              "      <td>-0.006847</td>\n",
              "      <td>0.013206</td>\n",
              "      <td>-0.009980</td>\n",
              "      <td>-0.008207</td>\n",
              "      <td>0.010237</td>\n",
              "      <td>-0.009849</td>\n",
              "      <td>0.014877</td>\n",
              "      <td>0.022725</td>\n",
              "      <td>-0.009483</td>\n",
              "      <td>-0.007460</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>7.688367</td>\n",
              "      <td>105.779651</td>\n",
              "      <td>46.881167</td>\n",
              "      <td>386.194247</td>\n",
              "      <td>1.093257</td>\n",
              "      <td>156.451379</td>\n",
              "      <td>1.484678</td>\n",
              "      <td>0.272904</td>\n",
              "      <td>0.555224</td>\n",
              "      <td>23.085199</td>\n",
              "      <td>5.388674</td>\n",
              "      <td>0.954813</td>\n",
              "      <td>1455.138966</td>\n",
              "      <td>1.238272</td>\n",
              "      <td>6.922343</td>\n",
              "      <td>9.048164</td>\n",
              "      <td>203.083355</td>\n",
              "      <td>1.361023</td>\n",
              "      <td>0.549890</td>\n",
              "      <td>0.371828</td>\n",
              "      <td>3.720943</td>\n",
              "      <td>1.443742</td>\n",
              "      <td>0.150145</td>\n",
              "      <td>55.487474</td>\n",
              "      <td>12.358336</td>\n",
              "      <td>295.218949</td>\n",
              "      <td>48.990889</td>\n",
              "      <td>0.030959</td>\n",
              "      <td>0.041524</td>\n",
              "      <td>0.183330</td>\n",
              "      <td>0.030463</td>\n",
              "      <td>0.040624</td>\n",
              "      <td>0.015608</td>\n",
              "      <td>0.825986</td>\n",
              "      <td>0.015069</td>\n",
              "      <td>0.107239</td>\n",
              "      <td>0.072712</td>\n",
              "      <td>0.054448</td>\n",
              "      <td>0.119956</td>\n",
              "      <td>0.052296</td>\n",
              "      <td>...</td>\n",
              "      <td>18.558856</td>\n",
              "      <td>1.345060</td>\n",
              "      <td>0.045922</td>\n",
              "      <td>15.254636</td>\n",
              "      <td>2.571180</td>\n",
              "      <td>0.034639</td>\n",
              "      <td>0.128727</td>\n",
              "      <td>0.617660</td>\n",
              "      <td>0.376355</td>\n",
              "      <td>0.235400</td>\n",
              "      <td>254524.057706</td>\n",
              "      <td>3979.800284</td>\n",
              "      <td>1.125335</td>\n",
              "      <td>0.696901</td>\n",
              "      <td>0.593561</td>\n",
              "      <td>0.522688</td>\n",
              "      <td>0.454492</td>\n",
              "      <td>0.454899</td>\n",
              "      <td>0.418094</td>\n",
              "      <td>0.427441</td>\n",
              "      <td>0.399548</td>\n",
              "      <td>0.349958</td>\n",
              "      <td>0.342780</td>\n",
              "      <td>0.328279</td>\n",
              "      <td>0.321352</td>\n",
              "      <td>0.289075</td>\n",
              "      <td>0.269336</td>\n",
              "      <td>0.289248</td>\n",
              "      <td>0.256030</td>\n",
              "      <td>0.272083</td>\n",
              "      <td>0.236014</td>\n",
              "      <td>0.246748</td>\n",
              "      <td>0.241832</td>\n",
              "      <td>0.241427</td>\n",
              "      <td>0.236721</td>\n",
              "      <td>0.206354</td>\n",
              "      <td>0.224524</td>\n",
              "      <td>0.211945</td>\n",
              "      <td>0.206822</td>\n",
              "      <td>0.200258</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-139554.920584</td>\n",
              "      <td>-7401.558240</td>\n",
              "      <td>-2.962405</td>\n",
              "      <td>-1.577342</td>\n",
              "      <td>-1.527263</td>\n",
              "      <td>-1.442463</td>\n",
              "      <td>-1.189817</td>\n",
              "      <td>-1.006999</td>\n",
              "      <td>-0.950851</td>\n",
              "      <td>-0.808687</td>\n",
              "      <td>-0.933856</td>\n",
              "      <td>-0.969439</td>\n",
              "      <td>-0.961805</td>\n",
              "      <td>-0.889532</td>\n",
              "      <td>-1.006043</td>\n",
              "      <td>-1.000203</td>\n",
              "      <td>-0.798091</td>\n",
              "      <td>-0.687075</td>\n",
              "      <td>-0.613786</td>\n",
              "      <td>-0.801703</td>\n",
              "      <td>-0.678342</td>\n",
              "      <td>-0.648339</td>\n",
              "      <td>-0.733317</td>\n",
              "      <td>-0.847795</td>\n",
              "      <td>-0.570389</td>\n",
              "      <td>-0.609083</td>\n",
              "      <td>-0.548479</td>\n",
              "      <td>-0.617991</td>\n",
              "      <td>-0.605527</td>\n",
              "      <td>-0.723781</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-133609.273672</td>\n",
              "      <td>-2363.368248</td>\n",
              "      <td>-0.820904</td>\n",
              "      <td>-0.671543</td>\n",
              "      <td>-0.476913</td>\n",
              "      <td>-0.345315</td>\n",
              "      <td>-0.246092</td>\n",
              "      <td>-0.288083</td>\n",
              "      <td>-0.170787</td>\n",
              "      <td>-0.283309</td>\n",
              "      <td>-0.302658</td>\n",
              "      <td>-0.334026</td>\n",
              "      <td>-0.184878</td>\n",
              "      <td>-0.222849</td>\n",
              "      <td>-0.152735</td>\n",
              "      <td>-0.216593</td>\n",
              "      <td>-0.193624</td>\n",
              "      <td>-0.174951</td>\n",
              "      <td>-0.125512</td>\n",
              "      <td>-0.157261</td>\n",
              "      <td>-0.129686</td>\n",
              "      <td>-0.151529</td>\n",
              "      <td>-0.136937</td>\n",
              "      <td>-0.176882</td>\n",
              "      <td>-0.154057</td>\n",
              "      <td>-0.150147</td>\n",
              "      <td>-0.133360</td>\n",
              "      <td>-0.110359</td>\n",
              "      <td>-0.132679</td>\n",
              "      <td>-0.104974</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-127748.761484</td>\n",
              "      <td>-252.350686</td>\n",
              "      <td>0.026713</td>\n",
              "      <td>-0.010729</td>\n",
              "      <td>-0.072314</td>\n",
              "      <td>-0.025157</td>\n",
              "      <td>-0.004068</td>\n",
              "      <td>0.040868</td>\n",
              "      <td>0.028471</td>\n",
              "      <td>-0.033440</td>\n",
              "      <td>-0.013315</td>\n",
              "      <td>0.040263</td>\n",
              "      <td>0.064911</td>\n",
              "      <td>-0.008833</td>\n",
              "      <td>0.054967</td>\n",
              "      <td>-0.051972</td>\n",
              "      <td>-0.027597</td>\n",
              "      <td>-0.030280</td>\n",
              "      <td>0.013200</td>\n",
              "      <td>-0.034030</td>\n",
              "      <td>-0.022273</td>\n",
              "      <td>-0.013266</td>\n",
              "      <td>-0.032071</td>\n",
              "      <td>0.007956</td>\n",
              "      <td>-0.022689</td>\n",
              "      <td>-0.022648</td>\n",
              "      <td>0.016770</td>\n",
              "      <td>0.003926</td>\n",
              "      <td>-0.039613</td>\n",
              "      <td>-0.002187</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>288218.293889</td>\n",
              "      <td>3014.232766</td>\n",
              "      <td>0.990554</td>\n",
              "      <td>0.434246</td>\n",
              "      <td>0.323139</td>\n",
              "      <td>0.277699</td>\n",
              "      <td>0.234861</td>\n",
              "      <td>0.226925</td>\n",
              "      <td>0.279577</td>\n",
              "      <td>0.287347</td>\n",
              "      <td>0.262978</td>\n",
              "      <td>0.165174</td>\n",
              "      <td>0.267027</td>\n",
              "      <td>0.190587</td>\n",
              "      <td>0.253358</td>\n",
              "      <td>0.109219</td>\n",
              "      <td>0.189096</td>\n",
              "      <td>0.171073</td>\n",
              "      <td>0.188646</td>\n",
              "      <td>0.185413</td>\n",
              "      <td>0.098563</td>\n",
              "      <td>0.180564</td>\n",
              "      <td>0.124176</td>\n",
              "      <td>0.156376</td>\n",
              "      <td>0.141215</td>\n",
              "      <td>0.110624</td>\n",
              "      <td>0.155096</td>\n",
              "      <td>0.154093</td>\n",
              "      <td>0.100122</td>\n",
              "      <td>0.109099</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>88.100000</td>\n",
              "      <td>1390.000000</td>\n",
              "      <td>386.000000</td>\n",
              "      <td>2480.000000</td>\n",
              "      <td>7.660000</td>\n",
              "      <td>995.000000</td>\n",
              "      <td>12.300000</td>\n",
              "      <td>1.920000</td>\n",
              "      <td>7.870000</td>\n",
              "      <td>95.400000</td>\n",
              "      <td>99.600000</td>\n",
              "      <td>12.900000</td>\n",
              "      <td>38300.000000</td>\n",
              "      <td>11.200000</td>\n",
              "      <td>51.100000</td>\n",
              "      <td>42.700000</td>\n",
              "      <td>1460.000000</td>\n",
              "      <td>15.800000</td>\n",
              "      <td>6.530000</td>\n",
              "      <td>7.780000</td>\n",
              "      <td>68.600000</td>\n",
              "      <td>51.300000</td>\n",
              "      <td>2.690000</td>\n",
              "      <td>283.000000</td>\n",
              "      <td>91.000000</td>\n",
              "      <td>1600.000000</td>\n",
              "      <td>338.000000</td>\n",
              "      <td>0.469000</td>\n",
              "      <td>0.806000</td>\n",
              "      <td>7.950000</td>\n",
              "      <td>0.721000</td>\n",
              "      <td>1.760000</td>\n",
              "      <td>0.264000</td>\n",
              "      <td>13.700000</td>\n",
              "      <td>0.193000</td>\n",
              "      <td>1.680000</td>\n",
              "      <td>1.570000</td>\n",
              "      <td>0.866000</td>\n",
              "      <td>2.370000</td>\n",
              "      <td>0.939000</td>\n",
              "      <td>...</td>\n",
              "      <td>360.000000</td>\n",
              "      <td>42.700000</td>\n",
              "      <td>1.580000</td>\n",
              "      <td>334.000000</td>\n",
              "      <td>80.800000</td>\n",
              "      <td>0.730000</td>\n",
              "      <td>2.300000</td>\n",
              "      <td>13.100000</td>\n",
              "      <td>10.800000</td>\n",
              "      <td>8.100000</td>\n",
              "      <td>646711.616853</td>\n",
              "      <td>7634.515887</td>\n",
              "      <td>2.647585</td>\n",
              "      <td>1.869309</td>\n",
              "      <td>1.433469</td>\n",
              "      <td>2.028810</td>\n",
              "      <td>1.592535</td>\n",
              "      <td>1.758867</td>\n",
              "      <td>1.547788</td>\n",
              "      <td>1.348236</td>\n",
              "      <td>1.068685</td>\n",
              "      <td>1.235694</td>\n",
              "      <td>1.200106</td>\n",
              "      <td>1.246412</td>\n",
              "      <td>1.140985</td>\n",
              "      <td>1.031464</td>\n",
              "      <td>0.746734</td>\n",
              "      <td>0.806215</td>\n",
              "      <td>1.049396</td>\n",
              "      <td>0.867459</td>\n",
              "      <td>1.478362</td>\n",
              "      <td>0.921429</td>\n",
              "      <td>0.756642</td>\n",
              "      <td>0.764919</td>\n",
              "      <td>0.832335</td>\n",
              "      <td>0.949775</td>\n",
              "      <td>0.774605</td>\n",
              "      <td>0.769898</td>\n",
              "      <td>0.961489</td>\n",
              "      <td>0.790081</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8 rows × 150 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "               0            1    ...          148          149\n",
              "count  2079.000000  2131.000000  ...  2621.000000  2621.000000\n",
              "mean      1.822968    29.281089  ...    -0.009483    -0.007460\n",
              "std       7.688367   105.779651  ...     0.206822     0.200258\n",
              "min       0.000000     0.000000  ...    -0.605527    -0.723781\n",
              "25%       0.000000     0.000000  ...    -0.132679    -0.104974\n",
              "50%       0.000000     0.000000  ...    -0.039613    -0.002187\n",
              "75%       0.000000     0.000000  ...     0.100122     0.109099\n",
              "max      88.100000  1390.000000  ...     0.961489     0.790081\n",
              "\n",
              "[8 rows x 150 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4oDxsxKS7Q1Q"
      },
      "source": [
        "# traditional methods test sets - impute across all foods\n",
        "X_train = pd.DataFrame(missed_data_train, columns = columns)\n",
        "X_test_mean_all = pd.DataFrame(missed_data, columns = columns)\n",
        "X_test_median_all = pd.DataFrame(missed_data, columns = columns)\n",
        "\n",
        "X_test_mean_all = impute_traditional_all_foods(X_train, X_test_mean_all, 'mean')\n",
        "X_test_median_all = impute_traditional_all_foods(X_train, X_test_median_all, 'median')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tmekCL2p7Q1R"
      },
      "source": [
        "# Traditional methods test sets - impute by food category\n",
        "X_test_mean = pd.DataFrame(missed_data, columns = columns, index=indices)\n",
        "X_test_median = X_test_mean.copy()\n",
        "df_mean_cat = df_mean[df_mean['fdc_id'].isin(indices)]\n",
        "X_test_mean = impute_traditional_by_category(X_test_mean, df_mean_cat, indices)\n",
        "df_median_cat = df_median[df_median['fdc_id'].isin(indices)]\n",
        "X_test_median = impute_traditional_by_category(X_test_median, df_median_cat, indices)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hDdpm9CE7Q1R"
      },
      "source": [
        "scaler = MinMaxScaler()\n",
        "scaler.fit(train_data)\n",
        "train_data = scaler.transform(train_data)\n",
        "test_data = scaler.transform(test_data)\n",
        "X_test_mean = scaler.transform(X_test_mean)\n",
        "X_test_median = scaler.transform(X_test_median)\n",
        "X_test_mean_all = scaler.transform(X_test_mean_all)\n",
        "X_test_median_all = scaler.transform(X_test_median_all)\n",
        "\n",
        "y_test = scaler.transform(y_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DRXHS3nl7Q1S"
      },
      "source": [
        "# datasets without embeddings for autoencoder\n",
        "train_data_noEmbeddings = pd.DataFrame(train_data, columns = columns)\n",
        "train_data_noEmbeddings = np.array(train_data_noEmbeddings.iloc[:, :-num_embeddings]) # df = df.iloc[: , :-1]\n",
        "test_mean_noEmbeddings = np.array(pd.DataFrame(X_test_mean, columns = columns).iloc[:, :-num_embeddings])\n",
        "test_median_noEmbeddings = np.array(pd.DataFrame(X_test_median, columns = columns).iloc[:, :-num_embeddings])\n",
        "test_mean_noEmbeddings_all = np.array(pd.DataFrame(X_test_mean_all, columns = columns).iloc[:, :-num_embeddings])\n",
        "test_median_noEmbeddings_all = np.array(pd.DataFrame(X_test_median_all, columns = columns).iloc[:, :-num_embeddings])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iycF1q1-7Q1T"
      },
      "source": [
        "missed_data = X_test_mean_all\n",
        "missed_data = torch.from_numpy(missed_data).float()\n",
        "\n",
        "train_data = torch.from_numpy(train_data).float()\n",
        "\n",
        "train_loader = torch.utils.data.DataLoader(dataset=train_data,\n",
        "                                           batch_size=batch_size,\n",
        "                                           shuffle=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kdW_6y2P7Q1T"
      },
      "source": [
        "missed_data_noEm = test_mean_noEmbeddings_all\n",
        "missed_data_noEm = torch.from_numpy(missed_data_noEm).float()\n",
        "\n",
        "train_data_noEmbeddings = torch.from_numpy(train_data_noEmbeddings).float()\n",
        "\n",
        "train_loader_noEm = torch.utils.data.DataLoader(dataset=train_data_noEmbeddings,\n",
        "                                           batch_size=batch_size,\n",
        "                                           shuffle=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "87ntMh087Q1U"
      },
      "source": [
        "## Autoencoder - without embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8J4Vobpb7Q1U"
      },
      "source": [
        "model = Autoencoder(dim=cols-num_embeddings).to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "utNHvU1k7Q1U"
      },
      "source": [
        "criterion = nn.MSELoss()\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U_aasBdy7Q1U",
        "outputId": "c4a01cc9-7691-4cec-ec91-9343688bafba"
      },
      "source": [
        "training(num_epochs, model, train_loader_noEm, criterion, optimizer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch : 1/200, recon loss = 0.00581219\n",
            "epoch : 2/200, recon loss = 0.00472384\n",
            "epoch : 3/200, recon loss = 0.00469371\n",
            "epoch : 4/200, recon loss = 0.00465589\n",
            "epoch : 5/200, recon loss = 0.00459831\n",
            "epoch : 6/200, recon loss = 0.00449245\n",
            "epoch : 7/200, recon loss = 0.00428894\n",
            "epoch : 8/200, recon loss = 0.00391550\n",
            "epoch : 9/200, recon loss = 0.00337470\n",
            "epoch : 10/200, recon loss = 0.00294085\n",
            "epoch : 11/200, recon loss = 0.00269202\n",
            "epoch : 12/200, recon loss = 0.00257523\n",
            "epoch : 13/200, recon loss = 0.00252215\n",
            "epoch : 14/200, recon loss = 0.00249634\n",
            "epoch : 15/200, recon loss = 0.00247268\n",
            "epoch : 16/200, recon loss = 0.00246725\n",
            "epoch : 17/200, recon loss = 0.00245641\n",
            "epoch : 18/200, recon loss = 0.00244636\n",
            "epoch : 19/200, recon loss = 0.00243947\n",
            "epoch : 20/200, recon loss = 0.00242779\n",
            "epoch : 21/200, recon loss = 0.00241965\n",
            "epoch : 22/200, recon loss = 0.00240268\n",
            "epoch : 23/200, recon loss = 0.00238925\n",
            "epoch : 24/200, recon loss = 0.00236624\n",
            "epoch : 25/200, recon loss = 0.00235028\n",
            "epoch : 26/200, recon loss = 0.00232582\n",
            "epoch : 27/200, recon loss = 0.00230292\n",
            "epoch : 28/200, recon loss = 0.00228021\n",
            "epoch : 29/200, recon loss = 0.00224309\n",
            "epoch : 30/200, recon loss = 0.00220576\n",
            "epoch : 31/200, recon loss = 0.00216861\n",
            "epoch : 32/200, recon loss = 0.00212957\n",
            "epoch : 33/200, recon loss = 0.00209775\n",
            "epoch : 34/200, recon loss = 0.00205561\n",
            "epoch : 35/200, recon loss = 0.00201735\n",
            "epoch : 36/200, recon loss = 0.00196773\n",
            "epoch : 37/200, recon loss = 0.00193999\n",
            "epoch : 38/200, recon loss = 0.00190519\n",
            "epoch : 39/200, recon loss = 0.00187243\n",
            "epoch : 40/200, recon loss = 0.00184095\n",
            "epoch : 41/200, recon loss = 0.00181372\n",
            "epoch : 42/200, recon loss = 0.00180114\n",
            "epoch : 43/200, recon loss = 0.00176659\n",
            "epoch : 44/200, recon loss = 0.00176652\n",
            "epoch : 45/200, recon loss = 0.00173519\n",
            "epoch : 46/200, recon loss = 0.00172707\n",
            "epoch : 47/200, recon loss = 0.00170773\n",
            "epoch : 48/200, recon loss = 0.00169642\n",
            "epoch : 49/200, recon loss = 0.00169948\n",
            "epoch : 50/200, recon loss = 0.00168554\n",
            "epoch : 51/200, recon loss = 0.00168756\n",
            "epoch : 52/200, recon loss = 0.00167748\n",
            "epoch : 53/200, recon loss = 0.00166956\n",
            "epoch : 54/200, recon loss = 0.00166567\n",
            "epoch : 55/200, recon loss = 0.00166810\n",
            "epoch : 56/200, recon loss = 0.00167475\n",
            "epoch : 57/200, recon loss = 0.00166172\n",
            "epoch : 58/200, recon loss = 0.00165934\n",
            "epoch : 59/200, recon loss = 0.00166357\n",
            "epoch : 60/200, recon loss = 0.00165137\n",
            "epoch : 61/200, recon loss = 0.00166391\n",
            "epoch : 62/200, recon loss = 0.00165015\n",
            "epoch : 63/200, recon loss = 0.00166332\n",
            "epoch : 64/200, recon loss = 0.00164248\n",
            "epoch : 65/200, recon loss = 0.00163307\n",
            "epoch : 66/200, recon loss = 0.00164418\n",
            "epoch : 67/200, recon loss = 0.00163263\n",
            "epoch : 68/200, recon loss = 0.00162179\n",
            "epoch : 69/200, recon loss = 0.00163741\n",
            "epoch : 70/200, recon loss = 0.00164869\n",
            "epoch : 71/200, recon loss = 0.00161619\n",
            "epoch : 72/200, recon loss = 0.00162730\n",
            "epoch : 73/200, recon loss = 0.00162647\n",
            "epoch : 74/200, recon loss = 0.00162442\n",
            "epoch : 75/200, recon loss = 0.00163264\n",
            "epoch : 76/200, recon loss = 0.00161768\n",
            "epoch : 77/200, recon loss = 0.00163103\n",
            "epoch : 78/200, recon loss = 0.00162558\n",
            "epoch : 79/200, recon loss = 0.00163367\n",
            "epoch : 80/200, recon loss = 0.00160397\n",
            "epoch : 81/200, recon loss = 0.00162261\n",
            "epoch : 82/200, recon loss = 0.00160586\n",
            "epoch : 83/200, recon loss = 0.00160481\n",
            "epoch : 84/200, recon loss = 0.00160852\n",
            "epoch : 85/200, recon loss = 0.00160007\n",
            "epoch : 86/200, recon loss = 0.00159608\n",
            "epoch : 87/200, recon loss = 0.00160344\n",
            "epoch : 88/200, recon loss = 0.00159400\n",
            "epoch : 89/200, recon loss = 0.00158683\n",
            "epoch : 90/200, recon loss = 0.00159344\n",
            "epoch : 91/200, recon loss = 0.00159453\n",
            "epoch : 92/200, recon loss = 0.00158778\n",
            "epoch : 93/200, recon loss = 0.00157792\n",
            "epoch : 94/200, recon loss = 0.00158722\n",
            "epoch : 95/200, recon loss = 0.00157023\n",
            "epoch : 96/200, recon loss = 0.00157989\n",
            "epoch : 97/200, recon loss = 0.00157766\n",
            "epoch : 98/200, recon loss = 0.00157501\n",
            "epoch : 99/200, recon loss = 0.00155958\n",
            "epoch : 100/200, recon loss = 0.00154399\n",
            "epoch : 101/200, recon loss = 0.00155229\n",
            "epoch : 102/200, recon loss = 0.00155295\n",
            "epoch : 103/200, recon loss = 0.00154858\n",
            "epoch : 104/200, recon loss = 0.00154500\n",
            "epoch : 105/200, recon loss = 0.00153226\n",
            "epoch : 106/200, recon loss = 0.00153742\n",
            "epoch : 107/200, recon loss = 0.00152510\n",
            "epoch : 108/200, recon loss = 0.00152644\n",
            "epoch : 109/200, recon loss = 0.00150284\n",
            "epoch : 110/200, recon loss = 0.00151330\n",
            "epoch : 111/200, recon loss = 0.00151725\n",
            "epoch : 112/200, recon loss = 0.00150211\n",
            "epoch : 113/200, recon loss = 0.00150176\n",
            "epoch : 114/200, recon loss = 0.00150874\n",
            "epoch : 115/200, recon loss = 0.00148702\n",
            "epoch : 116/200, recon loss = 0.00148766\n",
            "epoch : 117/200, recon loss = 0.00147896\n",
            "epoch : 118/200, recon loss = 0.00149257\n",
            "epoch : 119/200, recon loss = 0.00147551\n",
            "epoch : 120/200, recon loss = 0.00147615\n",
            "epoch : 121/200, recon loss = 0.00146435\n",
            "epoch : 122/200, recon loss = 0.00146380\n",
            "epoch : 123/200, recon loss = 0.00146118\n",
            "epoch : 124/200, recon loss = 0.00145393\n",
            "epoch : 125/200, recon loss = 0.00143742\n",
            "epoch : 126/200, recon loss = 0.00144571\n",
            "epoch : 127/200, recon loss = 0.00142680\n",
            "epoch : 128/200, recon loss = 0.00143138\n",
            "epoch : 129/200, recon loss = 0.00143292\n",
            "epoch : 130/200, recon loss = 0.00143398\n",
            "epoch : 131/200, recon loss = 0.00142020\n",
            "epoch : 132/200, recon loss = 0.00141143\n",
            "epoch : 133/200, recon loss = 0.00139989\n",
            "epoch : 134/200, recon loss = 0.00140902\n",
            "epoch : 135/200, recon loss = 0.00140475\n",
            "epoch : 136/200, recon loss = 0.00139361\n",
            "epoch : 137/200, recon loss = 0.00139916\n",
            "epoch : 138/200, recon loss = 0.00140991\n",
            "epoch : 139/200, recon loss = 0.00140580\n",
            "epoch : 140/200, recon loss = 0.00139502\n",
            "epoch : 141/200, recon loss = 0.00138796\n",
            "epoch : 142/200, recon loss = 0.00139541\n",
            "epoch : 143/200, recon loss = 0.00139096\n",
            "epoch : 144/200, recon loss = 0.00138167\n",
            "epoch : 145/200, recon loss = 0.00137559\n",
            "epoch : 146/200, recon loss = 0.00136690\n",
            "epoch : 147/200, recon loss = 0.00138650\n",
            "epoch : 148/200, recon loss = 0.00136625\n",
            "epoch : 149/200, recon loss = 0.00137716\n",
            "epoch : 150/200, recon loss = 0.00136413\n",
            "epoch : 151/200, recon loss = 0.00137139\n",
            "epoch : 152/200, recon loss = 0.00136241\n",
            "epoch : 153/200, recon loss = 0.00136165\n",
            "epoch : 154/200, recon loss = 0.00135901\n",
            "epoch : 155/200, recon loss = 0.00136752\n",
            "epoch : 156/200, recon loss = 0.00134967\n",
            "epoch : 157/200, recon loss = 0.00135764\n",
            "epoch : 158/200, recon loss = 0.00135763\n",
            "epoch : 159/200, recon loss = 0.00135199\n",
            "epoch : 160/200, recon loss = 0.00135732\n",
            "epoch : 161/200, recon loss = 0.00134642\n",
            "epoch : 162/200, recon loss = 0.00134217\n",
            "epoch : 163/200, recon loss = 0.00135949\n",
            "epoch : 164/200, recon loss = 0.00134325\n",
            "epoch : 165/200, recon loss = 0.00135411\n",
            "epoch : 166/200, recon loss = 0.00135250\n",
            "epoch : 167/200, recon loss = 0.00134181\n",
            "epoch : 168/200, recon loss = 0.00135398\n",
            "epoch : 169/200, recon loss = 0.00133306\n",
            "epoch : 170/200, recon loss = 0.00134522\n",
            "epoch : 171/200, recon loss = 0.00134666\n",
            "epoch : 172/200, recon loss = 0.00133627\n",
            "epoch : 173/200, recon loss = 0.00134323\n",
            "epoch : 174/200, recon loss = 0.00133837\n",
            "epoch : 175/200, recon loss = 0.00133440\n",
            "epoch : 176/200, recon loss = 0.00133051\n",
            "epoch : 177/200, recon loss = 0.00134240\n",
            "epoch : 178/200, recon loss = 0.00133451\n",
            "epoch : 179/200, recon loss = 0.00132365\n",
            "epoch : 180/200, recon loss = 0.00133596\n",
            "epoch : 181/200, recon loss = 0.00132577\n",
            "epoch : 182/200, recon loss = 0.00133855\n",
            "epoch : 183/200, recon loss = 0.00132049\n",
            "epoch : 184/200, recon loss = 0.00133629\n",
            "epoch : 185/200, recon loss = 0.00132535\n",
            "epoch : 186/200, recon loss = 0.00133035\n",
            "epoch : 187/200, recon loss = 0.00132734\n",
            "epoch : 188/200, recon loss = 0.00131999\n",
            "epoch : 189/200, recon loss = 0.00130625\n",
            "epoch : 190/200, recon loss = 0.00131549\n",
            "epoch : 191/200, recon loss = 0.00132005\n",
            "epoch : 192/200, recon loss = 0.00130914\n",
            "epoch : 193/200, recon loss = 0.00132515\n",
            "epoch : 194/200, recon loss = 0.00131629\n",
            "epoch : 195/200, recon loss = 0.00130015\n",
            "epoch : 196/200, recon loss = 0.00132615\n",
            "epoch : 197/200, recon loss = 0.00132214\n",
            "epoch : 198/200, recon loss = 0.00130590\n",
            "epoch : 199/200, recon loss = 0.00130339\n",
            "epoch : 200/200, recon loss = 0.00129977\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7NhlSfMO7Q1V",
        "outputId": "8d56f074-724e-4c05-95db-7a881d96c26d"
      },
      "source": [
        "model.eval()\n",
        "rmse_sum = 0\n",
        "\n",
        "filled_data = model(missed_data_noEm.to(device))\n",
        "filled_data = filled_data.cpu().detach().numpy()\n",
        "\n",
        "rmse_sum = rmse_error(test_data, filled_data, cols-num_embeddings, mask)\n",
        "\n",
        "print(rmse_sum)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "4.011076875370579\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q0MsdIyM7Q1V"
      },
      "source": [
        "## Autoencoder - with embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MMsQvYyE7Q1V"
      },
      "source": [
        "model = Autoencoder(dim=cols).to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bRDBJUjI7Q1W"
      },
      "source": [
        "criterion = nn.MSELoss()\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZSnzLSCY7Q1W",
        "outputId": "6f77041d-deb2-42e2-98fa-347ee4dcae2a"
      },
      "source": [
        "training(num_epochs, model, train_loader, criterion, optimizer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch : 1/200, recon loss = 0.01557843\n",
            "epoch : 2/200, recon loss = 0.00973100\n",
            "epoch : 3/200, recon loss = 0.00969905\n",
            "epoch : 4/200, recon loss = 0.00966056\n",
            "epoch : 5/200, recon loss = 0.00960115\n",
            "epoch : 6/200, recon loss = 0.00950456\n",
            "epoch : 7/200, recon loss = 0.00932529\n",
            "epoch : 8/200, recon loss = 0.00900926\n",
            "epoch : 9/200, recon loss = 0.00853238\n",
            "epoch : 10/200, recon loss = 0.00807931\n",
            "epoch : 11/200, recon loss = 0.00779773\n",
            "epoch : 12/200, recon loss = 0.00765350\n",
            "epoch : 13/200, recon loss = 0.00758035\n",
            "epoch : 14/200, recon loss = 0.00753677\n",
            "epoch : 15/200, recon loss = 0.00750988\n",
            "epoch : 16/200, recon loss = 0.00749236\n",
            "epoch : 17/200, recon loss = 0.00746782\n",
            "epoch : 18/200, recon loss = 0.00745426\n",
            "epoch : 19/200, recon loss = 0.00743363\n",
            "epoch : 20/200, recon loss = 0.00741140\n",
            "epoch : 21/200, recon loss = 0.00739428\n",
            "epoch : 22/200, recon loss = 0.00736950\n",
            "epoch : 23/200, recon loss = 0.00733880\n",
            "epoch : 24/200, recon loss = 0.00731051\n",
            "epoch : 25/200, recon loss = 0.00726884\n",
            "epoch : 26/200, recon loss = 0.00723411\n",
            "epoch : 27/200, recon loss = 0.00718327\n",
            "epoch : 28/200, recon loss = 0.00713830\n",
            "epoch : 29/200, recon loss = 0.00708942\n",
            "epoch : 30/200, recon loss = 0.00703881\n",
            "epoch : 31/200, recon loss = 0.00698729\n",
            "epoch : 32/200, recon loss = 0.00695349\n",
            "epoch : 33/200, recon loss = 0.00691012\n",
            "epoch : 34/200, recon loss = 0.00686962\n",
            "epoch : 35/200, recon loss = 0.00683114\n",
            "epoch : 36/200, recon loss = 0.00678989\n",
            "epoch : 37/200, recon loss = 0.00676414\n",
            "epoch : 38/200, recon loss = 0.00672439\n",
            "epoch : 39/200, recon loss = 0.00671076\n",
            "epoch : 40/200, recon loss = 0.00667860\n",
            "epoch : 41/200, recon loss = 0.00665087\n",
            "epoch : 42/200, recon loss = 0.00661973\n",
            "epoch : 43/200, recon loss = 0.00660596\n",
            "epoch : 44/200, recon loss = 0.00656275\n",
            "epoch : 45/200, recon loss = 0.00652542\n",
            "epoch : 46/200, recon loss = 0.00649792\n",
            "epoch : 47/200, recon loss = 0.00647319\n",
            "epoch : 48/200, recon loss = 0.00643694\n",
            "epoch : 49/200, recon loss = 0.00641074\n",
            "epoch : 50/200, recon loss = 0.00637118\n",
            "epoch : 51/200, recon loss = 0.00634761\n",
            "epoch : 52/200, recon loss = 0.00631277\n",
            "epoch : 53/200, recon loss = 0.00630631\n",
            "epoch : 54/200, recon loss = 0.00627188\n",
            "epoch : 55/200, recon loss = 0.00624074\n",
            "epoch : 56/200, recon loss = 0.00622542\n",
            "epoch : 57/200, recon loss = 0.00617215\n",
            "epoch : 58/200, recon loss = 0.00617465\n",
            "epoch : 59/200, recon loss = 0.00612736\n",
            "epoch : 60/200, recon loss = 0.00611366\n",
            "epoch : 61/200, recon loss = 0.00608916\n",
            "epoch : 62/200, recon loss = 0.00606933\n",
            "epoch : 63/200, recon loss = 0.00605500\n",
            "epoch : 64/200, recon loss = 0.00601967\n",
            "epoch : 65/200, recon loss = 0.00599360\n",
            "epoch : 66/200, recon loss = 0.00597538\n",
            "epoch : 67/200, recon loss = 0.00598871\n",
            "epoch : 68/200, recon loss = 0.00595796\n",
            "epoch : 69/200, recon loss = 0.00593215\n",
            "epoch : 70/200, recon loss = 0.00591018\n",
            "epoch : 71/200, recon loss = 0.00592285\n",
            "epoch : 72/200, recon loss = 0.00589800\n",
            "epoch : 73/200, recon loss = 0.00588372\n",
            "epoch : 74/200, recon loss = 0.00588508\n",
            "epoch : 75/200, recon loss = 0.00587045\n",
            "epoch : 76/200, recon loss = 0.00584459\n",
            "epoch : 77/200, recon loss = 0.00583642\n",
            "epoch : 78/200, recon loss = 0.00582807\n",
            "epoch : 79/200, recon loss = 0.00581826\n",
            "epoch : 80/200, recon loss = 0.00579415\n",
            "epoch : 81/200, recon loss = 0.00579680\n",
            "epoch : 82/200, recon loss = 0.00578280\n",
            "epoch : 83/200, recon loss = 0.00575891\n",
            "epoch : 84/200, recon loss = 0.00575432\n",
            "epoch : 85/200, recon loss = 0.00576907\n",
            "epoch : 86/200, recon loss = 0.00575282\n",
            "epoch : 87/200, recon loss = 0.00572981\n",
            "epoch : 88/200, recon loss = 0.00573571\n",
            "epoch : 89/200, recon loss = 0.00573571\n",
            "epoch : 90/200, recon loss = 0.00570834\n",
            "epoch : 91/200, recon loss = 0.00569397\n",
            "epoch : 92/200, recon loss = 0.00568324\n",
            "epoch : 93/200, recon loss = 0.00567470\n",
            "epoch : 94/200, recon loss = 0.00566913\n",
            "epoch : 95/200, recon loss = 0.00567494\n",
            "epoch : 96/200, recon loss = 0.00565850\n",
            "epoch : 97/200, recon loss = 0.00566198\n",
            "epoch : 98/200, recon loss = 0.00564863\n",
            "epoch : 99/200, recon loss = 0.00560954\n",
            "epoch : 100/200, recon loss = 0.00562683\n",
            "epoch : 101/200, recon loss = 0.00562335\n",
            "epoch : 102/200, recon loss = 0.00558997\n",
            "epoch : 103/200, recon loss = 0.00557733\n",
            "epoch : 104/200, recon loss = 0.00559805\n",
            "epoch : 105/200, recon loss = 0.00556811\n",
            "epoch : 106/200, recon loss = 0.00557677\n",
            "epoch : 107/200, recon loss = 0.00556151\n",
            "epoch : 108/200, recon loss = 0.00556119\n",
            "epoch : 109/200, recon loss = 0.00553979\n",
            "epoch : 110/200, recon loss = 0.00552588\n",
            "epoch : 111/200, recon loss = 0.00555133\n",
            "epoch : 112/200, recon loss = 0.00552397\n",
            "epoch : 113/200, recon loss = 0.00550333\n",
            "epoch : 114/200, recon loss = 0.00549433\n",
            "epoch : 115/200, recon loss = 0.00552006\n",
            "epoch : 116/200, recon loss = 0.00547262\n",
            "epoch : 117/200, recon loss = 0.00547992\n",
            "epoch : 118/200, recon loss = 0.00549022\n",
            "epoch : 119/200, recon loss = 0.00546815\n",
            "epoch : 120/200, recon loss = 0.00545649\n",
            "epoch : 121/200, recon loss = 0.00544193\n",
            "epoch : 122/200, recon loss = 0.00543152\n",
            "epoch : 123/200, recon loss = 0.00541485\n",
            "epoch : 124/200, recon loss = 0.00543605\n",
            "epoch : 125/200, recon loss = 0.00541932\n",
            "epoch : 126/200, recon loss = 0.00540030\n",
            "epoch : 127/200, recon loss = 0.00538989\n",
            "epoch : 128/200, recon loss = 0.00539237\n",
            "epoch : 129/200, recon loss = 0.00536578\n",
            "epoch : 130/200, recon loss = 0.00540962\n",
            "epoch : 131/200, recon loss = 0.00536861\n",
            "epoch : 132/200, recon loss = 0.00535153\n",
            "epoch : 133/200, recon loss = 0.00533832\n",
            "epoch : 134/200, recon loss = 0.00536408\n",
            "epoch : 135/200, recon loss = 0.00536568\n",
            "epoch : 136/200, recon loss = 0.00533382\n",
            "epoch : 137/200, recon loss = 0.00533624\n",
            "epoch : 138/200, recon loss = 0.00530730\n",
            "epoch : 139/200, recon loss = 0.00532927\n",
            "epoch : 140/200, recon loss = 0.00529366\n",
            "epoch : 141/200, recon loss = 0.00529187\n",
            "epoch : 142/200, recon loss = 0.00527716\n",
            "epoch : 143/200, recon loss = 0.00532387\n",
            "epoch : 144/200, recon loss = 0.00526284\n",
            "epoch : 145/200, recon loss = 0.00526937\n",
            "epoch : 146/200, recon loss = 0.00527969\n",
            "epoch : 147/200, recon loss = 0.00525020\n",
            "epoch : 148/200, recon loss = 0.00524428\n",
            "epoch : 149/200, recon loss = 0.00524097\n",
            "epoch : 150/200, recon loss = 0.00525627\n",
            "epoch : 151/200, recon loss = 0.00523650\n",
            "epoch : 152/200, recon loss = 0.00522249\n",
            "epoch : 153/200, recon loss = 0.00523002\n",
            "epoch : 154/200, recon loss = 0.00523287\n",
            "epoch : 155/200, recon loss = 0.00519970\n",
            "epoch : 156/200, recon loss = 0.00519689\n",
            "epoch : 157/200, recon loss = 0.00520558\n",
            "epoch : 158/200, recon loss = 0.00520600\n",
            "epoch : 159/200, recon loss = 0.00515854\n",
            "epoch : 160/200, recon loss = 0.00516483\n",
            "epoch : 161/200, recon loss = 0.00516659\n",
            "epoch : 162/200, recon loss = 0.00515686\n",
            "epoch : 163/200, recon loss = 0.00517842\n",
            "epoch : 164/200, recon loss = 0.00515908\n",
            "epoch : 165/200, recon loss = 0.00513053\n",
            "epoch : 166/200, recon loss = 0.00516997\n",
            "epoch : 167/200, recon loss = 0.00511828\n",
            "epoch : 168/200, recon loss = 0.00514299\n",
            "epoch : 169/200, recon loss = 0.00513920\n",
            "epoch : 170/200, recon loss = 0.00510410\n",
            "epoch : 171/200, recon loss = 0.00512746\n",
            "epoch : 172/200, recon loss = 0.00508489\n",
            "epoch : 173/200, recon loss = 0.00509341\n",
            "epoch : 174/200, recon loss = 0.00509843\n",
            "epoch : 175/200, recon loss = 0.00509557\n",
            "epoch : 176/200, recon loss = 0.00506269\n",
            "epoch : 177/200, recon loss = 0.00507915\n",
            "epoch : 178/200, recon loss = 0.00506043\n",
            "epoch : 179/200, recon loss = 0.00506352\n",
            "epoch : 180/200, recon loss = 0.00507773\n",
            "epoch : 181/200, recon loss = 0.00504253\n",
            "epoch : 182/200, recon loss = 0.00505526\n",
            "epoch : 183/200, recon loss = 0.00504745\n",
            "epoch : 184/200, recon loss = 0.00501873\n",
            "epoch : 185/200, recon loss = 0.00503982\n",
            "epoch : 186/200, recon loss = 0.00503878\n",
            "epoch : 187/200, recon loss = 0.00504300\n",
            "epoch : 188/200, recon loss = 0.00501691\n",
            "epoch : 189/200, recon loss = 0.00502912\n",
            "epoch : 190/200, recon loss = 0.00502388\n",
            "epoch : 191/200, recon loss = 0.00503161\n",
            "epoch : 192/200, recon loss = 0.00499823\n",
            "epoch : 193/200, recon loss = 0.00501385\n",
            "epoch : 194/200, recon loss = 0.00498510\n",
            "epoch : 195/200, recon loss = 0.00498602\n",
            "epoch : 196/200, recon loss = 0.00499118\n",
            "epoch : 197/200, recon loss = 0.00497259\n",
            "epoch : 198/200, recon loss = 0.00497185\n",
            "epoch : 199/200, recon loss = 0.00497884\n",
            "epoch : 200/200, recon loss = 0.00499796\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qxsjzAfP7Q1W",
        "outputId": "cc816174-511e-4530-8e44-7a722188bf3a"
      },
      "source": [
        "model.eval()\n",
        "rmse_sum = 0\n",
        "\n",
        "filled_data = model(missed_data.to(device))\n",
        "filled_data = filled_data.cpu().detach().numpy()\n",
        "\n",
        "rmse_sum = rmse_error(test_data, filled_data, cols, mask)\n",
        "\n",
        "print(rmse_sum)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "4.268289142753645\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Eg9hbUf37Q1W"
      },
      "source": [
        "## Mean and Median Imputation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SCagvv-K7Q1X"
      },
      "source": [
        "### By category"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ib79vSyI7Q1X",
        "outputId": "e7f07546-f087-428d-83b7-213807a42371"
      },
      "source": [
        "imputed = X_test_mean\n",
        "error = rmse_error(y_test, imputed, cols-num_embeddings, mask)\n",
        "print(error)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "5.519246675443418\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AJJtXeN77Q1X",
        "outputId": "99129f86-c99c-49e2-d137-9aaaaab246e4"
      },
      "source": [
        "imputed = X_test_median\n",
        "error = rmse_error(y_test, imputed, cols-num_embeddings, mask)\n",
        "print(error)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "5.730640468576924\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YWK4JQ8K7Q1X"
      },
      "source": [
        "### Across all foods"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yey_K6Rh7Q1X",
        "outputId": "e306a10d-5177-4d1d-b2e4-3695ec978ba5"
      },
      "source": [
        "imputed = X_test_mean_all\n",
        "error = rmse_error(y_test, imputed, cols-num_embeddings, mask)\n",
        "print(error)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "6.3815988582216585\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5U3fY4E-7Q1Y",
        "outputId": "ce700a99-2591-40f1-db1d-c0a8d291ff5f"
      },
      "source": [
        "imputed = X_test_median_all\n",
        "error = rmse_error(y_test, imputed, cols-num_embeddings, mask)\n",
        "print(error)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "6.5167976604056905\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q9fGUIfS7Q1Y"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bb_4TRhbGFBF"
      },
      "source": [
        "# 198 Columns"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3T0Wv_C-GFBG"
      },
      "source": [
        "data_path = 'datasets/average datasets/byColumnsWithEmbeddings/198_columns_embeddings.csv'\n",
        "num_embeddings = 50\n",
        "num_columns = 198"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 350
        },
        "id": "NHTZk16DGFBH",
        "outputId": "e6f9c516-47c9-4517-fbd2-3bff5b8ef43e"
      },
      "source": [
        "dataset = pd.read_csv(data_path, index_col=0)\n",
        "dataset[dataset.notna().all(axis=1)].describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>fdc_id</th>\n",
              "      <th>1002</th>\n",
              "      <th>1003</th>\n",
              "      <th>1004</th>\n",
              "      <th>1005</th>\n",
              "      <th>1007</th>\n",
              "      <th>1008</th>\n",
              "      <th>1009</th>\n",
              "      <th>1010</th>\n",
              "      <th>1011</th>\n",
              "      <th>1012</th>\n",
              "      <th>1013</th>\n",
              "      <th>1014</th>\n",
              "      <th>1024</th>\n",
              "      <th>1032</th>\n",
              "      <th>1039</th>\n",
              "      <th>1050</th>\n",
              "      <th>1051</th>\n",
              "      <th>1062</th>\n",
              "      <th>1063</th>\n",
              "      <th>1075</th>\n",
              "      <th>1079</th>\n",
              "      <th>1082</th>\n",
              "      <th>1084</th>\n",
              "      <th>1085</th>\n",
              "      <th>1087</th>\n",
              "      <th>1089</th>\n",
              "      <th>1090</th>\n",
              "      <th>1091</th>\n",
              "      <th>1092</th>\n",
              "      <th>1093</th>\n",
              "      <th>1094</th>\n",
              "      <th>1095</th>\n",
              "      <th>1097</th>\n",
              "      <th>1098</th>\n",
              "      <th>1100</th>\n",
              "      <th>1101</th>\n",
              "      <th>1102</th>\n",
              "      <th>1103</th>\n",
              "      <th>1105</th>\n",
              "      <th>...</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "      <th>30</th>\n",
              "      <th>31</th>\n",
              "      <th>32</th>\n",
              "      <th>33</th>\n",
              "      <th>34</th>\n",
              "      <th>35</th>\n",
              "      <th>36</th>\n",
              "      <th>37</th>\n",
              "      <th>38</th>\n",
              "      <th>39</th>\n",
              "      <th>40</th>\n",
              "      <th>41</th>\n",
              "      <th>42</th>\n",
              "      <th>43</th>\n",
              "      <th>44</th>\n",
              "      <th>45</th>\n",
              "      <th>46</th>\n",
              "      <th>47</th>\n",
              "      <th>48</th>\n",
              "      <th>49</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>1.310500e+04</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "      <td>13105.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>4.807517e+05</td>\n",
              "      <td>0.243665</td>\n",
              "      <td>1.942243</td>\n",
              "      <td>1.824697</td>\n",
              "      <td>0.164999</td>\n",
              "      <td>0.472742</td>\n",
              "      <td>1.999714</td>\n",
              "      <td>2.561062</td>\n",
              "      <td>0.147034</td>\n",
              "      <td>0.118187</td>\n",
              "      <td>0.120046</td>\n",
              "      <td>0.046356</td>\n",
              "      <td>0.008754</td>\n",
              "      <td>0.000476</td>\n",
              "      <td>0.681114</td>\n",
              "      <td>0.728577</td>\n",
              "      <td>0.049253</td>\n",
              "      <td>8.349769</td>\n",
              "      <td>8.369340</td>\n",
              "      <td>0.055411</td>\n",
              "      <td>0.002459</td>\n",
              "      <td>0.350355</td>\n",
              "      <td>0.005555</td>\n",
              "      <td>0.021892</td>\n",
              "      <td>0.106772</td>\n",
              "      <td>27.659061</td>\n",
              "      <td>0.476038</td>\n",
              "      <td>14.595597</td>\n",
              "      <td>55.746662</td>\n",
              "      <td>125.003892</td>\n",
              "      <td>133.394887</td>\n",
              "      <td>14.730179</td>\n",
              "      <td>0.384163</td>\n",
              "      <td>2.983579</td>\n",
              "      <td>0.077504</td>\n",
              "      <td>12.187135</td>\n",
              "      <td>0.149979</td>\n",
              "      <td>42.944411</td>\n",
              "      <td>0.840258</td>\n",
              "      <td>2.086684</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.005261</td>\n",
              "      <td>-0.053342</td>\n",
              "      <td>0.032862</td>\n",
              "      <td>0.017149</td>\n",
              "      <td>0.053914</td>\n",
              "      <td>-0.050015</td>\n",
              "      <td>-0.006802</td>\n",
              "      <td>-0.001312</td>\n",
              "      <td>0.045397</td>\n",
              "      <td>-0.002815</td>\n",
              "      <td>-0.004867</td>\n",
              "      <td>0.015156</td>\n",
              "      <td>-0.002852</td>\n",
              "      <td>-0.007990</td>\n",
              "      <td>0.007587</td>\n",
              "      <td>-0.005105</td>\n",
              "      <td>0.013259</td>\n",
              "      <td>0.026135</td>\n",
              "      <td>-0.009025</td>\n",
              "      <td>-0.004237</td>\n",
              "      <td>-0.008990</td>\n",
              "      <td>-0.000290</td>\n",
              "      <td>0.009895</td>\n",
              "      <td>-0.005317</td>\n",
              "      <td>0.001269</td>\n",
              "      <td>0.006334</td>\n",
              "      <td>0.018435</td>\n",
              "      <td>0.004424</td>\n",
              "      <td>0.009392</td>\n",
              "      <td>-0.003170</td>\n",
              "      <td>-0.001646</td>\n",
              "      <td>-0.006830</td>\n",
              "      <td>0.003214</td>\n",
              "      <td>-0.000149</td>\n",
              "      <td>0.004693</td>\n",
              "      <td>-0.004719</td>\n",
              "      <td>0.000873</td>\n",
              "      <td>-0.001455</td>\n",
              "      <td>0.006576</td>\n",
              "      <td>0.003596</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.544018e+05</td>\n",
              "      <td>0.999601</td>\n",
              "      <td>6.583455</td>\n",
              "      <td>7.838466</td>\n",
              "      <td>2.905073</td>\n",
              "      <td>4.416751</td>\n",
              "      <td>25.783702</td>\n",
              "      <td>9.446549</td>\n",
              "      <td>2.828219</td>\n",
              "      <td>1.055735</td>\n",
              "      <td>1.016245</td>\n",
              "      <td>0.441966</td>\n",
              "      <td>0.129818</td>\n",
              "      <td>0.022249</td>\n",
              "      <td>14.844129</td>\n",
              "      <td>15.911796</td>\n",
              "      <td>1.303063</td>\n",
              "      <td>23.549114</td>\n",
              "      <td>107.911712</td>\n",
              "      <td>1.491246</td>\n",
              "      <td>0.034625</td>\n",
              "      <td>1.186155</td>\n",
              "      <td>0.082114</td>\n",
              "      <td>0.319893</td>\n",
              "      <td>2.306569</td>\n",
              "      <td>99.924214</td>\n",
              "      <td>1.483360</td>\n",
              "      <td>45.754175</td>\n",
              "      <td>153.432660</td>\n",
              "      <td>382.322293</td>\n",
              "      <td>1736.846193</td>\n",
              "      <td>56.459805</td>\n",
              "      <td>1.055340</td>\n",
              "      <td>11.848737</td>\n",
              "      <td>0.264892</td>\n",
              "      <td>237.511065</td>\n",
              "      <td>0.515558</td>\n",
              "      <td>187.200801</td>\n",
              "      <td>5.232485</td>\n",
              "      <td>29.375360</td>\n",
              "      <td>...</td>\n",
              "      <td>0.400175</td>\n",
              "      <td>0.345475</td>\n",
              "      <td>0.349559</td>\n",
              "      <td>0.332054</td>\n",
              "      <td>0.321282</td>\n",
              "      <td>0.292924</td>\n",
              "      <td>0.270047</td>\n",
              "      <td>0.284935</td>\n",
              "      <td>0.261685</td>\n",
              "      <td>0.269203</td>\n",
              "      <td>0.240661</td>\n",
              "      <td>0.243893</td>\n",
              "      <td>0.245143</td>\n",
              "      <td>0.248005</td>\n",
              "      <td>0.235594</td>\n",
              "      <td>0.205819</td>\n",
              "      <td>0.225068</td>\n",
              "      <td>0.212354</td>\n",
              "      <td>0.209393</td>\n",
              "      <td>0.205605</td>\n",
              "      <td>0.208641</td>\n",
              "      <td>0.187673</td>\n",
              "      <td>0.193694</td>\n",
              "      <td>0.181126</td>\n",
              "      <td>0.188895</td>\n",
              "      <td>0.180591</td>\n",
              "      <td>0.172620</td>\n",
              "      <td>0.162121</td>\n",
              "      <td>0.180927</td>\n",
              "      <td>0.171087</td>\n",
              "      <td>0.170579</td>\n",
              "      <td>0.162898</td>\n",
              "      <td>0.166174</td>\n",
              "      <td>0.156805</td>\n",
              "      <td>0.165867</td>\n",
              "      <td>0.149952</td>\n",
              "      <td>0.151873</td>\n",
              "      <td>0.145215</td>\n",
              "      <td>0.151300</td>\n",
              "      <td>0.148431</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>3.198770e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.933855</td>\n",
              "      <td>-0.969972</td>\n",
              "      <td>-1.097137</td>\n",
              "      <td>-0.920027</td>\n",
              "      <td>-1.006138</td>\n",
              "      <td>-1.000187</td>\n",
              "      <td>-0.798815</td>\n",
              "      <td>-0.687112</td>\n",
              "      <td>-0.668615</td>\n",
              "      <td>-0.801637</td>\n",
              "      <td>-0.726105</td>\n",
              "      <td>-0.698135</td>\n",
              "      <td>-0.734302</td>\n",
              "      <td>-0.848955</td>\n",
              "      <td>-0.669846</td>\n",
              "      <td>-0.604000</td>\n",
              "      <td>-0.624276</td>\n",
              "      <td>-0.615452</td>\n",
              "      <td>-0.597283</td>\n",
              "      <td>-0.806154</td>\n",
              "      <td>-0.673437</td>\n",
              "      <td>-0.584369</td>\n",
              "      <td>-0.800978</td>\n",
              "      <td>-0.701795</td>\n",
              "      <td>-0.663740</td>\n",
              "      <td>-0.697520</td>\n",
              "      <td>-0.542536</td>\n",
              "      <td>-0.560396</td>\n",
              "      <td>-0.753790</td>\n",
              "      <td>-0.567381</td>\n",
              "      <td>-0.571239</td>\n",
              "      <td>-0.533034</td>\n",
              "      <td>-0.571749</td>\n",
              "      <td>-0.680382</td>\n",
              "      <td>-0.648647</td>\n",
              "      <td>-0.538644</td>\n",
              "      <td>-0.550678</td>\n",
              "      <td>-0.580334</td>\n",
              "      <td>-0.568859</td>\n",
              "      <td>-0.518081</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>3.256480e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.303209</td>\n",
              "      <td>-0.332181</td>\n",
              "      <td>-0.196565</td>\n",
              "      <td>-0.219470</td>\n",
              "      <td>-0.160205</td>\n",
              "      <td>-0.217190</td>\n",
              "      <td>-0.194588</td>\n",
              "      <td>-0.175356</td>\n",
              "      <td>-0.131521</td>\n",
              "      <td>-0.159043</td>\n",
              "      <td>-0.129569</td>\n",
              "      <td>-0.142713</td>\n",
              "      <td>-0.128844</td>\n",
              "      <td>-0.176975</td>\n",
              "      <td>-0.164039</td>\n",
              "      <td>-0.152442</td>\n",
              "      <td>-0.136194</td>\n",
              "      <td>-0.113140</td>\n",
              "      <td>-0.133148</td>\n",
              "      <td>-0.100405</td>\n",
              "      <td>-0.112391</td>\n",
              "      <td>-0.121311</td>\n",
              "      <td>-0.102899</td>\n",
              "      <td>-0.127997</td>\n",
              "      <td>-0.112148</td>\n",
              "      <td>-0.103513</td>\n",
              "      <td>-0.087577</td>\n",
              "      <td>-0.118496</td>\n",
              "      <td>-0.090855</td>\n",
              "      <td>-0.124071</td>\n",
              "      <td>-0.098138</td>\n",
              "      <td>-0.108403</td>\n",
              "      <td>-0.089859</td>\n",
              "      <td>-0.061942</td>\n",
              "      <td>-0.089926</td>\n",
              "      <td>-0.083567</td>\n",
              "      <td>-0.099206</td>\n",
              "      <td>-0.084546</td>\n",
              "      <td>-0.092055</td>\n",
              "      <td>-0.094561</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>3.314100e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.006464</td>\n",
              "      <td>0.045016</td>\n",
              "      <td>0.064438</td>\n",
              "      <td>-0.007061</td>\n",
              "      <td>0.051118</td>\n",
              "      <td>-0.054914</td>\n",
              "      <td>-0.020087</td>\n",
              "      <td>-0.030643</td>\n",
              "      <td>0.015376</td>\n",
              "      <td>-0.034391</td>\n",
              "      <td>-0.022269</td>\n",
              "      <td>0.000457</td>\n",
              "      <td>-0.022652</td>\n",
              "      <td>0.013189</td>\n",
              "      <td>-0.022747</td>\n",
              "      <td>-0.014965</td>\n",
              "      <td>0.015073</td>\n",
              "      <td>0.016768</td>\n",
              "      <td>-0.044640</td>\n",
              "      <td>-0.002055</td>\n",
              "      <td>-0.023092</td>\n",
              "      <td>-0.004990</td>\n",
              "      <td>0.001245</td>\n",
              "      <td>-0.016104</td>\n",
              "      <td>0.015313</td>\n",
              "      <td>0.010249</td>\n",
              "      <td>0.006607</td>\n",
              "      <td>0.005072</td>\n",
              "      <td>0.007793</td>\n",
              "      <td>-0.013320</td>\n",
              "      <td>-0.003852</td>\n",
              "      <td>-0.013888</td>\n",
              "      <td>0.001045</td>\n",
              "      <td>-0.004789</td>\n",
              "      <td>0.016949</td>\n",
              "      <td>-0.004967</td>\n",
              "      <td>0.005749</td>\n",
              "      <td>-0.009788</td>\n",
              "      <td>-0.003402</td>\n",
              "      <td>0.014599</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>7.474680e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.267125</td>\n",
              "      <td>0.166323</td>\n",
              "      <td>0.247990</td>\n",
              "      <td>0.188549</td>\n",
              "      <td>0.251014</td>\n",
              "      <td>0.109046</td>\n",
              "      <td>0.190527</td>\n",
              "      <td>0.178192</td>\n",
              "      <td>0.198413</td>\n",
              "      <td>0.177397</td>\n",
              "      <td>0.099106</td>\n",
              "      <td>0.180667</td>\n",
              "      <td>0.128173</td>\n",
              "      <td>0.157417</td>\n",
              "      <td>0.136158</td>\n",
              "      <td>0.119462</td>\n",
              "      <td>0.155412</td>\n",
              "      <td>0.157847</td>\n",
              "      <td>0.104137</td>\n",
              "      <td>0.111113</td>\n",
              "      <td>0.074621</td>\n",
              "      <td>0.119803</td>\n",
              "      <td>0.117055</td>\n",
              "      <td>0.095796</td>\n",
              "      <td>0.125334</td>\n",
              "      <td>0.100478</td>\n",
              "      <td>0.106755</td>\n",
              "      <td>0.110569</td>\n",
              "      <td>0.081504</td>\n",
              "      <td>0.121099</td>\n",
              "      <td>0.097413</td>\n",
              "      <td>0.104568</td>\n",
              "      <td>0.101486</td>\n",
              "      <td>0.103675</td>\n",
              "      <td>0.091954</td>\n",
              "      <td>0.081006</td>\n",
              "      <td>0.087104</td>\n",
              "      <td>0.080598</td>\n",
              "      <td>0.092365</td>\n",
              "      <td>0.091704</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.105897e+06</td>\n",
              "      <td>13.200000</td>\n",
              "      <td>79.900000</td>\n",
              "      <td>99.400000</td>\n",
              "      <td>99.600000</td>\n",
              "      <td>99.600000</td>\n",
              "      <td>833.000000</td>\n",
              "      <td>43.900000</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>25.900000</td>\n",
              "      <td>23.600000</td>\n",
              "      <td>6.340000</td>\n",
              "      <td>3.830000</td>\n",
              "      <td>1.040000</td>\n",
              "      <td>377.000000</td>\n",
              "      <td>429.000000</td>\n",
              "      <td>67.900000</td>\n",
              "      <td>96.700000</td>\n",
              "      <td>3480.000000</td>\n",
              "      <td>99.800000</td>\n",
              "      <td>1.100000</td>\n",
              "      <td>11.500000</td>\n",
              "      <td>2.900000</td>\n",
              "      <td>9.200000</td>\n",
              "      <td>94.600000</td>\n",
              "      <td>1390.000000</td>\n",
              "      <td>12.700000</td>\n",
              "      <td>386.000000</td>\n",
              "      <td>997.000000</td>\n",
              "      <td>2520.000000</td>\n",
              "      <td>40700.000000</td>\n",
              "      <td>311.000000</td>\n",
              "      <td>7.660000</td>\n",
              "      <td>93.000000</td>\n",
              "      <td>1.920000</td>\n",
              "      <td>7430.000000</td>\n",
              "      <td>7.870000</td>\n",
              "      <td>1790.000000</td>\n",
              "      <td>97.700000</td>\n",
              "      <td>912.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>1.122911</td>\n",
              "      <td>1.235694</td>\n",
              "      <td>1.230920</td>\n",
              "      <td>1.301511</td>\n",
              "      <td>1.225611</td>\n",
              "      <td>1.031472</td>\n",
              "      <td>0.770720</td>\n",
              "      <td>0.843955</td>\n",
              "      <td>1.049345</td>\n",
              "      <td>0.867486</td>\n",
              "      <td>1.478752</td>\n",
              "      <td>0.921318</td>\n",
              "      <td>0.756673</td>\n",
              "      <td>0.907832</td>\n",
              "      <td>0.918719</td>\n",
              "      <td>0.970210</td>\n",
              "      <td>0.778253</td>\n",
              "      <td>0.770962</td>\n",
              "      <td>0.961512</td>\n",
              "      <td>0.903499</td>\n",
              "      <td>0.642813</td>\n",
              "      <td>0.903780</td>\n",
              "      <td>0.722286</td>\n",
              "      <td>0.874324</td>\n",
              "      <td>0.835497</td>\n",
              "      <td>0.680991</td>\n",
              "      <td>0.846139</td>\n",
              "      <td>0.705718</td>\n",
              "      <td>0.985508</td>\n",
              "      <td>0.545425</td>\n",
              "      <td>0.757822</td>\n",
              "      <td>0.626882</td>\n",
              "      <td>0.575618</td>\n",
              "      <td>0.746988</td>\n",
              "      <td>0.949548</td>\n",
              "      <td>0.502315</td>\n",
              "      <td>0.644586</td>\n",
              "      <td>0.563422</td>\n",
              "      <td>0.705493</td>\n",
              "      <td>0.757814</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8 rows × 249 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "             fdc_id          1002  ...            48            49\n",
              "count  1.310500e+04  13105.000000  ...  13105.000000  13105.000000\n",
              "mean   4.807517e+05      0.243665  ...      0.006576      0.003596\n",
              "std    2.544018e+05      0.999601  ...      0.151300      0.148431\n",
              "min    3.198770e+05      0.000000  ...     -0.568859     -0.518081\n",
              "25%    3.256480e+05      0.000000  ...     -0.092055     -0.094561\n",
              "50%    3.314100e+05      0.000000  ...     -0.003402      0.014599\n",
              "75%    7.474680e+05      0.000000  ...      0.092365      0.091704\n",
              "max    1.105897e+06     13.200000  ...      0.705493      0.757814\n",
              "\n",
              "[8 rows x 249 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FjoWRaF3GFBI"
      },
      "source": [
        "data = dataset\n",
        "train_data, test_data = train_test_split(data, test_size=test_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 350
        },
        "id": "Qx-ScFf9GFBI",
        "outputId": "872ca858-438b-41cc-c9ab-7a9617ed6e72"
      },
      "source": [
        "train_data.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>fdc_id</th>\n",
              "      <th>1002</th>\n",
              "      <th>1003</th>\n",
              "      <th>1004</th>\n",
              "      <th>1005</th>\n",
              "      <th>1007</th>\n",
              "      <th>1008</th>\n",
              "      <th>1009</th>\n",
              "      <th>1010</th>\n",
              "      <th>1011</th>\n",
              "      <th>1012</th>\n",
              "      <th>1013</th>\n",
              "      <th>1014</th>\n",
              "      <th>1024</th>\n",
              "      <th>1032</th>\n",
              "      <th>1039</th>\n",
              "      <th>1050</th>\n",
              "      <th>1051</th>\n",
              "      <th>1062</th>\n",
              "      <th>1063</th>\n",
              "      <th>1075</th>\n",
              "      <th>1079</th>\n",
              "      <th>1082</th>\n",
              "      <th>1084</th>\n",
              "      <th>1085</th>\n",
              "      <th>1087</th>\n",
              "      <th>1089</th>\n",
              "      <th>1090</th>\n",
              "      <th>1091</th>\n",
              "      <th>1092</th>\n",
              "      <th>1093</th>\n",
              "      <th>1094</th>\n",
              "      <th>1095</th>\n",
              "      <th>1097</th>\n",
              "      <th>1098</th>\n",
              "      <th>1100</th>\n",
              "      <th>1101</th>\n",
              "      <th>1102</th>\n",
              "      <th>1103</th>\n",
              "      <th>1105</th>\n",
              "      <th>...</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "      <th>30</th>\n",
              "      <th>31</th>\n",
              "      <th>32</th>\n",
              "      <th>33</th>\n",
              "      <th>34</th>\n",
              "      <th>35</th>\n",
              "      <th>36</th>\n",
              "      <th>37</th>\n",
              "      <th>38</th>\n",
              "      <th>39</th>\n",
              "      <th>40</th>\n",
              "      <th>41</th>\n",
              "      <th>42</th>\n",
              "      <th>43</th>\n",
              "      <th>44</th>\n",
              "      <th>45</th>\n",
              "      <th>46</th>\n",
              "      <th>47</th>\n",
              "      <th>48</th>\n",
              "      <th>49</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>1.048400e+04</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "      <td>10484.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>4.794562e+05</td>\n",
              "      <td>0.241070</td>\n",
              "      <td>1.928870</td>\n",
              "      <td>1.821707</td>\n",
              "      <td>0.161388</td>\n",
              "      <td>0.461399</td>\n",
              "      <td>2.026159</td>\n",
              "      <td>2.518903</td>\n",
              "      <td>0.137158</td>\n",
              "      <td>0.118248</td>\n",
              "      <td>0.119258</td>\n",
              "      <td>0.046907</td>\n",
              "      <td>0.009217</td>\n",
              "      <td>0.000595</td>\n",
              "      <td>0.679321</td>\n",
              "      <td>0.732259</td>\n",
              "      <td>0.045408</td>\n",
              "      <td>8.340591</td>\n",
              "      <td>8.480275</td>\n",
              "      <td>0.046689</td>\n",
              "      <td>0.002474</td>\n",
              "      <td>0.344878</td>\n",
              "      <td>0.005790</td>\n",
              "      <td>0.022530</td>\n",
              "      <td>0.109062</td>\n",
              "      <td>27.787963</td>\n",
              "      <td>0.470211</td>\n",
              "      <td>14.459805</td>\n",
              "      <td>55.374952</td>\n",
              "      <td>123.405857</td>\n",
              "      <td>120.136112</td>\n",
              "      <td>14.461274</td>\n",
              "      <td>0.381541</td>\n",
              "      <td>2.938544</td>\n",
              "      <td>0.076159</td>\n",
              "      <td>12.661293</td>\n",
              "      <td>0.148089</td>\n",
              "      <td>42.606524</td>\n",
              "      <td>0.864411</td>\n",
              "      <td>2.053987</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.004485</td>\n",
              "      <td>-0.053181</td>\n",
              "      <td>0.035190</td>\n",
              "      <td>0.016537</td>\n",
              "      <td>0.055051</td>\n",
              "      <td>-0.049332</td>\n",
              "      <td>-0.005589</td>\n",
              "      <td>-0.001322</td>\n",
              "      <td>0.047027</td>\n",
              "      <td>-0.003175</td>\n",
              "      <td>-0.003616</td>\n",
              "      <td>0.015436</td>\n",
              "      <td>-0.002492</td>\n",
              "      <td>-0.006390</td>\n",
              "      <td>0.007534</td>\n",
              "      <td>-0.006210</td>\n",
              "      <td>0.014451</td>\n",
              "      <td>0.026644</td>\n",
              "      <td>-0.009088</td>\n",
              "      <td>-0.003639</td>\n",
              "      <td>-0.008473</td>\n",
              "      <td>0.000794</td>\n",
              "      <td>0.010315</td>\n",
              "      <td>-0.003527</td>\n",
              "      <td>0.001452</td>\n",
              "      <td>0.006197</td>\n",
              "      <td>0.019294</td>\n",
              "      <td>0.003951</td>\n",
              "      <td>0.009452</td>\n",
              "      <td>-0.003930</td>\n",
              "      <td>-0.001654</td>\n",
              "      <td>-0.007448</td>\n",
              "      <td>0.003339</td>\n",
              "      <td>-0.001735</td>\n",
              "      <td>0.004656</td>\n",
              "      <td>-0.004393</td>\n",
              "      <td>0.000464</td>\n",
              "      <td>-0.000058</td>\n",
              "      <td>0.005960</td>\n",
              "      <td>0.003356</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.532723e+05</td>\n",
              "      <td>0.993474</td>\n",
              "      <td>6.563354</td>\n",
              "      <td>7.902323</td>\n",
              "      <td>2.846050</td>\n",
              "      <td>4.304038</td>\n",
              "      <td>25.760127</td>\n",
              "      <td>9.375699</td>\n",
              "      <td>2.657333</td>\n",
              "      <td>1.057639</td>\n",
              "      <td>1.015281</td>\n",
              "      <td>0.442991</td>\n",
              "      <td>0.135252</td>\n",
              "      <td>0.024874</td>\n",
              "      <td>14.940270</td>\n",
              "      <td>16.128424</td>\n",
              "      <td>1.207659</td>\n",
              "      <td>23.558816</td>\n",
              "      <td>107.815927</td>\n",
              "      <td>1.235632</td>\n",
              "      <td>0.035390</td>\n",
              "      <td>1.176192</td>\n",
              "      <td>0.082493</td>\n",
              "      <td>0.321813</td>\n",
              "      <td>2.326975</td>\n",
              "      <td>101.368809</td>\n",
              "      <td>1.478138</td>\n",
              "      <td>45.646184</td>\n",
              "      <td>153.060188</td>\n",
              "      <td>379.104934</td>\n",
              "      <td>1575.954298</td>\n",
              "      <td>56.017479</td>\n",
              "      <td>1.053653</td>\n",
              "      <td>11.787381</td>\n",
              "      <td>0.262443</td>\n",
              "      <td>244.012777</td>\n",
              "      <td>0.516502</td>\n",
              "      <td>187.724232</td>\n",
              "      <td>5.320714</td>\n",
              "      <td>29.167232</td>\n",
              "      <td>...</td>\n",
              "      <td>0.399808</td>\n",
              "      <td>0.345308</td>\n",
              "      <td>0.347456</td>\n",
              "      <td>0.332473</td>\n",
              "      <td>0.321406</td>\n",
              "      <td>0.292574</td>\n",
              "      <td>0.270887</td>\n",
              "      <td>0.285244</td>\n",
              "      <td>0.262761</td>\n",
              "      <td>0.269514</td>\n",
              "      <td>0.241773</td>\n",
              "      <td>0.244525</td>\n",
              "      <td>0.246485</td>\n",
              "      <td>0.247560</td>\n",
              "      <td>0.236307</td>\n",
              "      <td>0.205018</td>\n",
              "      <td>0.225697</td>\n",
              "      <td>0.213409</td>\n",
              "      <td>0.209506</td>\n",
              "      <td>0.205961</td>\n",
              "      <td>0.209026</td>\n",
              "      <td>0.187867</td>\n",
              "      <td>0.194700</td>\n",
              "      <td>0.181905</td>\n",
              "      <td>0.189907</td>\n",
              "      <td>0.180628</td>\n",
              "      <td>0.172939</td>\n",
              "      <td>0.162417</td>\n",
              "      <td>0.180656</td>\n",
              "      <td>0.171054</td>\n",
              "      <td>0.170816</td>\n",
              "      <td>0.163941</td>\n",
              "      <td>0.166224</td>\n",
              "      <td>0.157380</td>\n",
              "      <td>0.166602</td>\n",
              "      <td>0.150019</td>\n",
              "      <td>0.152127</td>\n",
              "      <td>0.145858</td>\n",
              "      <td>0.151741</td>\n",
              "      <td>0.148118</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>3.198770e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.933843</td>\n",
              "      <td>-0.969439</td>\n",
              "      <td>-1.097137</td>\n",
              "      <td>-0.920027</td>\n",
              "      <td>-1.006138</td>\n",
              "      <td>-1.000187</td>\n",
              "      <td>-0.798815</td>\n",
              "      <td>-0.687112</td>\n",
              "      <td>-0.668615</td>\n",
              "      <td>-0.801637</td>\n",
              "      <td>-0.726105</td>\n",
              "      <td>-0.698135</td>\n",
              "      <td>-0.734302</td>\n",
              "      <td>-0.831774</td>\n",
              "      <td>-0.661366</td>\n",
              "      <td>-0.604000</td>\n",
              "      <td>-0.624276</td>\n",
              "      <td>-0.615452</td>\n",
              "      <td>-0.597283</td>\n",
              "      <td>-0.806154</td>\n",
              "      <td>-0.673437</td>\n",
              "      <td>-0.584369</td>\n",
              "      <td>-0.787943</td>\n",
              "      <td>-0.701795</td>\n",
              "      <td>-0.663740</td>\n",
              "      <td>-0.697520</td>\n",
              "      <td>-0.529472</td>\n",
              "      <td>-0.536528</td>\n",
              "      <td>-0.753790</td>\n",
              "      <td>-0.557471</td>\n",
              "      <td>-0.571239</td>\n",
              "      <td>-0.533034</td>\n",
              "      <td>-0.571749</td>\n",
              "      <td>-0.680382</td>\n",
              "      <td>-0.648647</td>\n",
              "      <td>-0.538644</td>\n",
              "      <td>-0.550678</td>\n",
              "      <td>-0.580334</td>\n",
              "      <td>-0.559276</td>\n",
              "      <td>-0.518081</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>3.256388e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.303365</td>\n",
              "      <td>-0.331833</td>\n",
              "      <td>-0.193712</td>\n",
              "      <td>-0.220025</td>\n",
              "      <td>-0.159651</td>\n",
              "      <td>-0.216723</td>\n",
              "      <td>-0.190823</td>\n",
              "      <td>-0.176393</td>\n",
              "      <td>-0.130259</td>\n",
              "      <td>-0.160182</td>\n",
              "      <td>-0.131100</td>\n",
              "      <td>-0.143404</td>\n",
              "      <td>-0.129463</td>\n",
              "      <td>-0.173109</td>\n",
              "      <td>-0.164184</td>\n",
              "      <td>-0.152609</td>\n",
              "      <td>-0.134107</td>\n",
              "      <td>-0.113856</td>\n",
              "      <td>-0.133126</td>\n",
              "      <td>-0.099676</td>\n",
              "      <td>-0.112175</td>\n",
              "      <td>-0.121307</td>\n",
              "      <td>-0.104513</td>\n",
              "      <td>-0.124913</td>\n",
              "      <td>-0.112120</td>\n",
              "      <td>-0.103636</td>\n",
              "      <td>-0.085785</td>\n",
              "      <td>-0.118501</td>\n",
              "      <td>-0.089202</td>\n",
              "      <td>-0.126307</td>\n",
              "      <td>-0.097939</td>\n",
              "      <td>-0.108631</td>\n",
              "      <td>-0.090715</td>\n",
              "      <td>-0.062085</td>\n",
              "      <td>-0.092483</td>\n",
              "      <td>-0.083392</td>\n",
              "      <td>-0.097852</td>\n",
              "      <td>-0.084365</td>\n",
              "      <td>-0.092098</td>\n",
              "      <td>-0.094591</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>3.313985e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.003707</td>\n",
              "      <td>0.045433</td>\n",
              "      <td>0.064924</td>\n",
              "      <td>-0.007322</td>\n",
              "      <td>0.051658</td>\n",
              "      <td>-0.053177</td>\n",
              "      <td>-0.019007</td>\n",
              "      <td>-0.030674</td>\n",
              "      <td>0.017497</td>\n",
              "      <td>-0.034493</td>\n",
              "      <td>-0.022160</td>\n",
              "      <td>0.002732</td>\n",
              "      <td>-0.020873</td>\n",
              "      <td>0.013576</td>\n",
              "      <td>-0.022006</td>\n",
              "      <td>-0.015323</td>\n",
              "      <td>0.015488</td>\n",
              "      <td>0.016824</td>\n",
              "      <td>-0.044433</td>\n",
              "      <td>0.000929</td>\n",
              "      <td>-0.021161</td>\n",
              "      <td>-0.003651</td>\n",
              "      <td>0.001622</td>\n",
              "      <td>-0.012053</td>\n",
              "      <td>0.015342</td>\n",
              "      <td>0.010372</td>\n",
              "      <td>0.007140</td>\n",
              "      <td>0.004978</td>\n",
              "      <td>0.007067</td>\n",
              "      <td>-0.013731</td>\n",
              "      <td>-0.003920</td>\n",
              "      <td>-0.013905</td>\n",
              "      <td>0.000959</td>\n",
              "      <td>-0.005007</td>\n",
              "      <td>0.014839</td>\n",
              "      <td>-0.004766</td>\n",
              "      <td>0.005744</td>\n",
              "      <td>-0.009697</td>\n",
              "      <td>-0.004400</td>\n",
              "      <td>0.013266</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>7.474598e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.267167</td>\n",
              "      <td>0.165864</td>\n",
              "      <td>0.247223</td>\n",
              "      <td>0.186875</td>\n",
              "      <td>0.252065</td>\n",
              "      <td>0.109015</td>\n",
              "      <td>0.191541</td>\n",
              "      <td>0.181806</td>\n",
              "      <td>0.199292</td>\n",
              "      <td>0.178718</td>\n",
              "      <td>0.099278</td>\n",
              "      <td>0.180669</td>\n",
              "      <td>0.128743</td>\n",
              "      <td>0.157482</td>\n",
              "      <td>0.136590</td>\n",
              "      <td>0.117972</td>\n",
              "      <td>0.156187</td>\n",
              "      <td>0.158928</td>\n",
              "      <td>0.103986</td>\n",
              "      <td>0.115242</td>\n",
              "      <td>0.073736</td>\n",
              "      <td>0.124609</td>\n",
              "      <td>0.121336</td>\n",
              "      <td>0.095865</td>\n",
              "      <td>0.125497</td>\n",
              "      <td>0.101867</td>\n",
              "      <td>0.106555</td>\n",
              "      <td>0.110043</td>\n",
              "      <td>0.081471</td>\n",
              "      <td>0.121044</td>\n",
              "      <td>0.097395</td>\n",
              "      <td>0.105240</td>\n",
              "      <td>0.099313</td>\n",
              "      <td>0.103633</td>\n",
              "      <td>0.092808</td>\n",
              "      <td>0.081048</td>\n",
              "      <td>0.086873</td>\n",
              "      <td>0.081748</td>\n",
              "      <td>0.092386</td>\n",
              "      <td>0.092551</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.105896e+06</td>\n",
              "      <td>13.200000</td>\n",
              "      <td>79.900000</td>\n",
              "      <td>99.400000</td>\n",
              "      <td>99.600000</td>\n",
              "      <td>99.600000</td>\n",
              "      <td>833.000000</td>\n",
              "      <td>43.900000</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>25.900000</td>\n",
              "      <td>23.600000</td>\n",
              "      <td>6.340000</td>\n",
              "      <td>3.620000</td>\n",
              "      <td>1.040000</td>\n",
              "      <td>377.000000</td>\n",
              "      <td>429.000000</td>\n",
              "      <td>67.900000</td>\n",
              "      <td>96.000000</td>\n",
              "      <td>3480.000000</td>\n",
              "      <td>99.800000</td>\n",
              "      <td>1.100000</td>\n",
              "      <td>11.500000</td>\n",
              "      <td>2.300000</td>\n",
              "      <td>9.200000</td>\n",
              "      <td>94.600000</td>\n",
              "      <td>1390.000000</td>\n",
              "      <td>12.700000</td>\n",
              "      <td>386.000000</td>\n",
              "      <td>997.000000</td>\n",
              "      <td>2520.000000</td>\n",
              "      <td>40300.000000</td>\n",
              "      <td>311.000000</td>\n",
              "      <td>7.660000</td>\n",
              "      <td>93.000000</td>\n",
              "      <td>1.920000</td>\n",
              "      <td>7430.000000</td>\n",
              "      <td>7.870000</td>\n",
              "      <td>1790.000000</td>\n",
              "      <td>97.700000</td>\n",
              "      <td>903.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>1.122911</td>\n",
              "      <td>1.235694</td>\n",
              "      <td>1.222554</td>\n",
              "      <td>1.301511</td>\n",
              "      <td>1.225611</td>\n",
              "      <td>1.031472</td>\n",
              "      <td>0.770720</td>\n",
              "      <td>0.843955</td>\n",
              "      <td>1.048918</td>\n",
              "      <td>0.867486</td>\n",
              "      <td>1.478667</td>\n",
              "      <td>0.921318</td>\n",
              "      <td>0.756673</td>\n",
              "      <td>0.907832</td>\n",
              "      <td>0.918719</td>\n",
              "      <td>0.955452</td>\n",
              "      <td>0.778253</td>\n",
              "      <td>0.770962</td>\n",
              "      <td>0.961512</td>\n",
              "      <td>0.903499</td>\n",
              "      <td>0.642813</td>\n",
              "      <td>0.903780</td>\n",
              "      <td>0.722286</td>\n",
              "      <td>0.874324</td>\n",
              "      <td>0.835497</td>\n",
              "      <td>0.652537</td>\n",
              "      <td>0.834180</td>\n",
              "      <td>0.705718</td>\n",
              "      <td>0.985508</td>\n",
              "      <td>0.545425</td>\n",
              "      <td>0.757822</td>\n",
              "      <td>0.626882</td>\n",
              "      <td>0.575618</td>\n",
              "      <td>0.746988</td>\n",
              "      <td>0.949548</td>\n",
              "      <td>0.502315</td>\n",
              "      <td>0.607102</td>\n",
              "      <td>0.563422</td>\n",
              "      <td>0.705493</td>\n",
              "      <td>0.757814</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8 rows × 249 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "             fdc_id          1002  ...            48            49\n",
              "count  1.048400e+04  10484.000000  ...  10484.000000  10484.000000\n",
              "mean   4.794562e+05      0.241070  ...      0.005960      0.003356\n",
              "std    2.532723e+05      0.993474  ...      0.151741      0.148118\n",
              "min    3.198770e+05      0.000000  ...     -0.559276     -0.518081\n",
              "25%    3.256388e+05      0.000000  ...     -0.092098     -0.094591\n",
              "50%    3.313985e+05      0.000000  ...     -0.004400      0.013266\n",
              "75%    7.474598e+05      0.000000  ...      0.092386      0.092551\n",
              "max    1.105896e+06     13.200000  ...      0.705493      0.757814\n",
              "\n",
              "[8 rows x 249 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 350
        },
        "id": "s3VsqvE_GFBJ",
        "outputId": "8c7ece87-e8d8-4569-c59d-23ea5e7d356b"
      },
      "source": [
        "test_data.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>fdc_id</th>\n",
              "      <th>1002</th>\n",
              "      <th>1003</th>\n",
              "      <th>1004</th>\n",
              "      <th>1005</th>\n",
              "      <th>1007</th>\n",
              "      <th>1008</th>\n",
              "      <th>1009</th>\n",
              "      <th>1010</th>\n",
              "      <th>1011</th>\n",
              "      <th>1012</th>\n",
              "      <th>1013</th>\n",
              "      <th>1014</th>\n",
              "      <th>1024</th>\n",
              "      <th>1032</th>\n",
              "      <th>1039</th>\n",
              "      <th>1050</th>\n",
              "      <th>1051</th>\n",
              "      <th>1062</th>\n",
              "      <th>1063</th>\n",
              "      <th>1075</th>\n",
              "      <th>1079</th>\n",
              "      <th>1082</th>\n",
              "      <th>1084</th>\n",
              "      <th>1085</th>\n",
              "      <th>1087</th>\n",
              "      <th>1089</th>\n",
              "      <th>1090</th>\n",
              "      <th>1091</th>\n",
              "      <th>1092</th>\n",
              "      <th>1093</th>\n",
              "      <th>1094</th>\n",
              "      <th>1095</th>\n",
              "      <th>1097</th>\n",
              "      <th>1098</th>\n",
              "      <th>1100</th>\n",
              "      <th>1101</th>\n",
              "      <th>1102</th>\n",
              "      <th>1103</th>\n",
              "      <th>1105</th>\n",
              "      <th>...</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "      <th>30</th>\n",
              "      <th>31</th>\n",
              "      <th>32</th>\n",
              "      <th>33</th>\n",
              "      <th>34</th>\n",
              "      <th>35</th>\n",
              "      <th>36</th>\n",
              "      <th>37</th>\n",
              "      <th>38</th>\n",
              "      <th>39</th>\n",
              "      <th>40</th>\n",
              "      <th>41</th>\n",
              "      <th>42</th>\n",
              "      <th>43</th>\n",
              "      <th>44</th>\n",
              "      <th>45</th>\n",
              "      <th>46</th>\n",
              "      <th>47</th>\n",
              "      <th>48</th>\n",
              "      <th>49</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>2.621000e+03</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.0</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.00000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>4.859338e+05</td>\n",
              "      <td>0.254044</td>\n",
              "      <td>1.995738</td>\n",
              "      <td>1.836654</td>\n",
              "      <td>0.179443</td>\n",
              "      <td>0.518111</td>\n",
              "      <td>1.893934</td>\n",
              "      <td>2.729699</td>\n",
              "      <td>0.186539</td>\n",
              "      <td>0.117944</td>\n",
              "      <td>0.123197</td>\n",
              "      <td>0.044151</td>\n",
              "      <td>0.006902</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.688287</td>\n",
              "      <td>0.713850</td>\n",
              "      <td>0.064632</td>\n",
              "      <td>8.386478</td>\n",
              "      <td>7.925601</td>\n",
              "      <td>0.090298</td>\n",
              "      <td>0.002396</td>\n",
              "      <td>0.372262</td>\n",
              "      <td>0.004617</td>\n",
              "      <td>0.019344</td>\n",
              "      <td>0.097612</td>\n",
              "      <td>27.143457</td>\n",
              "      <td>0.499348</td>\n",
              "      <td>15.138764</td>\n",
              "      <td>57.233499</td>\n",
              "      <td>131.396032</td>\n",
              "      <td>186.429989</td>\n",
              "      <td>15.805799</td>\n",
              "      <td>0.394651</td>\n",
              "      <td>3.163716</td>\n",
              "      <td>0.082882</td>\n",
              "      <td>10.29050</td>\n",
              "      <td>0.157542</td>\n",
              "      <td>44.295956</td>\n",
              "      <td>0.743647</td>\n",
              "      <td>2.217474</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.008367</td>\n",
              "      <td>-0.053989</td>\n",
              "      <td>0.023549</td>\n",
              "      <td>0.019596</td>\n",
              "      <td>0.049367</td>\n",
              "      <td>-0.052746</td>\n",
              "      <td>-0.011657</td>\n",
              "      <td>-0.001276</td>\n",
              "      <td>0.038876</td>\n",
              "      <td>-0.001373</td>\n",
              "      <td>-0.009874</td>\n",
              "      <td>0.014033</td>\n",
              "      <td>-0.004292</td>\n",
              "      <td>-0.014387</td>\n",
              "      <td>0.007798</td>\n",
              "      <td>-0.000686</td>\n",
              "      <td>0.008489</td>\n",
              "      <td>0.024095</td>\n",
              "      <td>-0.008769</td>\n",
              "      <td>-0.006628</td>\n",
              "      <td>-0.011060</td>\n",
              "      <td>-0.004628</td>\n",
              "      <td>0.008216</td>\n",
              "      <td>-0.012476</td>\n",
              "      <td>0.000541</td>\n",
              "      <td>0.006882</td>\n",
              "      <td>0.014999</td>\n",
              "      <td>0.006314</td>\n",
              "      <td>0.009149</td>\n",
              "      <td>-0.000132</td>\n",
              "      <td>-0.001614</td>\n",
              "      <td>-0.004360</td>\n",
              "      <td>0.002712</td>\n",
              "      <td>0.006195</td>\n",
              "      <td>0.004842</td>\n",
              "      <td>-0.006025</td>\n",
              "      <td>0.002510</td>\n",
              "      <td>-0.007046</td>\n",
              "      <td>0.009037</td>\n",
              "      <td>0.004557</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.588545e+05</td>\n",
              "      <td>1.023870</td>\n",
              "      <td>6.664248</td>\n",
              "      <td>7.579117</td>\n",
              "      <td>3.130588</td>\n",
              "      <td>4.842051</td>\n",
              "      <td>25.882445</td>\n",
              "      <td>9.724790</td>\n",
              "      <td>3.427916</td>\n",
              "      <td>1.048284</td>\n",
              "      <td>1.020284</td>\n",
              "      <td>0.437916</td>\n",
              "      <td>0.105316</td>\n",
              "      <td>0.0</td>\n",
              "      <td>14.455965</td>\n",
              "      <td>15.017005</td>\n",
              "      <td>1.629949</td>\n",
              "      <td>23.514720</td>\n",
              "      <td>108.313495</td>\n",
              "      <td>2.238760</td>\n",
              "      <td>0.031385</td>\n",
              "      <td>1.225180</td>\n",
              "      <td>0.080585</td>\n",
              "      <td>0.312139</td>\n",
              "      <td>2.223482</td>\n",
              "      <td>93.940655</td>\n",
              "      <td>1.504124</td>\n",
              "      <td>46.188392</td>\n",
              "      <td>154.934092</td>\n",
              "      <td>394.939205</td>\n",
              "      <td>2268.675857</td>\n",
              "      <td>58.194010</td>\n",
              "      <td>1.062197</td>\n",
              "      <td>12.091657</td>\n",
              "      <td>0.274454</td>\n",
              "      <td>209.52813</td>\n",
              "      <td>0.511793</td>\n",
              "      <td>185.121625</td>\n",
              "      <td>4.863353</td>\n",
              "      <td>30.198858</td>\n",
              "      <td>...</td>\n",
              "      <td>0.401700</td>\n",
              "      <td>0.346211</td>\n",
              "      <td>0.357762</td>\n",
              "      <td>0.330427</td>\n",
              "      <td>0.320807</td>\n",
              "      <td>0.294364</td>\n",
              "      <td>0.266655</td>\n",
              "      <td>0.283749</td>\n",
              "      <td>0.257284</td>\n",
              "      <td>0.268003</td>\n",
              "      <td>0.236137</td>\n",
              "      <td>0.241392</td>\n",
              "      <td>0.239741</td>\n",
              "      <td>0.249726</td>\n",
              "      <td>0.232766</td>\n",
              "      <td>0.208973</td>\n",
              "      <td>0.222512</td>\n",
              "      <td>0.208108</td>\n",
              "      <td>0.208984</td>\n",
              "      <td>0.204199</td>\n",
              "      <td>0.207120</td>\n",
              "      <td>0.186865</td>\n",
              "      <td>0.189641</td>\n",
              "      <td>0.177829</td>\n",
              "      <td>0.184825</td>\n",
              "      <td>0.180475</td>\n",
              "      <td>0.171326</td>\n",
              "      <td>0.160948</td>\n",
              "      <td>0.182043</td>\n",
              "      <td>0.171217</td>\n",
              "      <td>0.169660</td>\n",
              "      <td>0.158665</td>\n",
              "      <td>0.166007</td>\n",
              "      <td>0.154350</td>\n",
              "      <td>0.162924</td>\n",
              "      <td>0.149703</td>\n",
              "      <td>0.150870</td>\n",
              "      <td>0.142503</td>\n",
              "      <td>0.149526</td>\n",
              "      <td>0.149701</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>3.199080e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.933855</td>\n",
              "      <td>-0.969972</td>\n",
              "      <td>-0.973889</td>\n",
              "      <td>-0.889484</td>\n",
              "      <td>-0.844214</td>\n",
              "      <td>-0.975344</td>\n",
              "      <td>-0.795263</td>\n",
              "      <td>-0.683452</td>\n",
              "      <td>-0.613335</td>\n",
              "      <td>-0.801561</td>\n",
              "      <td>-0.678256</td>\n",
              "      <td>-0.625027</td>\n",
              "      <td>-0.734182</td>\n",
              "      <td>-0.848955</td>\n",
              "      <td>-0.669846</td>\n",
              "      <td>-0.603996</td>\n",
              "      <td>-0.549889</td>\n",
              "      <td>-0.615429</td>\n",
              "      <td>-0.597251</td>\n",
              "      <td>-0.639653</td>\n",
              "      <td>-0.673297</td>\n",
              "      <td>-0.525369</td>\n",
              "      <td>-0.800978</td>\n",
              "      <td>-0.603322</td>\n",
              "      <td>-0.656870</td>\n",
              "      <td>-0.595664</td>\n",
              "      <td>-0.542536</td>\n",
              "      <td>-0.560396</td>\n",
              "      <td>-0.528300</td>\n",
              "      <td>-0.567381</td>\n",
              "      <td>-0.546243</td>\n",
              "      <td>-0.532969</td>\n",
              "      <td>-0.571732</td>\n",
              "      <td>-0.606464</td>\n",
              "      <td>-0.621206</td>\n",
              "      <td>-0.538610</td>\n",
              "      <td>-0.515188</td>\n",
              "      <td>-0.532556</td>\n",
              "      <td>-0.568859</td>\n",
              "      <td>-0.486389</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>3.256660e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.300802</td>\n",
              "      <td>-0.333182</td>\n",
              "      <td>-0.211713</td>\n",
              "      <td>-0.219276</td>\n",
              "      <td>-0.162027</td>\n",
              "      <td>-0.220277</td>\n",
              "      <td>-0.199176</td>\n",
              "      <td>-0.173755</td>\n",
              "      <td>-0.138855</td>\n",
              "      <td>-0.157256</td>\n",
              "      <td>-0.125563</td>\n",
              "      <td>-0.140941</td>\n",
              "      <td>-0.126020</td>\n",
              "      <td>-0.188958</td>\n",
              "      <td>-0.163733</td>\n",
              "      <td>-0.151088</td>\n",
              "      <td>-0.139832</td>\n",
              "      <td>-0.108629</td>\n",
              "      <td>-0.133923</td>\n",
              "      <td>-0.110559</td>\n",
              "      <td>-0.113465</td>\n",
              "      <td>-0.121356</td>\n",
              "      <td>-0.100510</td>\n",
              "      <td>-0.132482</td>\n",
              "      <td>-0.112254</td>\n",
              "      <td>-0.100508</td>\n",
              "      <td>-0.102012</td>\n",
              "      <td>-0.118485</td>\n",
              "      <td>-0.091904</td>\n",
              "      <td>-0.118641</td>\n",
              "      <td>-0.098746</td>\n",
              "      <td>-0.105721</td>\n",
              "      <td>-0.087408</td>\n",
              "      <td>-0.061454</td>\n",
              "      <td>-0.078814</td>\n",
              "      <td>-0.084274</td>\n",
              "      <td>-0.099628</td>\n",
              "      <td>-0.087041</td>\n",
              "      <td>-0.091850</td>\n",
              "      <td>-0.094435</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>3.314670e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.014279</td>\n",
              "      <td>0.039952</td>\n",
              "      <td>0.050841</td>\n",
              "      <td>-0.003235</td>\n",
              "      <td>0.049275</td>\n",
              "      <td>-0.058812</td>\n",
              "      <td>-0.026589</td>\n",
              "      <td>-0.030488</td>\n",
              "      <td>0.002919</td>\n",
              "      <td>-0.033887</td>\n",
              "      <td>-0.022722</td>\n",
              "      <td>-0.007847</td>\n",
              "      <td>-0.028247</td>\n",
              "      <td>0.007928</td>\n",
              "      <td>-0.024120</td>\n",
              "      <td>-0.014729</td>\n",
              "      <td>0.013604</td>\n",
              "      <td>0.016520</td>\n",
              "      <td>-0.045117</td>\n",
              "      <td>-0.003871</td>\n",
              "      <td>-0.029207</td>\n",
              "      <td>-0.008502</td>\n",
              "      <td>-0.001047</td>\n",
              "      <td>-0.020061</td>\n",
              "      <td>0.015250</td>\n",
              "      <td>0.009938</td>\n",
              "      <td>0.004786</td>\n",
              "      <td>0.007806</td>\n",
              "      <td>0.011801</td>\n",
              "      <td>-0.011018</td>\n",
              "      <td>-0.003670</td>\n",
              "      <td>-0.013831</td>\n",
              "      <td>0.003276</td>\n",
              "      <td>-0.003831</td>\n",
              "      <td>0.025561</td>\n",
              "      <td>-0.009699</td>\n",
              "      <td>0.005768</td>\n",
              "      <td>-0.009950</td>\n",
              "      <td>-0.002146</td>\n",
              "      <td>0.015221</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>7.475440e+05</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.266160</td>\n",
              "      <td>0.166972</td>\n",
              "      <td>0.254764</td>\n",
              "      <td>0.193289</td>\n",
              "      <td>0.241754</td>\n",
              "      <td>0.109078</td>\n",
              "      <td>0.174927</td>\n",
              "      <td>0.169065</td>\n",
              "      <td>0.193289</td>\n",
              "      <td>0.173140</td>\n",
              "      <td>0.098100</td>\n",
              "      <td>0.180645</td>\n",
              "      <td>0.125638</td>\n",
              "      <td>0.157089</td>\n",
              "      <td>0.134833</td>\n",
              "      <td>0.126750</td>\n",
              "      <td>0.150886</td>\n",
              "      <td>0.149021</td>\n",
              "      <td>0.104876</td>\n",
              "      <td>0.096763</td>\n",
              "      <td>0.076813</td>\n",
              "      <td>0.115245</td>\n",
              "      <td>0.098988</td>\n",
              "      <td>0.092810</td>\n",
              "      <td>0.118545</td>\n",
              "      <td>0.095405</td>\n",
              "      <td>0.108273</td>\n",
              "      <td>0.110751</td>\n",
              "      <td>0.081606</td>\n",
              "      <td>0.127006</td>\n",
              "      <td>0.097460</td>\n",
              "      <td>0.102299</td>\n",
              "      <td>0.105051</td>\n",
              "      <td>0.107226</td>\n",
              "      <td>0.089180</td>\n",
              "      <td>0.080794</td>\n",
              "      <td>0.093440</td>\n",
              "      <td>0.075128</td>\n",
              "      <td>0.092280</td>\n",
              "      <td>0.085729</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.105897e+06</td>\n",
              "      <td>13.200000</td>\n",
              "      <td>40.900000</td>\n",
              "      <td>98.900000</td>\n",
              "      <td>99.600000</td>\n",
              "      <td>99.600000</td>\n",
              "      <td>654.000000</td>\n",
              "      <td>42.700000</td>\n",
              "      <td>99.900000</td>\n",
              "      <td>24.800000</td>\n",
              "      <td>22.900000</td>\n",
              "      <td>5.730000</td>\n",
              "      <td>3.830000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>341.000000</td>\n",
              "      <td>369.000000</td>\n",
              "      <td>62.800000</td>\n",
              "      <td>96.700000</td>\n",
              "      <td>2740.000000</td>\n",
              "      <td>99.800000</td>\n",
              "      <td>0.820000</td>\n",
              "      <td>11.300000</td>\n",
              "      <td>2.900000</td>\n",
              "      <td>8.300000</td>\n",
              "      <td>94.000000</td>\n",
              "      <td>1260.000000</td>\n",
              "      <td>10.000000</td>\n",
              "      <td>357.000000</td>\n",
              "      <td>997.000000</td>\n",
              "      <td>2460.000000</td>\n",
              "      <td>40700.000000</td>\n",
              "      <td>277.000000</td>\n",
              "      <td>7.440000</td>\n",
              "      <td>81.000000</td>\n",
              "      <td>1.850000</td>\n",
              "      <td>6170.00000</td>\n",
              "      <td>3.440000</td>\n",
              "      <td>1640.000000</td>\n",
              "      <td>81.300000</td>\n",
              "      <td>912.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>1.068850</td>\n",
              "      <td>1.167370</td>\n",
              "      <td>1.230920</td>\n",
              "      <td>1.266974</td>\n",
              "      <td>1.128774</td>\n",
              "      <td>1.031444</td>\n",
              "      <td>0.752794</td>\n",
              "      <td>0.805872</td>\n",
              "      <td>1.049345</td>\n",
              "      <td>0.867322</td>\n",
              "      <td>1.478752</td>\n",
              "      <td>0.881879</td>\n",
              "      <td>0.756606</td>\n",
              "      <td>0.763748</td>\n",
              "      <td>0.878603</td>\n",
              "      <td>0.970210</td>\n",
              "      <td>0.777881</td>\n",
              "      <td>0.770679</td>\n",
              "      <td>0.961082</td>\n",
              "      <td>0.849681</td>\n",
              "      <td>0.576198</td>\n",
              "      <td>0.609812</td>\n",
              "      <td>0.676092</td>\n",
              "      <td>0.818344</td>\n",
              "      <td>0.675452</td>\n",
              "      <td>0.680991</td>\n",
              "      <td>0.846139</td>\n",
              "      <td>0.700253</td>\n",
              "      <td>0.928878</td>\n",
              "      <td>0.511358</td>\n",
              "      <td>0.702707</td>\n",
              "      <td>0.620772</td>\n",
              "      <td>0.572546</td>\n",
              "      <td>0.701231</td>\n",
              "      <td>0.621155</td>\n",
              "      <td>0.483569</td>\n",
              "      <td>0.644586</td>\n",
              "      <td>0.563413</td>\n",
              "      <td>0.606385</td>\n",
              "      <td>0.716600</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8 rows × 249 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "             fdc_id         1002  ...           48           49\n",
              "count  2.621000e+03  2621.000000  ...  2621.000000  2621.000000\n",
              "mean   4.859338e+05     0.254044  ...     0.009037     0.004557\n",
              "std    2.588545e+05     1.023870  ...     0.149526     0.149701\n",
              "min    3.199080e+05     0.000000  ...    -0.568859    -0.486389\n",
              "25%    3.256660e+05     0.000000  ...    -0.091850    -0.094435\n",
              "50%    3.314670e+05     0.000000  ...    -0.002146     0.015221\n",
              "75%    7.475440e+05     0.000000  ...     0.092280     0.085729\n",
              "max    1.105897e+06    13.200000  ...     0.606385     0.716600\n",
              "\n",
              "[8 rows x 249 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UK9qFJ34GFBJ",
        "outputId": "01f4ba00-5637-4f5b-e67b-ac96456c5e61"
      },
      "source": [
        "indices = test_data['fdc_id']\n",
        "indices"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "7626     333186\n",
              "7678     333305\n",
              "6030     330174\n",
              "10633    748750\n",
              "2944     325099\n",
              "          ...  \n",
              "3560     326060\n",
              "7279     332543\n",
              "4019     326847\n",
              "1607     322877\n",
              "2520     324378\n",
              "Name: fdc_id, Length: 2621, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mp1fwIQMGFBK"
      },
      "source": [
        "del train_data['fdc_id']\n",
        "del test_data['fdc_id']\n",
        "columns = train_data.columns"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lZrQsRQXGFBK",
        "outputId": "cd3e9ece-2238-40fe-9271-f1526f56d510"
      },
      "source": [
        "train_data = train_data.to_numpy()\n",
        "test_data.fillna(0, inplace=True)\n",
        "test_data = test_data.to_numpy()\n",
        "data = dataset.values\n",
        "rows, cols = data.shape\n",
        "cols -= 1\n",
        "\n",
        "y_test = test_data.copy()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pandas/core/frame.py:4327: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  downcast=downcast,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 350
        },
        "id": "9efq7YXVGFBL",
        "outputId": "0d14659f-d875-4ba4-a154-d752f15cbb4d"
      },
      "source": [
        "missed_data, missed_data_train, mask = missing_method(test_data, train_data, num_embeddings)\n",
        "pd.DataFrame(missed_data).describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "      <th>30</th>\n",
              "      <th>31</th>\n",
              "      <th>32</th>\n",
              "      <th>33</th>\n",
              "      <th>34</th>\n",
              "      <th>35</th>\n",
              "      <th>36</th>\n",
              "      <th>37</th>\n",
              "      <th>38</th>\n",
              "      <th>39</th>\n",
              "      <th>...</th>\n",
              "      <th>208</th>\n",
              "      <th>209</th>\n",
              "      <th>210</th>\n",
              "      <th>211</th>\n",
              "      <th>212</th>\n",
              "      <th>213</th>\n",
              "      <th>214</th>\n",
              "      <th>215</th>\n",
              "      <th>216</th>\n",
              "      <th>217</th>\n",
              "      <th>218</th>\n",
              "      <th>219</th>\n",
              "      <th>220</th>\n",
              "      <th>221</th>\n",
              "      <th>222</th>\n",
              "      <th>223</th>\n",
              "      <th>224</th>\n",
              "      <th>225</th>\n",
              "      <th>226</th>\n",
              "      <th>227</th>\n",
              "      <th>228</th>\n",
              "      <th>229</th>\n",
              "      <th>230</th>\n",
              "      <th>231</th>\n",
              "      <th>232</th>\n",
              "      <th>233</th>\n",
              "      <th>234</th>\n",
              "      <th>235</th>\n",
              "      <th>236</th>\n",
              "      <th>237</th>\n",
              "      <th>238</th>\n",
              "      <th>239</th>\n",
              "      <th>240</th>\n",
              "      <th>241</th>\n",
              "      <th>242</th>\n",
              "      <th>243</th>\n",
              "      <th>244</th>\n",
              "      <th>245</th>\n",
              "      <th>246</th>\n",
              "      <th>247</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>2094.000000</td>\n",
              "      <td>2057.000000</td>\n",
              "      <td>2110.00000</td>\n",
              "      <td>2112.000000</td>\n",
              "      <td>2128.000000</td>\n",
              "      <td>2100.000000</td>\n",
              "      <td>2096.000000</td>\n",
              "      <td>2105.000000</td>\n",
              "      <td>2111.000000</td>\n",
              "      <td>2090.000000</td>\n",
              "      <td>2084.000000</td>\n",
              "      <td>2131.000000</td>\n",
              "      <td>2083.0</td>\n",
              "      <td>2117.000000</td>\n",
              "      <td>2081.000000</td>\n",
              "      <td>2127.000000</td>\n",
              "      <td>2111.000000</td>\n",
              "      <td>2074.000000</td>\n",
              "      <td>2093.000000</td>\n",
              "      <td>2095.000000</td>\n",
              "      <td>2075.000000</td>\n",
              "      <td>2098.000000</td>\n",
              "      <td>2067.000000</td>\n",
              "      <td>2142.000000</td>\n",
              "      <td>2113.000000</td>\n",
              "      <td>2087.000000</td>\n",
              "      <td>2087.000000</td>\n",
              "      <td>2151.000000</td>\n",
              "      <td>2086.000000</td>\n",
              "      <td>2085.000000</td>\n",
              "      <td>2079.000000</td>\n",
              "      <td>2103.000000</td>\n",
              "      <td>2122.000000</td>\n",
              "      <td>2118.000000</td>\n",
              "      <td>2119.000000</td>\n",
              "      <td>2125.000000</td>\n",
              "      <td>2092.000000</td>\n",
              "      <td>2124.000000</td>\n",
              "      <td>2122.000000</td>\n",
              "      <td>2103.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "      <td>2621.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>0.272364</td>\n",
              "      <td>1.939626</td>\n",
              "      <td>1.76163</td>\n",
              "      <td>0.173873</td>\n",
              "      <td>0.527603</td>\n",
              "      <td>1.359048</td>\n",
              "      <td>2.784098</td>\n",
              "      <td>0.168546</td>\n",
              "      <td>0.123302</td>\n",
              "      <td>0.108555</td>\n",
              "      <td>0.045763</td>\n",
              "      <td>0.007405</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.601323</td>\n",
              "      <td>0.590581</td>\n",
              "      <td>0.071180</td>\n",
              "      <td>8.551644</td>\n",
              "      <td>5.432015</td>\n",
              "      <td>0.074233</td>\n",
              "      <td>0.002869</td>\n",
              "      <td>0.364964</td>\n",
              "      <td>0.005434</td>\n",
              "      <td>0.017078</td>\n",
              "      <td>0.098735</td>\n",
              "      <td>27.946522</td>\n",
              "      <td>0.507834</td>\n",
              "      <td>14.693531</td>\n",
              "      <td>57.714551</td>\n",
              "      <td>133.066155</td>\n",
              "      <td>199.305995</td>\n",
              "      <td>16.016835</td>\n",
              "      <td>0.389544</td>\n",
              "      <td>2.962253</td>\n",
              "      <td>0.082634</td>\n",
              "      <td>9.752006</td>\n",
              "      <td>0.162012</td>\n",
              "      <td>45.254828</td>\n",
              "      <td>0.727260</td>\n",
              "      <td>2.500471</td>\n",
              "      <td>0.415597</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.008367</td>\n",
              "      <td>-0.053989</td>\n",
              "      <td>0.023549</td>\n",
              "      <td>0.019596</td>\n",
              "      <td>0.049367</td>\n",
              "      <td>-0.052746</td>\n",
              "      <td>-0.011657</td>\n",
              "      <td>-0.001276</td>\n",
              "      <td>0.038876</td>\n",
              "      <td>-0.001373</td>\n",
              "      <td>-0.009874</td>\n",
              "      <td>0.014033</td>\n",
              "      <td>-0.004292</td>\n",
              "      <td>-0.014387</td>\n",
              "      <td>0.007798</td>\n",
              "      <td>-0.000686</td>\n",
              "      <td>0.008489</td>\n",
              "      <td>0.024095</td>\n",
              "      <td>-0.008769</td>\n",
              "      <td>-0.006628</td>\n",
              "      <td>-0.011060</td>\n",
              "      <td>-0.004628</td>\n",
              "      <td>0.008216</td>\n",
              "      <td>-0.012476</td>\n",
              "      <td>0.000541</td>\n",
              "      <td>0.006882</td>\n",
              "      <td>0.014999</td>\n",
              "      <td>0.006314</td>\n",
              "      <td>0.009149</td>\n",
              "      <td>-0.000132</td>\n",
              "      <td>-0.001614</td>\n",
              "      <td>-0.004360</td>\n",
              "      <td>0.002712</td>\n",
              "      <td>0.006195</td>\n",
              "      <td>0.004842</td>\n",
              "      <td>-0.006025</td>\n",
              "      <td>0.002510</td>\n",
              "      <td>-0.007046</td>\n",
              "      <td>0.009037</td>\n",
              "      <td>0.004557</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>1.096187</td>\n",
              "      <td>6.569316</td>\n",
              "      <td>7.18062</td>\n",
              "      <td>2.980519</td>\n",
              "      <td>4.902768</td>\n",
              "      <td>20.176795</td>\n",
              "      <td>9.807325</td>\n",
              "      <td>3.137919</td>\n",
              "      <td>1.126675</td>\n",
              "      <td>0.908371</td>\n",
              "      <td>0.437763</td>\n",
              "      <td>0.113087</td>\n",
              "      <td>0.0</td>\n",
              "      <td>13.862775</td>\n",
              "      <td>13.577470</td>\n",
              "      <td>1.766844</td>\n",
              "      <td>23.868884</td>\n",
              "      <td>80.292984</td>\n",
              "      <td>2.237338</td>\n",
              "      <td>0.034784</td>\n",
              "      <td>1.214510</td>\n",
              "      <td>0.088752</td>\n",
              "      <td>0.289027</td>\n",
              "      <td>2.368098</td>\n",
              "      <td>98.178407</td>\n",
              "      <td>1.521939</td>\n",
              "      <td>45.325499</td>\n",
              "      <td>154.395943</td>\n",
              "      <td>398.289975</td>\n",
              "      <td>2395.089148</td>\n",
              "      <td>58.614368</td>\n",
              "      <td>1.049673</td>\n",
              "      <td>11.754408</td>\n",
              "      <td>0.273274</td>\n",
              "      <td>190.699412</td>\n",
              "      <td>0.522117</td>\n",
              "      <td>187.593139</td>\n",
              "      <td>4.862275</td>\n",
              "      <td>33.054742</td>\n",
              "      <td>10.612560</td>\n",
              "      <td>...</td>\n",
              "      <td>0.401700</td>\n",
              "      <td>0.346211</td>\n",
              "      <td>0.357762</td>\n",
              "      <td>0.330427</td>\n",
              "      <td>0.320807</td>\n",
              "      <td>0.294364</td>\n",
              "      <td>0.266655</td>\n",
              "      <td>0.283749</td>\n",
              "      <td>0.257284</td>\n",
              "      <td>0.268003</td>\n",
              "      <td>0.236137</td>\n",
              "      <td>0.241392</td>\n",
              "      <td>0.239741</td>\n",
              "      <td>0.249726</td>\n",
              "      <td>0.232766</td>\n",
              "      <td>0.208973</td>\n",
              "      <td>0.222512</td>\n",
              "      <td>0.208108</td>\n",
              "      <td>0.208984</td>\n",
              "      <td>0.204199</td>\n",
              "      <td>0.207120</td>\n",
              "      <td>0.186865</td>\n",
              "      <td>0.189641</td>\n",
              "      <td>0.177829</td>\n",
              "      <td>0.184825</td>\n",
              "      <td>0.180475</td>\n",
              "      <td>0.171326</td>\n",
              "      <td>0.160948</td>\n",
              "      <td>0.182043</td>\n",
              "      <td>0.171217</td>\n",
              "      <td>0.169660</td>\n",
              "      <td>0.158665</td>\n",
              "      <td>0.166007</td>\n",
              "      <td>0.154350</td>\n",
              "      <td>0.162924</td>\n",
              "      <td>0.149703</td>\n",
              "      <td>0.150870</td>\n",
              "      <td>0.142503</td>\n",
              "      <td>0.149526</td>\n",
              "      <td>0.149701</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.933855</td>\n",
              "      <td>-0.969972</td>\n",
              "      <td>-0.973889</td>\n",
              "      <td>-0.889484</td>\n",
              "      <td>-0.844214</td>\n",
              "      <td>-0.975344</td>\n",
              "      <td>-0.795263</td>\n",
              "      <td>-0.683452</td>\n",
              "      <td>-0.613335</td>\n",
              "      <td>-0.801561</td>\n",
              "      <td>-0.678256</td>\n",
              "      <td>-0.625027</td>\n",
              "      <td>-0.734182</td>\n",
              "      <td>-0.848955</td>\n",
              "      <td>-0.669846</td>\n",
              "      <td>-0.603996</td>\n",
              "      <td>-0.549889</td>\n",
              "      <td>-0.615429</td>\n",
              "      <td>-0.597251</td>\n",
              "      <td>-0.639653</td>\n",
              "      <td>-0.673297</td>\n",
              "      <td>-0.525369</td>\n",
              "      <td>-0.800978</td>\n",
              "      <td>-0.603322</td>\n",
              "      <td>-0.656870</td>\n",
              "      <td>-0.595664</td>\n",
              "      <td>-0.542536</td>\n",
              "      <td>-0.560396</td>\n",
              "      <td>-0.528300</td>\n",
              "      <td>-0.567381</td>\n",
              "      <td>-0.546243</td>\n",
              "      <td>-0.532969</td>\n",
              "      <td>-0.571732</td>\n",
              "      <td>-0.606464</td>\n",
              "      <td>-0.621206</td>\n",
              "      <td>-0.538610</td>\n",
              "      <td>-0.515188</td>\n",
              "      <td>-0.532556</td>\n",
              "      <td>-0.568859</td>\n",
              "      <td>-0.486389</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.300802</td>\n",
              "      <td>-0.333182</td>\n",
              "      <td>-0.211713</td>\n",
              "      <td>-0.219276</td>\n",
              "      <td>-0.162027</td>\n",
              "      <td>-0.220277</td>\n",
              "      <td>-0.199176</td>\n",
              "      <td>-0.173755</td>\n",
              "      <td>-0.138855</td>\n",
              "      <td>-0.157256</td>\n",
              "      <td>-0.125563</td>\n",
              "      <td>-0.140941</td>\n",
              "      <td>-0.126020</td>\n",
              "      <td>-0.188958</td>\n",
              "      <td>-0.163733</td>\n",
              "      <td>-0.151088</td>\n",
              "      <td>-0.139832</td>\n",
              "      <td>-0.108629</td>\n",
              "      <td>-0.133923</td>\n",
              "      <td>-0.110559</td>\n",
              "      <td>-0.113465</td>\n",
              "      <td>-0.121356</td>\n",
              "      <td>-0.100510</td>\n",
              "      <td>-0.132482</td>\n",
              "      <td>-0.112254</td>\n",
              "      <td>-0.100508</td>\n",
              "      <td>-0.102012</td>\n",
              "      <td>-0.118485</td>\n",
              "      <td>-0.091904</td>\n",
              "      <td>-0.118641</td>\n",
              "      <td>-0.098746</td>\n",
              "      <td>-0.105721</td>\n",
              "      <td>-0.087408</td>\n",
              "      <td>-0.061454</td>\n",
              "      <td>-0.078814</td>\n",
              "      <td>-0.084274</td>\n",
              "      <td>-0.099628</td>\n",
              "      <td>-0.087041</td>\n",
              "      <td>-0.091850</td>\n",
              "      <td>-0.094435</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.014279</td>\n",
              "      <td>0.039952</td>\n",
              "      <td>0.050841</td>\n",
              "      <td>-0.003235</td>\n",
              "      <td>0.049275</td>\n",
              "      <td>-0.058812</td>\n",
              "      <td>-0.026589</td>\n",
              "      <td>-0.030488</td>\n",
              "      <td>0.002919</td>\n",
              "      <td>-0.033887</td>\n",
              "      <td>-0.022722</td>\n",
              "      <td>-0.007847</td>\n",
              "      <td>-0.028247</td>\n",
              "      <td>0.007928</td>\n",
              "      <td>-0.024120</td>\n",
              "      <td>-0.014729</td>\n",
              "      <td>0.013604</td>\n",
              "      <td>0.016520</td>\n",
              "      <td>-0.045117</td>\n",
              "      <td>-0.003871</td>\n",
              "      <td>-0.029207</td>\n",
              "      <td>-0.008502</td>\n",
              "      <td>-0.001047</td>\n",
              "      <td>-0.020061</td>\n",
              "      <td>0.015250</td>\n",
              "      <td>0.009938</td>\n",
              "      <td>0.004786</td>\n",
              "      <td>0.007806</td>\n",
              "      <td>0.011801</td>\n",
              "      <td>-0.011018</td>\n",
              "      <td>-0.003670</td>\n",
              "      <td>-0.013831</td>\n",
              "      <td>0.003276</td>\n",
              "      <td>-0.003831</td>\n",
              "      <td>0.025561</td>\n",
              "      <td>-0.009699</td>\n",
              "      <td>0.005768</td>\n",
              "      <td>-0.009950</td>\n",
              "      <td>-0.002146</td>\n",
              "      <td>0.015221</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.266160</td>\n",
              "      <td>0.166972</td>\n",
              "      <td>0.254764</td>\n",
              "      <td>0.193289</td>\n",
              "      <td>0.241754</td>\n",
              "      <td>0.109078</td>\n",
              "      <td>0.174927</td>\n",
              "      <td>0.169065</td>\n",
              "      <td>0.193289</td>\n",
              "      <td>0.173140</td>\n",
              "      <td>0.098100</td>\n",
              "      <td>0.180645</td>\n",
              "      <td>0.125638</td>\n",
              "      <td>0.157089</td>\n",
              "      <td>0.134833</td>\n",
              "      <td>0.126750</td>\n",
              "      <td>0.150886</td>\n",
              "      <td>0.149021</td>\n",
              "      <td>0.104876</td>\n",
              "      <td>0.096763</td>\n",
              "      <td>0.076813</td>\n",
              "      <td>0.115245</td>\n",
              "      <td>0.098988</td>\n",
              "      <td>0.092810</td>\n",
              "      <td>0.118545</td>\n",
              "      <td>0.095405</td>\n",
              "      <td>0.108273</td>\n",
              "      <td>0.110751</td>\n",
              "      <td>0.081606</td>\n",
              "      <td>0.127006</td>\n",
              "      <td>0.097460</td>\n",
              "      <td>0.102299</td>\n",
              "      <td>0.105051</td>\n",
              "      <td>0.107226</td>\n",
              "      <td>0.089180</td>\n",
              "      <td>0.080794</td>\n",
              "      <td>0.093440</td>\n",
              "      <td>0.075128</td>\n",
              "      <td>0.092280</td>\n",
              "      <td>0.085729</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>13.200000</td>\n",
              "      <td>40.900000</td>\n",
              "      <td>82.00000</td>\n",
              "      <td>99.600000</td>\n",
              "      <td>99.600000</td>\n",
              "      <td>408.000000</td>\n",
              "      <td>42.000000</td>\n",
              "      <td>99.800000</td>\n",
              "      <td>24.800000</td>\n",
              "      <td>22.900000</td>\n",
              "      <td>5.050000</td>\n",
              "      <td>3.830000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>341.000000</td>\n",
              "      <td>369.000000</td>\n",
              "      <td>62.800000</td>\n",
              "      <td>96.700000</td>\n",
              "      <td>1640.000000</td>\n",
              "      <td>99.800000</td>\n",
              "      <td>0.820000</td>\n",
              "      <td>11.300000</td>\n",
              "      <td>2.900000</td>\n",
              "      <td>8.300000</td>\n",
              "      <td>94.000000</td>\n",
              "      <td>1260.000000</td>\n",
              "      <td>9.280000</td>\n",
              "      <td>357.000000</td>\n",
              "      <td>982.000000</td>\n",
              "      <td>2460.000000</td>\n",
              "      <td>40700.000000</td>\n",
              "      <td>277.000000</td>\n",
              "      <td>7.440000</td>\n",
              "      <td>81.000000</td>\n",
              "      <td>1.850000</td>\n",
              "      <td>5080.000000</td>\n",
              "      <td>3.440000</td>\n",
              "      <td>1640.000000</td>\n",
              "      <td>81.300000</td>\n",
              "      <td>912.000000</td>\n",
              "      <td>316.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>1.068850</td>\n",
              "      <td>1.167370</td>\n",
              "      <td>1.230920</td>\n",
              "      <td>1.266974</td>\n",
              "      <td>1.128774</td>\n",
              "      <td>1.031444</td>\n",
              "      <td>0.752794</td>\n",
              "      <td>0.805872</td>\n",
              "      <td>1.049345</td>\n",
              "      <td>0.867322</td>\n",
              "      <td>1.478752</td>\n",
              "      <td>0.881879</td>\n",
              "      <td>0.756606</td>\n",
              "      <td>0.763748</td>\n",
              "      <td>0.878603</td>\n",
              "      <td>0.970210</td>\n",
              "      <td>0.777881</td>\n",
              "      <td>0.770679</td>\n",
              "      <td>0.961082</td>\n",
              "      <td>0.849681</td>\n",
              "      <td>0.576198</td>\n",
              "      <td>0.609812</td>\n",
              "      <td>0.676092</td>\n",
              "      <td>0.818344</td>\n",
              "      <td>0.675452</td>\n",
              "      <td>0.680991</td>\n",
              "      <td>0.846139</td>\n",
              "      <td>0.700253</td>\n",
              "      <td>0.928878</td>\n",
              "      <td>0.511358</td>\n",
              "      <td>0.702707</td>\n",
              "      <td>0.620772</td>\n",
              "      <td>0.572546</td>\n",
              "      <td>0.701231</td>\n",
              "      <td>0.621155</td>\n",
              "      <td>0.483569</td>\n",
              "      <td>0.644586</td>\n",
              "      <td>0.563413</td>\n",
              "      <td>0.606385</td>\n",
              "      <td>0.716600</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8 rows × 248 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "               0            1    ...          246          247\n",
              "count  2094.000000  2057.000000  ...  2621.000000  2621.000000\n",
              "mean      0.272364     1.939626  ...     0.009037     0.004557\n",
              "std       1.096187     6.569316  ...     0.149526     0.149701\n",
              "min       0.000000     0.000000  ...    -0.568859    -0.486389\n",
              "25%       0.000000     0.000000  ...    -0.091850    -0.094435\n",
              "50%       0.000000     0.000000  ...    -0.002146     0.015221\n",
              "75%       0.000000     0.000000  ...     0.092280     0.085729\n",
              "max      13.200000    40.900000  ...     0.606385     0.716600\n",
              "\n",
              "[8 rows x 248 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zkLPiTE-GFBL"
      },
      "source": [
        "# traditional methods test sets - impute across all foods\n",
        "X_train = pd.DataFrame(missed_data_train, columns = columns)\n",
        "X_test_mean_all = pd.DataFrame(missed_data, columns = columns)\n",
        "X_test_median_all = pd.DataFrame(missed_data, columns = columns)\n",
        "\n",
        "X_test_mean_all = impute_traditional_all_foods(X_train, X_test_mean_all, 'mean')\n",
        "X_test_median_all = impute_traditional_all_foods(X_train, X_test_median_all, 'median')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RaZXZSxJGFBL"
      },
      "source": [
        "# Traditional methods test sets - impute by food category\n",
        "X_test_mean = pd.DataFrame(missed_data, columns = columns, index=indices)\n",
        "X_test_median = X_test_mean.copy()\n",
        "df_mean_cat = df_mean[df_mean['fdc_id'].isin(indices)]\n",
        "X_test_mean = impute_traditional_by_category(X_test_mean, df_mean_cat, indices)\n",
        "df_median_cat = df_median[df_median['fdc_id'].isin(indices)]\n",
        "X_test_median = impute_traditional_by_category(X_test_median, df_median_cat, indices)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7P8HqZv7GFBM"
      },
      "source": [
        "scaler = MinMaxScaler()\n",
        "scaler.fit(train_data)\n",
        "train_data = scaler.transform(train_data)\n",
        "test_data = scaler.transform(test_data)\n",
        "X_test_mean = scaler.transform(X_test_mean)\n",
        "X_test_median = scaler.transform(X_test_median)\n",
        "X_test_mean_all = scaler.transform(X_test_mean_all)\n",
        "X_test_median_all = scaler.transform(X_test_median_all)\n",
        "\n",
        "y_test = scaler.transform(y_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yI6EI2dDGFBM"
      },
      "source": [
        "# datasets without embeddings for autoencoder\n",
        "train_data_noEmbeddings = pd.DataFrame(train_data, columns = columns)\n",
        "train_data_noEmbeddings = np.array(train_data_noEmbeddings.iloc[:, :-num_embeddings]) # df = df.iloc[: , :-1]\n",
        "test_mean_noEmbeddings = np.array(pd.DataFrame(X_test_mean, columns = columns).iloc[:, :-num_embeddings])\n",
        "test_median_noEmbeddings = np.array(pd.DataFrame(X_test_median, columns = columns).iloc[:, :-num_embeddings])\n",
        "test_mean_noEmbeddings_all = np.array(pd.DataFrame(X_test_mean_all, columns = columns).iloc[:, :-num_embeddings])\n",
        "test_median_noEmbeddings_all = np.array(pd.DataFrame(X_test_median_all, columns = columns).iloc[:, :-num_embeddings])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OSXGHvo2GFBM"
      },
      "source": [
        "missed_data = X_test_mean_all\n",
        "missed_data = torch.from_numpy(missed_data).float()\n",
        "\n",
        "train_data = torch.from_numpy(train_data).float()\n",
        "\n",
        "train_loader = torch.utils.data.DataLoader(dataset=train_data,\n",
        "                                           batch_size=batch_size,\n",
        "                                           shuffle=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BiqY-yRaGFBM"
      },
      "source": [
        "missed_data_noEm = test_mean_noEmbeddings_all\n",
        "missed_data_noEm = torch.from_numpy(missed_data_noEm).float()\n",
        "\n",
        "train_data_noEmbeddings = torch.from_numpy(train_data_noEmbeddings).float()\n",
        "\n",
        "train_loader_noEm = torch.utils.data.DataLoader(dataset=train_data_noEmbeddings,\n",
        "                                           batch_size=batch_size,\n",
        "                                           shuffle=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jvmUXv2LGFBN"
      },
      "source": [
        "## Autoencoder - without embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "54aBP3c4GFBN"
      },
      "source": [
        "model = Autoencoder(dim=cols-num_embeddings).to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tOmy-1iuGFBN"
      },
      "source": [
        "criterion = nn.MSELoss()\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O-xNO7PcGFBN",
        "outputId": "20e58345-7a69-45f6-c408-9c7b03fd6c01"
      },
      "source": [
        "training(num_epochs, model, train_loader_noEm, criterion, optimizer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch : 1/200, recon loss = 0.00376239\n",
            "epoch : 2/200, recon loss = 0.00293869\n",
            "epoch : 3/200, recon loss = 0.00290759\n",
            "epoch : 4/200, recon loss = 0.00288642\n",
            "epoch : 5/200, recon loss = 0.00285750\n",
            "epoch : 6/200, recon loss = 0.00281565\n",
            "epoch : 7/200, recon loss = 0.00275599\n",
            "epoch : 8/200, recon loss = 0.00266495\n",
            "epoch : 9/200, recon loss = 0.00252990\n",
            "epoch : 10/200, recon loss = 0.00233611\n",
            "epoch : 11/200, recon loss = 0.00214065\n",
            "epoch : 12/200, recon loss = 0.00196707\n",
            "epoch : 13/200, recon loss = 0.00185716\n",
            "epoch : 14/200, recon loss = 0.00178805\n",
            "epoch : 15/200, recon loss = 0.00174686\n",
            "epoch : 16/200, recon loss = 0.00172417\n",
            "epoch : 17/200, recon loss = 0.00171114\n",
            "epoch : 18/200, recon loss = 0.00169873\n",
            "epoch : 19/200, recon loss = 0.00168953\n",
            "epoch : 20/200, recon loss = 0.00168483\n",
            "epoch : 21/200, recon loss = 0.00168238\n",
            "epoch : 22/200, recon loss = 0.00167891\n",
            "epoch : 23/200, recon loss = 0.00167378\n",
            "epoch : 24/200, recon loss = 0.00167387\n",
            "epoch : 25/200, recon loss = 0.00166987\n",
            "epoch : 26/200, recon loss = 0.00166427\n",
            "epoch : 27/200, recon loss = 0.00166056\n",
            "epoch : 28/200, recon loss = 0.00165728\n",
            "epoch : 29/200, recon loss = 0.00165303\n",
            "epoch : 30/200, recon loss = 0.00164928\n",
            "epoch : 31/200, recon loss = 0.00164424\n",
            "epoch : 32/200, recon loss = 0.00164147\n",
            "epoch : 33/200, recon loss = 0.00163247\n",
            "epoch : 34/200, recon loss = 0.00162558\n",
            "epoch : 35/200, recon loss = 0.00162162\n",
            "epoch : 36/200, recon loss = 0.00161218\n",
            "epoch : 37/200, recon loss = 0.00160455\n",
            "epoch : 38/200, recon loss = 0.00159689\n",
            "epoch : 39/200, recon loss = 0.00159031\n",
            "epoch : 40/200, recon loss = 0.00157853\n",
            "epoch : 41/200, recon loss = 0.00156546\n",
            "epoch : 42/200, recon loss = 0.00155261\n",
            "epoch : 43/200, recon loss = 0.00154730\n",
            "epoch : 44/200, recon loss = 0.00153094\n",
            "epoch : 45/200, recon loss = 0.00151134\n",
            "epoch : 46/200, recon loss = 0.00150357\n",
            "epoch : 47/200, recon loss = 0.00148505\n",
            "epoch : 48/200, recon loss = 0.00146632\n",
            "epoch : 49/200, recon loss = 0.00145693\n",
            "epoch : 50/200, recon loss = 0.00143861\n",
            "epoch : 51/200, recon loss = 0.00142198\n",
            "epoch : 52/200, recon loss = 0.00140859\n",
            "epoch : 53/200, recon loss = 0.00139422\n",
            "epoch : 54/200, recon loss = 0.00137905\n",
            "epoch : 55/200, recon loss = 0.00136422\n",
            "epoch : 56/200, recon loss = 0.00135093\n",
            "epoch : 57/200, recon loss = 0.00132914\n",
            "epoch : 58/200, recon loss = 0.00132432\n",
            "epoch : 59/200, recon loss = 0.00131017\n",
            "epoch : 60/200, recon loss = 0.00129823\n",
            "epoch : 61/200, recon loss = 0.00129038\n",
            "epoch : 62/200, recon loss = 0.00127912\n",
            "epoch : 63/200, recon loss = 0.00127361\n",
            "epoch : 64/200, recon loss = 0.00126511\n",
            "epoch : 65/200, recon loss = 0.00126045\n",
            "epoch : 66/200, recon loss = 0.00124794\n",
            "epoch : 67/200, recon loss = 0.00124236\n",
            "epoch : 68/200, recon loss = 0.00123751\n",
            "epoch : 69/200, recon loss = 0.00122896\n",
            "epoch : 70/200, recon loss = 0.00123038\n",
            "epoch : 71/200, recon loss = 0.00122085\n",
            "epoch : 72/200, recon loss = 0.00121294\n",
            "epoch : 73/200, recon loss = 0.00121110\n",
            "epoch : 74/200, recon loss = 0.00120836\n",
            "epoch : 75/200, recon loss = 0.00120401\n",
            "epoch : 76/200, recon loss = 0.00119742\n",
            "epoch : 77/200, recon loss = 0.00119714\n",
            "epoch : 78/200, recon loss = 0.00119600\n",
            "epoch : 79/200, recon loss = 0.00118555\n",
            "epoch : 80/200, recon loss = 0.00119439\n",
            "epoch : 81/200, recon loss = 0.00118372\n",
            "epoch : 82/200, recon loss = 0.00118660\n",
            "epoch : 83/200, recon loss = 0.00118353\n",
            "epoch : 84/200, recon loss = 0.00117835\n",
            "epoch : 85/200, recon loss = 0.00117871\n",
            "epoch : 86/200, recon loss = 0.00118167\n",
            "epoch : 87/200, recon loss = 0.00117505\n",
            "epoch : 88/200, recon loss = 0.00117551\n",
            "epoch : 89/200, recon loss = 0.00117677\n",
            "epoch : 90/200, recon loss = 0.00117432\n",
            "epoch : 91/200, recon loss = 0.00116731\n",
            "epoch : 92/200, recon loss = 0.00116405\n",
            "epoch : 93/200, recon loss = 0.00116727\n",
            "epoch : 94/200, recon loss = 0.00116639\n",
            "epoch : 95/200, recon loss = 0.00116391\n",
            "epoch : 96/200, recon loss = 0.00115720\n",
            "epoch : 97/200, recon loss = 0.00116404\n",
            "epoch : 98/200, recon loss = 0.00115853\n",
            "epoch : 99/200, recon loss = 0.00116037\n",
            "epoch : 100/200, recon loss = 0.00115635\n",
            "epoch : 101/200, recon loss = 0.00115190\n",
            "epoch : 102/200, recon loss = 0.00114479\n",
            "epoch : 103/200, recon loss = 0.00115298\n",
            "epoch : 104/200, recon loss = 0.00114764\n",
            "epoch : 105/200, recon loss = 0.00115301\n",
            "epoch : 106/200, recon loss = 0.00114245\n",
            "epoch : 107/200, recon loss = 0.00113964\n",
            "epoch : 108/200, recon loss = 0.00114399\n",
            "epoch : 109/200, recon loss = 0.00113150\n",
            "epoch : 110/200, recon loss = 0.00113328\n",
            "epoch : 111/200, recon loss = 0.00113554\n",
            "epoch : 112/200, recon loss = 0.00112817\n",
            "epoch : 113/200, recon loss = 0.00113175\n",
            "epoch : 114/200, recon loss = 0.00113170\n",
            "epoch : 115/200, recon loss = 0.00112447\n",
            "epoch : 116/200, recon loss = 0.00112003\n",
            "epoch : 117/200, recon loss = 0.00112214\n",
            "epoch : 118/200, recon loss = 0.00112777\n",
            "epoch : 119/200, recon loss = 0.00111619\n",
            "epoch : 120/200, recon loss = 0.00111907\n",
            "epoch : 121/200, recon loss = 0.00111175\n",
            "epoch : 122/200, recon loss = 0.00111217\n",
            "epoch : 123/200, recon loss = 0.00111229\n",
            "epoch : 124/200, recon loss = 0.00110917\n",
            "epoch : 125/200, recon loss = 0.00111195\n",
            "epoch : 126/200, recon loss = 0.00110871\n",
            "epoch : 127/200, recon loss = 0.00110821\n",
            "epoch : 128/200, recon loss = 0.00110096\n",
            "epoch : 129/200, recon loss = 0.00110183\n",
            "epoch : 130/200, recon loss = 0.00110162\n",
            "epoch : 131/200, recon loss = 0.00108839\n",
            "epoch : 132/200, recon loss = 0.00108760\n",
            "epoch : 133/200, recon loss = 0.00109340\n",
            "epoch : 134/200, recon loss = 0.00108367\n",
            "epoch : 135/200, recon loss = 0.00109163\n",
            "epoch : 136/200, recon loss = 0.00107803\n",
            "epoch : 137/200, recon loss = 0.00108627\n",
            "epoch : 138/200, recon loss = 0.00108246\n",
            "epoch : 139/200, recon loss = 0.00108235\n",
            "epoch : 140/200, recon loss = 0.00107837\n",
            "epoch : 141/200, recon loss = 0.00108367\n",
            "epoch : 142/200, recon loss = 0.00107154\n",
            "epoch : 143/200, recon loss = 0.00107544\n",
            "epoch : 144/200, recon loss = 0.00107390\n",
            "epoch : 145/200, recon loss = 0.00107280\n",
            "epoch : 146/200, recon loss = 0.00106813\n",
            "epoch : 147/200, recon loss = 0.00107025\n",
            "epoch : 148/200, recon loss = 0.00106861\n",
            "epoch : 149/200, recon loss = 0.00107352\n",
            "epoch : 150/200, recon loss = 0.00107206\n",
            "epoch : 151/200, recon loss = 0.00106639\n",
            "epoch : 152/200, recon loss = 0.00106170\n",
            "epoch : 153/200, recon loss = 0.00106162\n",
            "epoch : 154/200, recon loss = 0.00105644\n",
            "epoch : 155/200, recon loss = 0.00105381\n",
            "epoch : 156/200, recon loss = 0.00105999\n",
            "epoch : 157/200, recon loss = 0.00105512\n",
            "epoch : 158/200, recon loss = 0.00105423\n",
            "epoch : 159/200, recon loss = 0.00105112\n",
            "epoch : 160/200, recon loss = 0.00105305\n",
            "epoch : 161/200, recon loss = 0.00105114\n",
            "epoch : 162/200, recon loss = 0.00104725\n",
            "epoch : 163/200, recon loss = 0.00105040\n",
            "epoch : 164/200, recon loss = 0.00104062\n",
            "epoch : 165/200, recon loss = 0.00104327\n",
            "epoch : 166/200, recon loss = 0.00104889\n",
            "epoch : 167/200, recon loss = 0.00104343\n",
            "epoch : 168/200, recon loss = 0.00103173\n",
            "epoch : 169/200, recon loss = 0.00103961\n",
            "epoch : 170/200, recon loss = 0.00103932\n",
            "epoch : 171/200, recon loss = 0.00103530\n",
            "epoch : 172/200, recon loss = 0.00104053\n",
            "epoch : 173/200, recon loss = 0.00103051\n",
            "epoch : 174/200, recon loss = 0.00103928\n",
            "epoch : 175/200, recon loss = 0.00102861\n",
            "epoch : 176/200, recon loss = 0.00103446\n",
            "epoch : 177/200, recon loss = 0.00102741\n",
            "epoch : 178/200, recon loss = 0.00102792\n",
            "epoch : 179/200, recon loss = 0.00102569\n",
            "epoch : 180/200, recon loss = 0.00102362\n",
            "epoch : 181/200, recon loss = 0.00102560\n",
            "epoch : 182/200, recon loss = 0.00102149\n",
            "epoch : 183/200, recon loss = 0.00101022\n",
            "epoch : 184/200, recon loss = 0.00101913\n",
            "epoch : 185/200, recon loss = 0.00101911\n",
            "epoch : 186/200, recon loss = 0.00100915\n",
            "epoch : 187/200, recon loss = 0.00100908\n",
            "epoch : 188/200, recon loss = 0.00101552\n",
            "epoch : 189/200, recon loss = 0.00101886\n",
            "epoch : 190/200, recon loss = 0.00101802\n",
            "epoch : 191/200, recon loss = 0.00100425\n",
            "epoch : 192/200, recon loss = 0.00100644\n",
            "epoch : 193/200, recon loss = 0.00100309\n",
            "epoch : 194/200, recon loss = 0.00101018\n",
            "epoch : 195/200, recon loss = 0.00100608\n",
            "epoch : 196/200, recon loss = 0.00100147\n",
            "epoch : 197/200, recon loss = 0.00100313\n",
            "epoch : 198/200, recon loss = 0.00100134\n",
            "epoch : 199/200, recon loss = 0.00100524\n",
            "epoch : 200/200, recon loss = 0.00100131\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IoDV7kX5GFBN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "389bb5e0-6f8b-460a-80b8-f2cf215079ee"
      },
      "source": [
        "model.eval()\n",
        "rmse_sum = 0\n",
        "\n",
        "filled_data = model(missed_data_noEm.to(device))\n",
        "filled_data = filled_data.cpu().detach().numpy()\n",
        "\n",
        "rmse_sum = rmse_error(test_data, filled_data, cols-num_embeddings, mask)\n",
        "\n",
        "print(rmse_sum)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "5.215949090468736\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ita_CeF3GFBO"
      },
      "source": [
        "## Autoencoder - with embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yAW8PRsVGFBO"
      },
      "source": [
        "model = Autoencoder(dim=cols).to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JQcqURM9GFBO"
      },
      "source": [
        "criterion = nn.MSELoss()\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V-7NDl9HGFBO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3f861048-ec1c-4d0b-f3d8-3d2cdc9e996d"
      },
      "source": [
        "training(num_epochs, model, train_loader, criterion, optimizer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch : 1/200, recon loss = 0.01425203\n",
            "epoch : 2/200, recon loss = 0.00729474\n",
            "epoch : 3/200, recon loss = 0.00727854\n",
            "epoch : 4/200, recon loss = 0.00726253\n",
            "epoch : 5/200, recon loss = 0.00724003\n",
            "epoch : 6/200, recon loss = 0.00721195\n",
            "epoch : 7/200, recon loss = 0.00717784\n",
            "epoch : 8/200, recon loss = 0.00713054\n",
            "epoch : 9/200, recon loss = 0.00706220\n",
            "epoch : 10/200, recon loss = 0.00696389\n",
            "epoch : 11/200, recon loss = 0.00682904\n",
            "epoch : 12/200, recon loss = 0.00666035\n",
            "epoch : 13/200, recon loss = 0.00648561\n",
            "epoch : 14/200, recon loss = 0.00634780\n",
            "epoch : 15/200, recon loss = 0.00624832\n",
            "epoch : 16/200, recon loss = 0.00618158\n",
            "epoch : 17/200, recon loss = 0.00613688\n",
            "epoch : 18/200, recon loss = 0.00610990\n",
            "epoch : 19/200, recon loss = 0.00608614\n",
            "epoch : 20/200, recon loss = 0.00607367\n",
            "epoch : 21/200, recon loss = 0.00606013\n",
            "epoch : 22/200, recon loss = 0.00605006\n",
            "epoch : 23/200, recon loss = 0.00604083\n",
            "epoch : 24/200, recon loss = 0.00603483\n",
            "epoch : 25/200, recon loss = 0.00602478\n",
            "epoch : 26/200, recon loss = 0.00601923\n",
            "epoch : 27/200, recon loss = 0.00601234\n",
            "epoch : 28/200, recon loss = 0.00600451\n",
            "epoch : 29/200, recon loss = 0.00599319\n",
            "epoch : 30/200, recon loss = 0.00598654\n",
            "epoch : 31/200, recon loss = 0.00597812\n",
            "epoch : 32/200, recon loss = 0.00596559\n",
            "epoch : 33/200, recon loss = 0.00596005\n",
            "epoch : 34/200, recon loss = 0.00594373\n",
            "epoch : 35/200, recon loss = 0.00593367\n",
            "epoch : 36/200, recon loss = 0.00592035\n",
            "epoch : 37/200, recon loss = 0.00590587\n",
            "epoch : 38/200, recon loss = 0.00589365\n",
            "epoch : 39/200, recon loss = 0.00587737\n",
            "epoch : 40/200, recon loss = 0.00586097\n",
            "epoch : 41/200, recon loss = 0.00584357\n",
            "epoch : 42/200, recon loss = 0.00582752\n",
            "epoch : 43/200, recon loss = 0.00580776\n",
            "epoch : 44/200, recon loss = 0.00579117\n",
            "epoch : 45/200, recon loss = 0.00577078\n",
            "epoch : 46/200, recon loss = 0.00575982\n",
            "epoch : 47/200, recon loss = 0.00573786\n",
            "epoch : 48/200, recon loss = 0.00572194\n",
            "epoch : 49/200, recon loss = 0.00570946\n",
            "epoch : 50/200, recon loss = 0.00569229\n",
            "epoch : 51/200, recon loss = 0.00567377\n",
            "epoch : 52/200, recon loss = 0.00566119\n",
            "epoch : 53/200, recon loss = 0.00564561\n",
            "epoch : 54/200, recon loss = 0.00563498\n",
            "epoch : 55/200, recon loss = 0.00561618\n",
            "epoch : 56/200, recon loss = 0.00560931\n",
            "epoch : 57/200, recon loss = 0.00559104\n",
            "epoch : 58/200, recon loss = 0.00557781\n",
            "epoch : 59/200, recon loss = 0.00556966\n",
            "epoch : 60/200, recon loss = 0.00554995\n",
            "epoch : 61/200, recon loss = 0.00553949\n",
            "epoch : 62/200, recon loss = 0.00553010\n",
            "epoch : 63/200, recon loss = 0.00550881\n",
            "epoch : 64/200, recon loss = 0.00549645\n",
            "epoch : 65/200, recon loss = 0.00548328\n",
            "epoch : 66/200, recon loss = 0.00546838\n",
            "epoch : 67/200, recon loss = 0.00545147\n",
            "epoch : 68/200, recon loss = 0.00543791\n",
            "epoch : 69/200, recon loss = 0.00541935\n",
            "epoch : 70/200, recon loss = 0.00541008\n",
            "epoch : 71/200, recon loss = 0.00538261\n",
            "epoch : 72/200, recon loss = 0.00537420\n",
            "epoch : 73/200, recon loss = 0.00535269\n",
            "epoch : 74/200, recon loss = 0.00534577\n",
            "epoch : 75/200, recon loss = 0.00531623\n",
            "epoch : 76/200, recon loss = 0.00530915\n",
            "epoch : 77/200, recon loss = 0.00529012\n",
            "epoch : 78/200, recon loss = 0.00527743\n",
            "epoch : 79/200, recon loss = 0.00526118\n",
            "epoch : 80/200, recon loss = 0.00524865\n",
            "epoch : 81/200, recon loss = 0.00523737\n",
            "epoch : 82/200, recon loss = 0.00523516\n",
            "epoch : 83/200, recon loss = 0.00519935\n",
            "epoch : 84/200, recon loss = 0.00519654\n",
            "epoch : 85/200, recon loss = 0.00517561\n",
            "epoch : 86/200, recon loss = 0.00518259\n",
            "epoch : 87/200, recon loss = 0.00515858\n",
            "epoch : 88/200, recon loss = 0.00514555\n",
            "epoch : 89/200, recon loss = 0.00514630\n",
            "epoch : 90/200, recon loss = 0.00513738\n",
            "epoch : 91/200, recon loss = 0.00512398\n",
            "epoch : 92/200, recon loss = 0.00510947\n",
            "epoch : 93/200, recon loss = 0.00509480\n",
            "epoch : 94/200, recon loss = 0.00509544\n",
            "epoch : 95/200, recon loss = 0.00508736\n",
            "epoch : 96/200, recon loss = 0.00508676\n",
            "epoch : 97/200, recon loss = 0.00506670\n",
            "epoch : 98/200, recon loss = 0.00505739\n",
            "epoch : 99/200, recon loss = 0.00505074\n",
            "epoch : 100/200, recon loss = 0.00503704\n",
            "epoch : 101/200, recon loss = 0.00504494\n",
            "epoch : 102/200, recon loss = 0.00503324\n",
            "epoch : 103/200, recon loss = 0.00502059\n",
            "epoch : 104/200, recon loss = 0.00502635\n",
            "epoch : 105/200, recon loss = 0.00500397\n",
            "epoch : 106/200, recon loss = 0.00501517\n",
            "epoch : 107/200, recon loss = 0.00499504\n",
            "epoch : 108/200, recon loss = 0.00498969\n",
            "epoch : 109/200, recon loss = 0.00498790\n",
            "epoch : 110/200, recon loss = 0.00497876\n",
            "epoch : 111/200, recon loss = 0.00497774\n",
            "epoch : 112/200, recon loss = 0.00497004\n",
            "epoch : 113/200, recon loss = 0.00495718\n",
            "epoch : 114/200, recon loss = 0.00494891\n",
            "epoch : 115/200, recon loss = 0.00494216\n",
            "epoch : 116/200, recon loss = 0.00494864\n",
            "epoch : 117/200, recon loss = 0.00493493\n",
            "epoch : 118/200, recon loss = 0.00494290\n",
            "epoch : 119/200, recon loss = 0.00492829\n",
            "epoch : 120/200, recon loss = 0.00491803\n",
            "epoch : 121/200, recon loss = 0.00491117\n",
            "epoch : 122/200, recon loss = 0.00491292\n",
            "epoch : 123/200, recon loss = 0.00490537\n",
            "epoch : 124/200, recon loss = 0.00489040\n",
            "epoch : 125/200, recon loss = 0.00487956\n",
            "epoch : 126/200, recon loss = 0.00487980\n",
            "epoch : 127/200, recon loss = 0.00487886\n",
            "epoch : 128/200, recon loss = 0.00487050\n",
            "epoch : 129/200, recon loss = 0.00486564\n",
            "epoch : 130/200, recon loss = 0.00486516\n",
            "epoch : 131/200, recon loss = 0.00486460\n",
            "epoch : 132/200, recon loss = 0.00486543\n",
            "epoch : 133/200, recon loss = 0.00483987\n",
            "epoch : 134/200, recon loss = 0.00485234\n",
            "epoch : 135/200, recon loss = 0.00482612\n",
            "epoch : 136/200, recon loss = 0.00483727\n",
            "epoch : 137/200, recon loss = 0.00482024\n",
            "epoch : 138/200, recon loss = 0.00482275\n",
            "epoch : 139/200, recon loss = 0.00481800\n",
            "epoch : 140/200, recon loss = 0.00481540\n",
            "epoch : 141/200, recon loss = 0.00480161\n",
            "epoch : 142/200, recon loss = 0.00479168\n",
            "epoch : 143/200, recon loss = 0.00479894\n",
            "epoch : 144/200, recon loss = 0.00478407\n",
            "epoch : 145/200, recon loss = 0.00477690\n",
            "epoch : 146/200, recon loss = 0.00478641\n",
            "epoch : 147/200, recon loss = 0.00476755\n",
            "epoch : 148/200, recon loss = 0.00477008\n",
            "epoch : 149/200, recon loss = 0.00477305\n",
            "epoch : 150/200, recon loss = 0.00475728\n",
            "epoch : 151/200, recon loss = 0.00476371\n",
            "epoch : 152/200, recon loss = 0.00475672\n",
            "epoch : 153/200, recon loss = 0.00474428\n",
            "epoch : 154/200, recon loss = 0.00473444\n",
            "epoch : 155/200, recon loss = 0.00474479\n",
            "epoch : 156/200, recon loss = 0.00473564\n",
            "epoch : 157/200, recon loss = 0.00472716\n",
            "epoch : 158/200, recon loss = 0.00472146\n",
            "epoch : 159/200, recon loss = 0.00472725\n",
            "epoch : 160/200, recon loss = 0.00471851\n",
            "epoch : 161/200, recon loss = 0.00471759\n",
            "epoch : 162/200, recon loss = 0.00470151\n",
            "epoch : 163/200, recon loss = 0.00469994\n",
            "epoch : 164/200, recon loss = 0.00469551\n",
            "epoch : 165/200, recon loss = 0.00467991\n",
            "epoch : 166/200, recon loss = 0.00468829\n",
            "epoch : 167/200, recon loss = 0.00468449\n",
            "epoch : 168/200, recon loss = 0.00468084\n",
            "epoch : 169/200, recon loss = 0.00467390\n",
            "epoch : 170/200, recon loss = 0.00468351\n",
            "epoch : 171/200, recon loss = 0.00467881\n",
            "epoch : 172/200, recon loss = 0.00467240\n",
            "epoch : 173/200, recon loss = 0.00464757\n",
            "epoch : 174/200, recon loss = 0.00464134\n",
            "epoch : 175/200, recon loss = 0.00464644\n",
            "epoch : 176/200, recon loss = 0.00464206\n",
            "epoch : 177/200, recon loss = 0.00462888\n",
            "epoch : 178/200, recon loss = 0.00464186\n",
            "epoch : 179/200, recon loss = 0.00463612\n",
            "epoch : 180/200, recon loss = 0.00463489\n",
            "epoch : 181/200, recon loss = 0.00462229\n",
            "epoch : 182/200, recon loss = 0.00462153\n",
            "epoch : 183/200, recon loss = 0.00461057\n",
            "epoch : 184/200, recon loss = 0.00461521\n",
            "epoch : 185/200, recon loss = 0.00460071\n",
            "epoch : 186/200, recon loss = 0.00461094\n",
            "epoch : 187/200, recon loss = 0.00459102\n",
            "epoch : 188/200, recon loss = 0.00459748\n",
            "epoch : 189/200, recon loss = 0.00458719\n",
            "epoch : 190/200, recon loss = 0.00458950\n",
            "epoch : 191/200, recon loss = 0.00457908\n",
            "epoch : 192/200, recon loss = 0.00457453\n",
            "epoch : 193/200, recon loss = 0.00457912\n",
            "epoch : 194/200, recon loss = 0.00457121\n",
            "epoch : 195/200, recon loss = 0.00456146\n",
            "epoch : 196/200, recon loss = 0.00455226\n",
            "epoch : 197/200, recon loss = 0.00456437\n",
            "epoch : 198/200, recon loss = 0.00454988\n",
            "epoch : 199/200, recon loss = 0.00453961\n",
            "epoch : 200/200, recon loss = 0.00453764\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QKAvWoGkGFBO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d7609eba-8069-4bc1-b7af-d8d8ba54cfc4"
      },
      "source": [
        "model.eval()\n",
        "rmse_sum = 0\n",
        "\n",
        "filled_data = model(missed_data.to(device))\n",
        "filled_data = filled_data.cpu().detach().numpy()\n",
        "\n",
        "rmse_sum = rmse_error(test_data, filled_data, cols, mask)\n",
        "\n",
        "print(rmse_sum)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "5.712720665341518\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IDSkxq4bGFBP"
      },
      "source": [
        "## Mean and Median Imputation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9sns3b64GFBS"
      },
      "source": [
        "### By category"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-_HsixUhGFBS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "915605a8-cd11-4f54-9042-c7f712365e27"
      },
      "source": [
        "imputed = X_test_mean\n",
        "error = rmse_error(y_test, imputed, cols-num_embeddings, mask)\n",
        "print(error)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "6.562400490537268\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0WtatYBfGFBS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e9faf68a-77ef-4152-bb0d-d23f055d6ecc"
      },
      "source": [
        "imputed = X_test_median\n",
        "error = rmse_error(y_test, imputed, cols-num_embeddings, mask)\n",
        "print(error)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "6.729803902928729\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JJ06kE4zGFBT"
      },
      "source": [
        "### Across all foods"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Eq0KaVJiGFBT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "19369369-d370-4896-dfda-99f5068eb603"
      },
      "source": [
        "imputed = X_test_mean_all\n",
        "error = rmse_error(y_test, imputed, cols-num_embeddings, mask)\n",
        "print(error)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "7.4467183945414215\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tlzvU_-nGFBT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "01efdbf8-435b-4781-fc8b-6c2b50d9a5dd"
      },
      "source": [
        "imputed = X_test_median_all\n",
        "error = rmse_error(y_test, imputed, cols-num_embeddings, mask)\n",
        "print(error)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "7.573344537892341\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "feRrIFmiWSNC"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}