spark3_regression.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.8"
    },
    "colab": {
      "provenance": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "nearby-chancellor"
      },
      "source": [
        "# Regression using pyspark"
      ],
      "id": "nearby-chancellor"
    },
    {
      "cell_type": "code",
      "source": [
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null # install Java8\n",
        "!wget -q https://archive.apache.org/dist/spark/spark-3.1.1/spark-3.1.1-bin-hadoop2.7.tgz # download spark 3.1.1\n",
        "!tar xf spark-3.1.1-bin-hadoop2.7.tgz # unzip it\n",
        "!pip install -q findspark # install findspark\n",
        "!pip install pyspark==3.1.1 # install pyspark\n",
        "!ls\n",
        "\n",
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = \"/content/spark-3.1.1-bin-hadoop2.7\"\n",
        "\n",
        "import findspark\n",
        "findspark.init()\n",
        "\n",
        "import seaborn as sns\n",
        "import pyspark.sql.functions as F\n",
        "from pyspark.ml.linalg import Vectors\n",
        "from pyspark.ml import feature\n",
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.builder.master(\"local[*]\").getOrCreate()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z4fcOn4AgGKt",
        "outputId": "3139ead2-9952-4d78-b7af-4474117c3194"
      },
      "id": "Z4fcOn4AgGKt",
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: pyspark==3.1.1 in /usr/local/lib/python3.10/dist-packages (3.1.1)\n",
            "Requirement already satisfied: py4j==0.10.9 in /usr/local/lib/python3.10/dist-packages (from pyspark==3.1.1) (0.10.9)\n",
            "gas_emissions.csv  spark-3.1.1-bin-hadoop2.7\t  spark-3.1.1-bin-hadoop2.7.tgz.1\n",
            "sample_data\t   spark-3.1.1-bin-hadoop2.7.tgz\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "forced-occasions"
      },
      "source": [
        "df = spark.read.csv(\"/content/gas_emissions.csv\", header = True, inferSchema = True)"
      ],
      "id": "forced-occasions",
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "regulation-bahrain",
        "outputId": "d87fb089-7a08-48a0-d1ce-3be97cb2877f"
      },
      "source": [
        "df.printSchema()"
      ],
      "id": "regulation-bahrain",
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- AT: double (nullable = true)\n",
            " |-- AP: double (nullable = true)\n",
            " |-- AH: double (nullable = true)\n",
            " |-- AFDP: double (nullable = true)\n",
            " |-- GTEP: double (nullable = true)\n",
            " |-- TIT: double (nullable = true)\n",
            " |-- TAT: double (nullable = true)\n",
            " |-- TEY: double (nullable = true)\n",
            " |-- CDP: double (nullable = true)\n",
            " |-- CO: double (nullable = true)\n",
            " |-- NOX: double (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k_IyKbwGVBMP",
        "outputId": "d8190264-b061-4f18-dae6-1c2f8aecb3d7"
      },
      "source": [
        "df.show()"
      ],
      "id": "k_IyKbwGVBMP",
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+------+------+------+------+------+------+------+------+------+------+\n",
            "|      AT|    AP|    AH|  AFDP|  GTEP|   TIT|   TAT|   TEY|   CDP|    CO|   NOX|\n",
            "+--------+------+------+------+------+------+------+------+------+------+------+\n",
            "|  1.9532|1020.1|84.985|2.5304|20.116|1048.7|544.92|116.27|10.799|7.4491|113.25|\n",
            "|  1.2191|1020.1|87.523|2.3937|18.584|1045.5| 548.5|109.18|10.347|6.4684|112.02|\n",
            "| 0.94915|1022.2|78.335|2.7789|22.264|1068.8|549.95|125.88|11.256|3.6335|88.147|\n",
            "|  1.0075|1021.7|76.942| 2.817|23.358|1075.2|549.63|132.21|11.702|3.1972|87.078|\n",
            "|  1.2858|1021.6|76.732|2.8377|23.483|1076.2|549.68|133.58|11.737|2.3833|82.515|\n",
            "|  1.8319|1021.7|76.411| 2.841|23.495|1076.4|549.92|133.58|11.829|2.0812|81.193|\n",
            "|   2.074|1022.0|75.974|2.7981|22.945|1073.7|549.98|131.53|11.687|2.2529|83.171|\n",
            "|  1.7824|1022.6|73.535|2.8327|23.337|1075.7|550.01|133.18|11.745| 3.735|85.749|\n",
            "|   1.593|1023.2|72.873|2.8729|23.654|1078.5|550.06|135.38|11.772|3.6398|86.491|\n",
            "|  1.6819|1023.8|72.441|2.9058|23.463|1077.9|550.12|134.86|11.742|3.5866|86.328|\n",
            "|  1.9002|1024.5|71.376|2.9126|23.562|1078.2|550.12|134.98| 11.77|3.5605|84.117|\n",
            "|  1.7797|1025.1|68.528|2.8725|23.276|1077.0|550.03|134.21|11.782|3.6902|85.317|\n",
            "|  1.1722|1025.3|65.626|  2.85|23.215|1077.3|550.09|134.44|11.873|3.1766|86.431|\n",
            "| 0.72327|1025.4|64.393|2.8395|23.101|1076.2|550.15|133.82|11.797| 2.562|84.708|\n",
            "| 0.48348|1025.8|64.144|2.8746|23.299|1078.2|549.96|135.36|11.897|2.1854|84.104|\n",
            "| 0.42953|1025.9|64.504| 2.866|23.069|1077.0|549.73|134.47|11.805|2.4286|83.869|\n",
            "| 0.48238|1026.1|63.519|2.5481|20.185|1037.1| 535.4|115.92| 10.86|12.659|118.27|\n",
            "|0.059447|1026.0|52.124|2.4651|18.478|1048.4|548.78| 111.2|10.384|4.6896|104.56|\n",
            "|-0.21598|1026.7|51.978|2.5301|19.438|1050.4|546.71|115.46|10.767|7.0983|116.96|\n",
            "|  0.0167|1026.8|54.805|3.3399|27.408|1085.6|540.72|148.05| 12.79|3.8632| 75.64|\n",
            "+--------+------+------+------+------+------+------+------+------+------+------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bearing-medicaid",
        "outputId": "ae19c89c-8640-4d91-8ad6-6f40c7e55f67"
      },
      "source": [
        "df.corr('CO', 'NOX')"
      ],
      "id": "bearing-medicaid",
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.6783940167718986"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fundamental-childhood",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 443
        },
        "outputId": "37c62f67-e560-4005-ec6b-ebd0eb1dcd57"
      },
      "source": [
        "dfp = df.toPandas()\n",
        "dfp"
      ],
      "id": "fundamental-childhood",
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "           AT      AP      AH    AFDP    GTEP     TIT     TAT     TEY     CDP  \\\n",
              "0     1.95320  1020.1  84.985  2.5304  20.116  1048.7  544.92  116.27  10.799   \n",
              "1     1.21910  1020.1  87.523  2.3937  18.584  1045.5  548.50  109.18  10.347   \n",
              "2     0.94915  1022.2  78.335  2.7789  22.264  1068.8  549.95  125.88  11.256   \n",
              "3     1.00750  1021.7  76.942  2.8170  23.358  1075.2  549.63  132.21  11.702   \n",
              "4     1.28580  1021.6  76.732  2.8377  23.483  1076.2  549.68  133.58  11.737   \n",
              "...       ...     ...     ...     ...     ...     ...     ...     ...     ...   \n",
              "7379  3.62680  1028.5  93.200  3.1661  19.087  1037.0  541.59  109.08  10.411   \n",
              "7380  4.16740  1028.6  94.036  3.1923  19.016  1037.6  542.28  108.79  10.344   \n",
              "7381  5.48200  1028.5  95.219  3.3128  18.857  1038.0  543.48  107.81  10.462   \n",
              "7382  5.88370  1028.7  94.200  3.9831  23.563  1076.9  550.11  131.41  11.771   \n",
              "7383  6.03920  1028.8  94.547  3.8752  22.524  1067.9  548.23  125.41  11.462   \n",
              "\n",
              "           CO      NOX  \n",
              "0      7.4491  113.250  \n",
              "1      6.4684  112.020  \n",
              "2      3.6335   88.147  \n",
              "3      3.1972   87.078  \n",
              "4      2.3833   82.515  \n",
              "...       ...      ...  \n",
              "7379  10.9930   89.172  \n",
              "7380  11.1440   88.849  \n",
              "7381  11.4140   96.147  \n",
              "7382   3.3134   64.738  \n",
              "7383  11.9810  109.240  \n",
              "\n",
              "[7384 rows x 11 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-1e7f27ab-8b58-4ab3-b353-bad1e4c4102f\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>AT</th>\n",
              "      <th>AP</th>\n",
              "      <th>AH</th>\n",
              "      <th>AFDP</th>\n",
              "      <th>GTEP</th>\n",
              "      <th>TIT</th>\n",
              "      <th>TAT</th>\n",
              "      <th>TEY</th>\n",
              "      <th>CDP</th>\n",
              "      <th>CO</th>\n",
              "      <th>NOX</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1.95320</td>\n",
              "      <td>1020.1</td>\n",
              "      <td>84.985</td>\n",
              "      <td>2.5304</td>\n",
              "      <td>20.116</td>\n",
              "      <td>1048.7</td>\n",
              "      <td>544.92</td>\n",
              "      <td>116.27</td>\n",
              "      <td>10.799</td>\n",
              "      <td>7.4491</td>\n",
              "      <td>113.250</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.21910</td>\n",
              "      <td>1020.1</td>\n",
              "      <td>87.523</td>\n",
              "      <td>2.3937</td>\n",
              "      <td>18.584</td>\n",
              "      <td>1045.5</td>\n",
              "      <td>548.50</td>\n",
              "      <td>109.18</td>\n",
              "      <td>10.347</td>\n",
              "      <td>6.4684</td>\n",
              "      <td>112.020</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.94915</td>\n",
              "      <td>1022.2</td>\n",
              "      <td>78.335</td>\n",
              "      <td>2.7789</td>\n",
              "      <td>22.264</td>\n",
              "      <td>1068.8</td>\n",
              "      <td>549.95</td>\n",
              "      <td>125.88</td>\n",
              "      <td>11.256</td>\n",
              "      <td>3.6335</td>\n",
              "      <td>88.147</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.00750</td>\n",
              "      <td>1021.7</td>\n",
              "      <td>76.942</td>\n",
              "      <td>2.8170</td>\n",
              "      <td>23.358</td>\n",
              "      <td>1075.2</td>\n",
              "      <td>549.63</td>\n",
              "      <td>132.21</td>\n",
              "      <td>11.702</td>\n",
              "      <td>3.1972</td>\n",
              "      <td>87.078</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1.28580</td>\n",
              "      <td>1021.6</td>\n",
              "      <td>76.732</td>\n",
              "      <td>2.8377</td>\n",
              "      <td>23.483</td>\n",
              "      <td>1076.2</td>\n",
              "      <td>549.68</td>\n",
              "      <td>133.58</td>\n",
              "      <td>11.737</td>\n",
              "      <td>2.3833</td>\n",
              "      <td>82.515</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7379</th>\n",
              "      <td>3.62680</td>\n",
              "      <td>1028.5</td>\n",
              "      <td>93.200</td>\n",
              "      <td>3.1661</td>\n",
              "      <td>19.087</td>\n",
              "      <td>1037.0</td>\n",
              "      <td>541.59</td>\n",
              "      <td>109.08</td>\n",
              "      <td>10.411</td>\n",
              "      <td>10.9930</td>\n",
              "      <td>89.172</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7380</th>\n",
              "      <td>4.16740</td>\n",
              "      <td>1028.6</td>\n",
              "      <td>94.036</td>\n",
              "      <td>3.1923</td>\n",
              "      <td>19.016</td>\n",
              "      <td>1037.6</td>\n",
              "      <td>542.28</td>\n",
              "      <td>108.79</td>\n",
              "      <td>10.344</td>\n",
              "      <td>11.1440</td>\n",
              "      <td>88.849</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7381</th>\n",
              "      <td>5.48200</td>\n",
              "      <td>1028.5</td>\n",
              "      <td>95.219</td>\n",
              "      <td>3.3128</td>\n",
              "      <td>18.857</td>\n",
              "      <td>1038.0</td>\n",
              "      <td>543.48</td>\n",
              "      <td>107.81</td>\n",
              "      <td>10.462</td>\n",
              "      <td>11.4140</td>\n",
              "      <td>96.147</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7382</th>\n",
              "      <td>5.88370</td>\n",
              "      <td>1028.7</td>\n",
              "      <td>94.200</td>\n",
              "      <td>3.9831</td>\n",
              "      <td>23.563</td>\n",
              "      <td>1076.9</td>\n",
              "      <td>550.11</td>\n",
              "      <td>131.41</td>\n",
              "      <td>11.771</td>\n",
              "      <td>3.3134</td>\n",
              "      <td>64.738</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7383</th>\n",
              "      <td>6.03920</td>\n",
              "      <td>1028.8</td>\n",
              "      <td>94.547</td>\n",
              "      <td>3.8752</td>\n",
              "      <td>22.524</td>\n",
              "      <td>1067.9</td>\n",
              "      <td>548.23</td>\n",
              "      <td>125.41</td>\n",
              "      <td>11.462</td>\n",
              "      <td>11.9810</td>\n",
              "      <td>109.240</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>7384 rows × 11 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1e7f27ab-8b58-4ab3-b353-bad1e4c4102f')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-1e7f27ab-8b58-4ab3-b353-bad1e4c4102f button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-1e7f27ab-8b58-4ab3-b353-bad1e4c4102f');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-610588e1-f3fa-4074-85f0-1c48c9ba8d6c\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-610588e1-f3fa-4074-85f0-1c48c9ba8d6c')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-610588e1-f3fa-4074-85f0-1c48c9ba8d6c button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_20feb18d-1ce1-485d-bf8f-d6c372a91370\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('dfp')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_20feb18d-1ce1-485d-bf8f-d6c372a91370 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('dfp');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "dfp",
              "summary": "{\n  \"name\": \"dfp\",\n  \"rows\": 7384,\n  \"fields\": [\n    {\n      \"column\": \"AT\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 8.095783332850191,\n        \"min\": -6.2348,\n        \"max\": 37.103,\n        \"num_unique_values\": 6662,\n        \"samples\": [\n          4.7595,\n          21.522,\n          9.5721\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"AP\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 6.8954301329731456,\n        \"min\": 989.4,\n        \"max\": 1036.6,\n        \"num_unique_values\": 440,\n        \"samples\": [\n          993.42,\n          1010.2,\n          1005.8\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"AH\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 13.541116395037708,\n        \"min\": 24.085,\n        \"max\": 96.666,\n        \"num_unique_values\": 6819,\n        \"samples\": [\n          77.365,\n          78.933,\n          71.862\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"AFDP\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.6102257770413468,\n        \"min\": 2.3688,\n        \"max\": 5.2395,\n        \"num_unique_values\": 6194,\n        \"samples\": [\n          3.7928,\n          4.3224,\n          3.5804\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"GTEP\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 4.47373665317691,\n        \"min\": 17.698,\n        \"max\": 40.716,\n        \"num_unique_values\": 5726,\n        \"samples\": [\n          29.861,\n          26.987,\n          25.563\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"TIT\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 19.762448980766607,\n        \"min\": 1016.0,\n        \"max\": 1100.4,\n        \"num_unique_values\": 730,\n        \"samples\": [\n          1028.0,\n          1044.3,\n          1042.8\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"TAT\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 5.489066228681691,\n        \"min\": 516.04,\n        \"max\": 550.59,\n        \"num_unique_values\": 1583,\n        \"samples\": [\n          542.18,\n          529.6,\n          547.65\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"TEY\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 16.17920839342477,\n        \"min\": 100.02,\n        \"max\": 179.5,\n        \"num_unique_values\": 3640,\n        \"samples\": [\n          115.88,\n          117.3,\n          145.32\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"CDP\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1.1366005366935754,\n        \"min\": 9.8708,\n        \"max\": 15.159,\n        \"num_unique_values\": 3129,\n        \"samples\": [\n          10.754,\n          10.905,\n          14.069\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"CO\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 2.2349624266177437,\n        \"min\": 0.2128,\n        \"max\": 41.097,\n        \"num_unique_values\": 6796,\n        \"samples\": [\n          4.5362,\n          4.5129,\n          5.7789\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"NOX\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 11.132463621891832,\n        \"min\": 25.905,\n        \"max\": 119.68,\n        \"num_unique_values\": 6540,\n        \"samples\": [\n          54.64,\n          57.799,\n          58.136\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 414
        },
        "id": "democratic-cheat",
        "outputId": "5b4a63ae-473e-425e-fde3-b0b468d7cca1"
      },
      "source": [
        "dfp.corr()"
      ],
      "id": "democratic-cheat",
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            AT        AP        AH      AFDP      GTEP       TIT       TAT  \\\n",
              "AT    1.000000 -0.493098 -0.466288  0.468976  0.193578  0.330112  0.208277   \n",
              "AP   -0.493098  1.000000  0.084381 -0.094144 -0.043730 -0.081605 -0.290147   \n",
              "AH   -0.466288  0.084381  1.000000 -0.245456 -0.297708 -0.260683  0.026251   \n",
              "AFDP  0.468976 -0.094144 -0.245456  1.000000  0.843958  0.915128 -0.519807   \n",
              "GTEP  0.193578 -0.043730 -0.297708  0.843958  1.000000  0.892851 -0.620652   \n",
              "TIT   0.330112 -0.081605 -0.260683  0.915128  0.892851  1.000000 -0.396161   \n",
              "TAT   0.208277 -0.290147  0.026251 -0.519807 -0.620652 -0.396161  1.000000   \n",
              "TEY   0.109437  0.050326 -0.182732  0.884954  0.932337  0.951813 -0.633933   \n",
              "CDP   0.200909  0.029420 -0.221706  0.922991  0.938142  0.951590 -0.656613   \n",
              "CO   -0.390647  0.200945  0.158999 -0.640789 -0.557177 -0.738092  0.025768   \n",
              "NOX  -0.593580  0.214236  0.065351 -0.584452 -0.366655 -0.520081  0.054455   \n",
              "\n",
              "           TEY       CDP        CO       NOX  \n",
              "AT    0.109437  0.200909 -0.390647 -0.593580  \n",
              "AP    0.050326  0.029420  0.200945  0.214236  \n",
              "AH   -0.182732 -0.221706  0.158999  0.065351  \n",
              "AFDP  0.884954  0.922991 -0.640789 -0.584452  \n",
              "GTEP  0.932337  0.938142 -0.557177 -0.366655  \n",
              "TIT   0.951813  0.951590 -0.738092 -0.520081  \n",
              "TAT  -0.633933 -0.656613  0.025768  0.054455  \n",
              "TEY   1.000000  0.991207 -0.616791 -0.403278  \n",
              "CDP   0.991207  1.000000 -0.612653 -0.443093  \n",
              "CO   -0.616791 -0.612653  1.000000  0.678394  \n",
              "NOX  -0.403278 -0.443093  0.678394  1.000000  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-c11ca1e3-9c9d-4785-9aab-3a4762f6bc54\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>AT</th>\n",
              "      <th>AP</th>\n",
              "      <th>AH</th>\n",
              "      <th>AFDP</th>\n",
              "      <th>GTEP</th>\n",
              "      <th>TIT</th>\n",
              "      <th>TAT</th>\n",
              "      <th>TEY</th>\n",
              "      <th>CDP</th>\n",
              "      <th>CO</th>\n",
              "      <th>NOX</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>AT</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>-0.493098</td>\n",
              "      <td>-0.466288</td>\n",
              "      <td>0.468976</td>\n",
              "      <td>0.193578</td>\n",
              "      <td>0.330112</td>\n",
              "      <td>0.208277</td>\n",
              "      <td>0.109437</td>\n",
              "      <td>0.200909</td>\n",
              "      <td>-0.390647</td>\n",
              "      <td>-0.593580</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>AP</th>\n",
              "      <td>-0.493098</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.084381</td>\n",
              "      <td>-0.094144</td>\n",
              "      <td>-0.043730</td>\n",
              "      <td>-0.081605</td>\n",
              "      <td>-0.290147</td>\n",
              "      <td>0.050326</td>\n",
              "      <td>0.029420</td>\n",
              "      <td>0.200945</td>\n",
              "      <td>0.214236</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>AH</th>\n",
              "      <td>-0.466288</td>\n",
              "      <td>0.084381</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-0.245456</td>\n",
              "      <td>-0.297708</td>\n",
              "      <td>-0.260683</td>\n",
              "      <td>0.026251</td>\n",
              "      <td>-0.182732</td>\n",
              "      <td>-0.221706</td>\n",
              "      <td>0.158999</td>\n",
              "      <td>0.065351</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>AFDP</th>\n",
              "      <td>0.468976</td>\n",
              "      <td>-0.094144</td>\n",
              "      <td>-0.245456</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.843958</td>\n",
              "      <td>0.915128</td>\n",
              "      <td>-0.519807</td>\n",
              "      <td>0.884954</td>\n",
              "      <td>0.922991</td>\n",
              "      <td>-0.640789</td>\n",
              "      <td>-0.584452</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>GTEP</th>\n",
              "      <td>0.193578</td>\n",
              "      <td>-0.043730</td>\n",
              "      <td>-0.297708</td>\n",
              "      <td>0.843958</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.892851</td>\n",
              "      <td>-0.620652</td>\n",
              "      <td>0.932337</td>\n",
              "      <td>0.938142</td>\n",
              "      <td>-0.557177</td>\n",
              "      <td>-0.366655</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>TIT</th>\n",
              "      <td>0.330112</td>\n",
              "      <td>-0.081605</td>\n",
              "      <td>-0.260683</td>\n",
              "      <td>0.915128</td>\n",
              "      <td>0.892851</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-0.396161</td>\n",
              "      <td>0.951813</td>\n",
              "      <td>0.951590</td>\n",
              "      <td>-0.738092</td>\n",
              "      <td>-0.520081</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>TAT</th>\n",
              "      <td>0.208277</td>\n",
              "      <td>-0.290147</td>\n",
              "      <td>0.026251</td>\n",
              "      <td>-0.519807</td>\n",
              "      <td>-0.620652</td>\n",
              "      <td>-0.396161</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-0.633933</td>\n",
              "      <td>-0.656613</td>\n",
              "      <td>0.025768</td>\n",
              "      <td>0.054455</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>TEY</th>\n",
              "      <td>0.109437</td>\n",
              "      <td>0.050326</td>\n",
              "      <td>-0.182732</td>\n",
              "      <td>0.884954</td>\n",
              "      <td>0.932337</td>\n",
              "      <td>0.951813</td>\n",
              "      <td>-0.633933</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.991207</td>\n",
              "      <td>-0.616791</td>\n",
              "      <td>-0.403278</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>CDP</th>\n",
              "      <td>0.200909</td>\n",
              "      <td>0.029420</td>\n",
              "      <td>-0.221706</td>\n",
              "      <td>0.922991</td>\n",
              "      <td>0.938142</td>\n",
              "      <td>0.951590</td>\n",
              "      <td>-0.656613</td>\n",
              "      <td>0.991207</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-0.612653</td>\n",
              "      <td>-0.443093</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>CO</th>\n",
              "      <td>-0.390647</td>\n",
              "      <td>0.200945</td>\n",
              "      <td>0.158999</td>\n",
              "      <td>-0.640789</td>\n",
              "      <td>-0.557177</td>\n",
              "      <td>-0.738092</td>\n",
              "      <td>0.025768</td>\n",
              "      <td>-0.616791</td>\n",
              "      <td>-0.612653</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.678394</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>NOX</th>\n",
              "      <td>-0.593580</td>\n",
              "      <td>0.214236</td>\n",
              "      <td>0.065351</td>\n",
              "      <td>-0.584452</td>\n",
              "      <td>-0.366655</td>\n",
              "      <td>-0.520081</td>\n",
              "      <td>0.054455</td>\n",
              "      <td>-0.403278</td>\n",
              "      <td>-0.443093</td>\n",
              "      <td>0.678394</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c11ca1e3-9c9d-4785-9aab-3a4762f6bc54')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-c11ca1e3-9c9d-4785-9aab-3a4762f6bc54 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-c11ca1e3-9c9d-4785-9aab-3a4762f6bc54');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-da74f524-05a1-4e94-b624-a7b99549d769\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-da74f524-05a1-4e94-b624-a7b99549d769')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-da74f524-05a1-4e94-b624-a7b99549d769 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"dfp\",\n  \"rows\": 11,\n  \"fields\": [\n    {\n      \"column\": \"AT\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.48972657236743233,\n        \"min\": -0.5935801816289322,\n        \"max\": 1.0,\n        \"num_unique_values\": 11,\n        \"samples\": [\n          0.3301116181539493,\n          1.0,\n          -0.39064671428013364\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"AP\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.37567766145431736,\n        \"min\": -0.49309788447741326,\n        \"max\": 1.0,\n        \"num_unique_values\": 11,\n        \"samples\": [\n          -0.08160450566034161,\n          -0.49309788447741326,\n          0.20094462495211735\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"AH\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.3928970132146455,\n        \"min\": -0.46628847151646263,\n        \"max\": 1.0,\n        \"num_unique_values\": 11,\n        \"samples\": [\n          -0.2606826886249583,\n          -0.46628847151646263,\n          0.15899855279947497\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"AFDP\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.6858831155161333,\n        \"min\": -0.640788643467625,\n        \"max\": 1.0,\n        \"num_unique_values\": 11,\n        \"samples\": [\n          0.9151277689704831,\n          0.46897582197822535,\n          -0.640788643467625\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"GTEP\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.6666687061950922,\n        \"min\": -0.6206520074545581,\n        \"max\": 1.0,\n        \"num_unique_values\": 11,\n        \"samples\": [\n          0.8928513142694373,\n          0.19357757737905953,\n          -0.5571772944087198\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"TIT\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.6898600605013095,\n        \"min\": -0.7380922679787333,\n        \"max\": 1.0,\n        \"num_unique_values\": 11,\n        \"samples\": [\n          1.0,\n          0.3301116181539493,\n          -0.7380922679787333\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"TAT\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.4981829079817136,\n        \"min\": -0.6566129848376039,\n        \"max\": 1.0,\n        \"num_unique_values\": 11,\n        \"samples\": [\n          -0.3961611861299234,\n          0.20827659820555447,\n          0.025768196838346007\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"TEY\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.6833300352101879,\n        \"min\": -0.6339334874062894,\n        \"max\": 1.0,\n        \"num_unique_values\": 11,\n        \"samples\": [\n          0.9518125918356007,\n          0.10943665575375053,\n          -0.6167909745570379\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"CDP\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.6956644077871932,\n        \"min\": -0.6566129848376039,\n        \"max\": 1.0,\n        \"num_unique_values\": 11,\n        \"samples\": [\n          0.9515900311506349,\n          0.20090892174021202,\n          -0.6126526246874934\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"CO\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.5914707591370133,\n        \"min\": -0.7380922679787333,\n        \"max\": 1.0,\n        \"num_unique_values\": 11,\n        \"samples\": [\n          -0.7380922679787333,\n          -0.39064671428013364,\n          1.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"NOX\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.5383059724011551,\n        \"min\": -0.5935801816289322,\n        \"max\": 1.0,\n        \"num_unique_values\": 11,\n        \"samples\": [\n          -0.5200808009795239,\n          -0.5935801816289322,\n          0.6783940167718986\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 475
        },
        "id": "lucky-mandate",
        "outputId": "1e851768-74e5-40b4-b5c9-42c67ba9d864"
      },
      "source": [
        "sns.heatmap(dfp.corr())"
      ],
      "id": "lucky-mandate",
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<Axes: >"
            ]
          },
          "metadata": {},
          "execution_count": 8
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 2 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiUAAAG5CAYAAABRIUROAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABLlUlEQVR4nO3deVxVdf4/8Ne9F72gbOLCYhjiBu4mA4OOpSOCy9hYNi5JCm6TiZpkKZVgWpLlruSSl9QmDcvc+2KKmi0k7mUBakrOqBcXZBH1Ktzz+8OfN4/cg4D3nnPA13Me5zHxOdsbRHj7eX8WjSAIAoiIiIgUplU6ACIiIiKASQkRERGpBJMSIiIiUgUmJURERKQKTEqIiIhIFZiUEBERkSowKSEiIiJVYFJCREREqsCkhIiIiFSBSQkRERGpApMSIiKix8D+/fvRv39/+Pj4QKPRYPPmzQ+9Z9++fXjqqaeg1+vRvHlzrF692q4xMikhIiJ6DBQXF6NDhw5ISkqq0PVnz55Fv3790KNHDxw7dgyvvvoqRo8ejZ07d9otRg035CMiInq8aDQabNq0CQMGDJC8ZurUqdixYwdOnDhhaRsyZAjy8/ORmppql7jYU0JERFQNmUwmFBYWig6TyWSz56enpyMsLEzUFhERgfT0dJu940EOdnuySty5ckbpEES6to9WOgQRvUZ93wJfD3RSOgSR86lmpUMQadC0WOkQRPLOqevP60p+XaVDEKnvdkPpEESm3NApHYLIieL/Kh2CyNmrx+3+Dlv9XkpcuhbvvPOOqC0hIQEzZsywyfONRiM8PT1FbZ6enigsLMTNmzfh5GT7v/vq+41ERERUk5lLbfKYuLg4xMbGitr0er1Nnq0UJiVERETVkF6vt2sS4uXlhdzcXFFbbm4uXF1d7dJLAjApISIikpegrpKwlNDQUHz99deitl27diE0NNRu7+RAVyIiIjmZzbY5Kun69es4duwYjh07BuDulN9jx47h3LlzAO6Wg4YPH265/uWXX8aZM2fwxhtvICsrCx999BE2bNiAyZMn2+TLYA2TEiIiosfAoUOH0KlTJ3Tq1AkAEBsbi06dOiE+Ph4AcPHiRUuCAgBNmzbFjh07sGvXLnTo0AHz5s3DqlWrEBERYbcYWb4hIiKSkaBQ+aZ79+4ob2kya6u1du/eHUePHrVjVGJMSoiIiORUhdLL44LlGyIiIlIF9pQQERHJqZrMvlECkxIiIiI52WjxtJpIFeWb9PR06HQ69OvXDwAQFRUFjUYjefj5+SkbMBEREdmcKpISg8GACRMmYP/+/bhw4QIWLVqEixcvWg4A+OSTTywfHzx4UOGIiYiIqkgw2+aogRQv31y/fh0pKSk4dOgQjEYjVq9ejTfffBNubm6i69zd3eHl5aVQlERERDbC2TeSFO8p2bBhAwICAtCqVStERkYiOTm53HnURERE1ZkgmG1y1ESKJyUGgwGRkZEAgN69e6OgoADffvutwlERERGR3BQt32RnZyMjIwObNm26G4yDAwYPHgyDwYDu3btX+nkmkwkmk0nUpjWZqv1WzkREVIOwfCNJ0Z4Sg8GAkpIS+Pj4wMHBAQ4ODli2bBk2btyIgoKCSj8vMTERbm5uomPOouV2iJyIiKiKONBVkmI9JSUlJVi7di3mzZuH8PBw0bkBAwZg/fr1ePnllyv1zLi4OMTGxoratEXnHzlWIiIisj/FkpLt27fj2rVrGDVqVJmZNgMHDoTBYKh0UqLX68uUau7cvvLIsRIREdkMF0+TpFj5xmAwICwsrExCAtxNSg4dOoSff/5ZgciIiIjsiOUbSYr1lGzbtk3yXHBwsGhaMKcIExER1XyKL55GRET0WOHsG0lMSoiIiORUQ0svtqD44mlEREREAHtKiIiI5MXyjSQmJURERDISBE4JlsKkhIiISE4cUyKJY0qIiIhIFdhTQkREJCeOKZHEpISIiEhOLN9IYvmGiIiIVIE9JURERHLihnySmJQQERHJieUbSTU+KenaPlrpEER++PkTpUMQSW87VekQylizSa90CCKuKvv5ceM3D6VDEDmku6V0CCKDzOr6sXbtmrvSIYjMrZevdAgiOmcvpUMgFVHX314iIqKajrNvJDEpISIikhPLN5I4+4aIiIhUgT0lREREcmL5RhJ7SoiIiORkNtvmqIKkpCT4+fnB0dERISEhyMjIKPf6hQsXolWrVnBycoKvry8mT56MW7fsN7idPSVEREQyUmqX4JSUFMTGxmL58uUICQnBwoULERERgezsbDRq1KjM9evWrcO0adOQnJyMLl264OTJk4iKioJGo8H8+fPtEiN7SoiIiB4D8+fPx5gxYxAdHY3WrVtj+fLlqFOnDpKTk61e/+OPP6Jr16548cUX4efnh/DwcAwdOvShvSuPgkkJERGRnGxUvjGZTCgsLBQdJpPJ6itv376Nw4cPIywszNKm1WoRFhaG9PR0q/d06dIFhw8ftiQhZ86cwddff42+ffva/mtyLya7PZmIiIjKEsw2ORITE+Hm5iY6EhMTrb7yypUrKC0thaenp6jd09MTRqPR6j0vvvgiZs6cib/97W+oVasWmjVrhu7du+PNN9+0+ZfkHiYlRERE1VBcXBwKCgpER1xcnM2ev2/fPsyePRsfffQRjhw5gq+++go7duzArFmzbPaOBymelKSnp0On06Ffv36i9pycHGg0GstRv359hIeH4+jRowpFSkREZAM2Kt/o9Xq4urqKDr3e+jYdDRo0gE6nQ25urqg9NzcXXl7Wl/qfPn06XnrpJYwePRrt2rXDc889h9mzZyMxMRFmO01rVjwpMRgMmDBhAvbv348LFy6UOb97925cvHgRO3fuxPXr19GnTx/k5+fLHygREZEt2Kh8Uxm1a9dG586dkZaWZmkzm81IS0tDaGio1Xtu3LgBrVacJuh0urufgiBU8pOuGEWTkuvXryMlJQXjxo1Dv379sHr16jLX1K9fH15eXggKCsLcuXORm5uLAwcOyB8sERFRNRYbG4uPP/4Ya9asQWZmJsaNG4fi4mJER9/duHb48OGi8k///v2xbNkyfP755zh79ix27dqF6dOno3///pbkxNYUXadkw4YNCAgIQKtWrRAZGYlXX30VcXFx0Gg0Vq93cnICcHcUMRERUbWk0IqugwcPxuXLlxEfHw+j0YiOHTsiNTXVMvj13Llzop6Rt99+GxqNBm+//TbOnz+Phg0bon///njvvffsFqOiSYnBYEBkZCQAoHfv3igoKMC3336L7t27l7k2Pz8fs2bNgrOzM4KDg60+z2QylZkOZRbM0GoUr1IRERHdpeCGfDExMYiJibF6bt++faKPHRwckJCQgISEBBkiu0ux39bZ2dnIyMjA0KFDAdz95AcPHgyDwSC6rkuXLnB2dka9evVw/PhxpKSklJnSdI+16VEXr5+z++dCREREj06xnhKDwYCSkhL4+PhY2gRBgF6vx9KlSy1tKSkpaN26NerXrw93d/dynxkXF4fY2FhR299b9ZO4moiISAHckE+SIklJSUkJ1q5di3nz5iE8PFx0bsCAAVi/fj169+4NAPD19UWzZs0q9Fy9Xl9mOhRLN0REpCpMSiQpkpRs374d165dw6hRo+Dm5iY6N3DgQBgMBktSQkREVKMoOKZE7RTpRjAYDAgLCyuTkAB3k5JDhw6hsLBQgciIiIhIKYr0lGzbtk3yXHBwsGVRFnstzkJERKQYlm8kKTolmIiI6LHD8o0kjgIlIiIiVWBPCRERkZxYvpHEpISIiEhOLN9IYvmGiIiIVIE9JURERHJi+UYSkxIiIiI5MSmRxPINERERqQJ7SoiIiOTEhUElMSkhIiKSE8s3kpiUEBERyYlJiaQan5ToNer6FNPbTlU6BJHQE3OUDqGMjUFvKh2CSJGuROkQROpAp3QIIv+8VUvpEES+d1LXULlbUNcvIN9b6vqZePlmHaVDEGmudACPOXV9dxIREdV0XDxNEpMSIiIiObF8I0ld/ZxERET02GJPCRERkZw4JVgSkxIiIiI5sXwjieUbIiIiUgX2lBAREcmJPSWSmJQQERHJiVOCJbF8Q0RERKrAnhIiIiIZCWbOvpHCpISIiEhOHFMiSbHyTXp6OnQ6Hfr16ydqz8nJgUajwbFjx8rc0717d7z66qvyBEhERGQPgtk2Rw2kWFJiMBgwYcIE7N+/HxcuXFAqDCIiIlIJRZKS69evIyUlBePGjUO/fv2wevVqJcIgIiKSn1mwzVEFSUlJ8PPzg6OjI0JCQpCRkVHu9fn5+Rg/fjy8vb2h1+vRsmVLfP3111V6d0UokpRs2LABAQEBaNWqFSIjI5GcnAyBy+4SEdHjwGy2zVFJKSkpiI2NRUJCAo4cOYIOHTogIiICly5dsnr97du30atXL+Tk5ODLL79EdnY2Pv74YzRu3PhRvwKSFBnoajAYEBkZCQDo3bs3CgoK8O2336J79+6Wa7p06QKtVpwz3bx5Ex07dpR8rslkgslkErWZBTO0Gs58JiKix9v8+fMxZswYREdHAwCWL1+OHTt2IDk5GdOmTStzfXJyMvLy8vDjjz+iVq1aAAA/Pz+7xij7b+vs7GxkZGRg6NChAAAHBwcMHjwYBoNBdF1KSgqOHTsmOoKCgsp9dmJiItzc3ETHf4ty7PWpEBERVZ6NekpMJhMKCwtFx4P/ML/n9u3bOHz4MMLCwixtWq0WYWFhSE9Pt3rP1q1bERoaivHjx8PT0xNt27bF7NmzUVpaapcvC6BAUmIwGFBSUgIfHx84ODjAwcEBy5Ytw8aNG1FQUGC5ztfXF82bNxcdTk5O5T47Li4OBQUFosPXxc/OnxEREVElCIJNDmv/EE9MTLT6yitXrqC0tBSenp6idk9PTxiNRqv3nDlzBl9++SVKS0vx9ddfY/r06Zg3bx7effddm39J7pG1fFNSUoK1a9di3rx5CA8PF50bMGAA1q9fj969e1f5+Xq9Hnq9XtTG0g0REdVEcXFxiI2NFbU9+DvwUZjNZjRq1AgrV66ETqdD586dcf78eXz44YdISEiw2XvuJ2tSsn37dly7dg2jRo2Cm5ub6NzAgQNhMBgeKSkhIiJSPRstnmbtH+JSGjRoAJ1Oh9zcXFF7bm4uvLy8rN7j7e2NWrVqQafTWdoCAwNhNBpx+/Zt1K5du+rBS5C1G8FgMCAsLKxMQgLcTUoOHTqEwsJCOUMiIiKSlwJTgmvXro3OnTsjLS3tzzDMZqSlpSE0NNTqPV27dsXp06dhvi+JOnnyJLy9ve2SkAAy95Rs27ZN8lxwcLBlWrDU9OB9+/bZIywiIqIaLzY2FiNGjEBQUBCCg4OxcOFCFBcXW2bjDB8+HI0bN7aMSxk3bhyWLl2KSZMmYcKECTh16hRmz56NiRMn2i1G7n1DREQkJ4WWiB88eDAuX76M+Ph4GI1GdOzYEampqZbBr+fOnRMtxeHr64udO3di8uTJaN++PRo3boxJkyZh6tSpdouRSQkREZGcFNwlOCYmBjExMVbPWatGhIaG4qeffrJzVH9iUkJERCQjgbsES+J8WSIiIlIF9pQQERHJScHyjdoxKSEiIpKTQgNdqwOWb4iIiEgV2FNCREQkJ5ZvJDEpISIikhNn30hi+YaIiIhUgT0lREREcmL5RlKNT0q+HuikdAgiazbZbltpW9gY9KbSIZQx99BspUMQKT17VOkQRDRunkqHIPLXp+235HRV/LR9htIhiOnrKh2BSJOgkUqHIFK3lrp+Rp+R4yWcfSOJ5RsiIiJShRrfU0JERKQqLN9IYlJCREQkI+59I41JCRERkZzYUyKJY0qIiIhIFdhTQkREJCf2lEhiUkJERCQnTgmWxPINERERqQJ7SoiIiOTE8o0kJiVEREQyEpiUSKpS+SY9PR06nQ79+vUTtefk5ECj0ZQ5IiMjrZ53cXFBmzZtMH78eJw6dUr0rNWrV1uu02q1eOKJJxAdHY1Lly5V8VMlIiIiNatST4nBYMCECRNgMBhw4cIF+Pj4iM7v3r0bbdq0sXzs5ORk9fyNGzfwyy+/YNGiRejQoQO2bduGnj17Wq5zdXVFdnY2zGYzjh8/jujoaFy4cAE7d+6sSthERETKY0+JpEonJdevX0dKSgoOHToEo9GI1atX4803xZu61a9fH15eXpLPuP+8v78/+vfvj549e2LUqFH4/fffodPpAAAajcZynY+PDyZOnIjp06fj5s2bZRIdIiKiaoErukqqdPlmw4YNCAgIQKtWrRAZGYnk5GQIwqNlfVqtFpMmTcIff/yBw4cPS17n5OQEs9mMkpKSR3ofERERqU+lkxKDwWAZI9K7d28UFBTg22+/FV3TpUsXODs7W46jRx++9XtAQACAu+NOrDl16hSWL1+OoKAguLi4VDZsIiIidTALtjlqoEqVb7Kzs5GRkYFNmzbdvdnBAYMHD4bBYED37t0t16WkpCAwMNDysa+v70Offa+3RaPRWNoKCgrg7OwMs9mMW7du4W9/+xtWrVol+QyTyQSTySRqu11aCv3/LwcREREproYmFLZQqaTEYDCgpKRENLBVEATo9XosXbrU0ubr64vmzZtXKpDMzEwAQNOmTS1tLi4uOHLkCLRaLby9vR86jiQxMRHvvPOOqG1acAu8GdKyUrEQERHZy6MOeajJKly+KSkpwdq1azFv3jwcO3bMchw/fhw+Pj5Yv359lYMwm81YvHgxmjZtik6dOv0ZnFaL5s2bw9/fv0IDW+Pi4lBQUCA6XgtqVuW4iIiISD4V7inZvn07rl27hlGjRsHNzU10buDAgTAYDOjdu3eFnnX16lUYjUbcuHEDJ06cwMKFC5GRkYEdO3ZYZt5UhV6vh16vF7UVsXRDRERqwvKNpAonJQaDAWFhYWUSEuBuUvLBBx+gsLCwQs8KCwsDANSpUwdPPvkkevTogZUrV1a65ENERFTtMCmRVOGkZNu2bZLngoODLTWy8mplfn5+Fa6lRUVFISoqqqLhERERUTXHvW+IiIhkxL1vpFVp7xsiIiKqIgXXKUlKSoKfnx8cHR0REhKCjIyMCt33+eefQ6PRYMCAAVV6b0UxKSEiInoMpKSkIDY2FgkJCThy5Ag6dOiAiIiIh250m5OTgylTpqBbt252j5FJCRERkZzMNjoqaf78+RgzZgyio6PRunVrLF++HHXq1EFycrLkPaWlpRg2bBjeeecd+Pv7V/6llcSkhIiISEaCWbDJYTKZUFhYKDoeXNX8ntu3b+Pw4cOW2a/A3bXAwsLCkJ6eLhnrzJkz0ahRI4waNcrmXwdrmJQQERFVQ4mJiXBzcxMdiYmJVq+9cuUKSktL4enpKWr39PSE0Wi0es/3338Pg8GAjz/+2OaxS+HsGyIiIjnZaPZNXFwcYmNjRW0PLiBaVUVFRXjppZfw8ccfo0GDBjZ5ZkUwKSEiIpJTFcaDWGNtFXMpDRo0gE6nQ25urqg9NzcXXl5eZa7//fffkZOTg/79+1vazOa7gTs4OCA7OxvNmtl+GxeWb4iIiGRkqzEllVG7dm107twZaWlpljaz2Yy0tDSEhoaWuT4gIAC//PKLaK+7Z599Fj169MCxY8fg6+v7yF8Ha9hTQkRE9BiIjY3FiBEjEBQUhODgYCxcuBDFxcWIjo4GAAwfPhyNGzdGYmIiHB0d0bZtW9H97u7uAFCm3ZaYlBAREcnJRuWbyho8eDAuX76M+Ph4GI1GdOzYEampqZbBr+fOnYNWq2wBpcYnJedTFfrTl+CqrnBQpCtROoQySs8eVToEEV3TTkqHIGLOPat0CCKXTPlKhyCiqVN201D6Uz29i9IhiLR28lY6BNkpucx8TEwMYmJirJ7bt29fufeuXr3a9gE9gGNKiIiISBVqfE8JERGRqqisx1xNmJQQERHJSGBSIonlGyIiIlIF9pQQERHJiT0lkpiUEBERyYjlG2ks3xAREZEqsKeEiIhITuwpkcSkhIiISEYs30hjUkJERCQjJiXSOKaEiIiIVKHKSYnRaMSkSZPQvHlzODo6wtPTE127dsWyZcsQHBwMjUYjeXTv3h0A4OfnZ/X8+++/DwDIyckRtdevXx/h4eE4elRde6MQERFVlGC2zVETVal8c+bMGXTt2hXu7u6YPXs22rVrB71ej19++QUrV65ETEwMwsPDAQD//e9/ERwcjN27d6NNmzYAgNq1a1ueNXPmTIwZM0b0fBcX8YZR9+793//+h4kTJ6JPnz7IysqybKNMRERUbQgapSNQrSolJa+88gocHBxw6NAh1K1b19Lu7++Pf/7znxAEARrN3S/6rVu3AAD169eHl5dXmWe5uLhYbb/fvXu9vLwwd+5cdO3aFQcOHEBERERVwiciIiIVqnRScvXqVXzzzTeYPXu2KCG5372ExB6cnJwAALdv37bbO4iIiOylppZebKHSY0pOnz4NQRDQqlUrUXuDBg3g7OwMZ2dnTJ06tcLPmzp1quW+e8d3331n9dr8/HzMmjULzs7OCA4OLnPeZDKhsLBQdNw2l1buEyQiIrIjwayxyVET2WxKcEZGBsxmM4YNGwaTyVTh+15//XVERUWJ2ho3biz6uEuXLtBqtSguLoa/vz9SUlLg6elZ5lmJiYl45513RG3jPZojpn6Lin8iREREpIhKJyXNmzeHRqNBdna2qN3f3x/An+WVimrQoAGaN29e7jUpKSlo3bo16tevX+7g1ri4OMTGxoracp76V6XiISIisieWb6RVunxTv3599OrVC0uXLkVxcbE9YirD19cXzZo1e+hsG71eD1dXV9FRW6uTJUYiIqKKEASNTY6aqErrlHz00UcoKSlBUFAQUlJSkJmZiezsbPznP/9BVlYWdLqKJwJFRUUwGo2io7CwsCphERERUTVWpaSkWbNmOHr0KMLCwhAXF4cOHTogKCgIS5YswZQpUzBr1qwKPys+Ph7e3t6i44033qhKWERERKrHxdOkVXmgq7e3N5YsWYIlS5aUe52fnx8EQbB6Licnp8r3EhERVUc1deaMLXBDPiIiIhnx39rSuCEfERERqQJ7SoiIiGTE8o00JiVEREQyYlIijeUbIiIiUgX2lBAREcmIA12lMSkhIiKSEcs30li+ISIiIlVgTwkREZGMauq+NbbAnhIiIiIZKbnMfFJSEvz8/ODo6IiQkBBkZGRIXvvxxx+jW7duqFevHurVq4ewsLByr7cFJiVERESPgZSUFMTGxiIhIQFHjhxBhw4dEBERgUuXLlm9ft++fRg6dCj27t2L9PR0+Pr6Ijw8HOfPn7dbjDW+fNOgabHSIYjc+M1D6RBE6qDiOzrLRePmqXQIIubcs0qHIKL1bKp0CCKdXPyUDkFMV+N/rD2S0/kXlA5BxElbW+kQZGdWqHwzf/58jBkzBtHR0QCA5cuXY8eOHUhOTsa0adPKXP/ZZ5+JPl61ahU2btyItLQ0DB8+3C4x8m8vERGRjGw1psRkMsFkMona9Ho99Hp9mWtv376Nw4cPIy4uztKm1WoRFhaG9PT0Cr3vxo0buHPnDjw87PePa5ZviIiIZCSYNTY5EhMT4ebmJjoSExOtvvPKlSsoLS2Fp6e4J9rT0xNGo7FCcU+dOhU+Pj4ICwt75K+BFPaUEBERVUNxcXGIjY0VtVnrJbGF999/H59//jn27dsHR0dHu7wDYFJCREQkK1ut6CpVqrGmQYMG0Ol0yM3NFbXn5ubCy8ur3Hvnzp2L999/H7t370b79u2rHG9FsHxDREQkI1uVbyqjdu3a6Ny5M9LS0ixtZrMZaWlpCA0Nlbzvgw8+wKxZs5CamoqgoKAqf84VxZ4SIiKix0BsbCxGjBiBoKAgBAcHY+HChSguLrbMxhk+fDgaN25sGZcyZ84cxMfHY926dfDz87OMPXF2doazs7NdYmRSQkREJCOlpgQPHjwYly9fRnx8PIxGIzp27IjU1FTL4Ndz585Bq/2zgLJs2TLcvn0bL7zwgug5CQkJmDFjhl1iZFJCREQkIyWXmY+JiUFMTIzVc/v27RN9nJOTY/+AHsAxJURERKQK7CkhIiKSka1m39REsvWUaDSaco8ZM2YgJycHGo0Gx44dw4wZMx56DxERUXVjFjQ2OWoi2XpKLl68aPnvlJQUxMfHIzs729Lm7OyMK1euWD6eMmUKXn75ZcvHf/nLXzB27FiMGTNGnoCJiIhIVrIlJfcvzuLm5gaNRlNmwZb7k5IHpxzpdDq4uLg8dJEXIiIiNVNyoKvacUwJERGRjDimRBqTEiIiIhnV1PEgtlCjkhJr2zibzGbotZz5TEREpHY16re1tW2cF505p3RYREREFoKgsclRE9WopCQuLg4FBQWiY5J/E6XDIiIisuCUYGk1qnxjbRvn2yzdEBERVQs1KikhIiJSO06+kaZIUhIVFYWoqKgy7X5+fhAk5kopsTEQERGRrdXU0ostsLZBREREqsDyDRERkYxq6swZW2BSQkREJCOz0gGoGMs3REREpArsKSEiIpKRAJZvpDApISIikpGZc4IlMSkhIiKSkZk9JZI4poSIiIhUgT0lREREMuKYEmlMSoiIiGTEKcHSWL4hIiIiVWBPCRERkYxYvpFW45OSvHNOSocgckh3S+kQRP55q5bSIZTx16enKh2CyCVTvtIhiHRy8VM6BJFNR5YoHYKIb/N+SocgUmIuVToEkU0eTysdgoheYhPWmozlG2ks3xAREZEq1PieEiIiIjVhT4k0JiVEREQy4pgSaSzfEBERkSqwp4SIiEhGZnaUSGJSQkREJCPufSON5RsiIiIZCTY6qiIpKQl+fn5wdHRESEgIMjIyyr3+iy++QEBAABwdHdGuXTt8/fXXVXxzxTApISIiegykpKQgNjYWCQkJOHLkCDp06ICIiAhcunTJ6vU//vgjhg4dilGjRuHo0aMYMGAABgwYgBMnTtgtRiYlREREMjLb6Kis+fPnY8yYMYiOjkbr1q2xfPly1KlTB8nJyVavX7RoEXr37o3XX38dgYGBmDVrFp566iksXbq0Cm+vGCYlREREMjJrNDY5TCYTCgsLRYfJZLL6ztu3b+Pw4cMICwuztGm1WoSFhSE9Pd3qPenp6aLrASAiIkLyeluwW1Ki0WjKPWbMmGG5NiAgAHq9HkajEQCwb9++h96/b98+e4VORESkeomJiXBzcxMdiYmJVq+9cuUKSktL4enpKWr39PS0/O59kNForNT1tmC32TcXL160/HdKSgri4+ORnZ1taXN2dgYAfP/997h58yZeeOEFrFmzBlOnTkWXLl1E90+aNAmFhYX45JNPLG0eHh72Cp2IiMhubLXbT1xcHGJjY0Vter3eRk9Xht2SEi8vL8t/u7m5QaPRiNruMRgMePHFF/HMM89g0qRJmDp1KmrXri261snJCSaTyer9RERE1YmtlpnX6/UVTkIaNGgAnU6H3NxcUXtubq7k71YvL69KXW8Lio4pKSoqwhdffIHIyEj06tULBQUF+O6775QMiYiIqMapXbs2OnfujLS0NEub2WxGWloaQkNDrd4TGhoquh4Adu3aJXm9LSi6eNrnn3+OFi1aoE2bNgCAIUOGwGAwoFu3blV6nslkKjPI57bZjNpajuclIiJ1UGpF19jYWIwYMQJBQUEIDg7GwoULUVxcjOjoaADA8OHD0bhxY8u4lEmTJuGZZ57BvHnz0K9fP3z++ec4dOgQVq5cabcYFf1tnZycjMjISMvHkZGR+OKLL1BUVFSl51kb9LPi6hlbhUtERPTIzNDY5KiswYMHY+7cuYiPj0fHjh1x7NgxpKamWgaznjt3TjSes0uXLli3bh1WrlyJDh064Msvv8TmzZvRtm1bm30tHqRYT8lvv/2Gn376CRkZGZg6daqlvbS0FJ9//jnGjBlT6WdaG/Rz7i8vPHKsRERENUFMTAxiYmKsnrM2q/Vf//oX/vWvf9k5qj8plpQYDAY8/fTTSEpKErV/8sknMBgMVUpKrA36YemGiIjUxFazb2oiRZKSO3fu4NNPP8XMmTPLdAONHj0a8+fPx6+//moZa0JERFRTcJdgaYp0I2zduhVXr17Fc889V+ZcYGAgAgMDYTAYFIiMiIjIvpRaZr46kKWnJCoqClFRUZaPBw4ciNLSUsnrf/vtN9HHq1evtlNkREREpBaKTgkmIiJ63HBMiTQmJURERDLimBJpnJpCREREqsCeEiIiIhnV1EGqtsCkhIiISEZMSqSxfENERESqwJ4SIiIiGQkc6CqJSQkREZGMWL6RxqSEiIhIRkxKpHFMCREREakCe0qIiIhkxBVdpdX4pORKfl2lQxAZZFbXl/x7J/V1lv20fYbSIYho6rgpHYKYTl3fQ77N+ykdgsh/T+9QOgQxs/Q+X0rwazVA6RBEOrg8qXQIIuEyvIMrukpT328kIiIieiyp659cRERENRwHukpjUkJERCQjJiXSWL4hIiIiVWBPCRERkYw4+0YakxIiIiIZcfaNNJZviIiISBXYU0JERCQjDnSVxqSEiIhIRhxTIo1JCRERkYzMTEsk2WVMiUajKfeYMWMGcnJyJM//9NNPmDVrFry9vZGXlyd69vHjx6HX67F9+3Z7hE5EREQKsUtPycWLFy3/nZKSgvj4eGRnZ1vanJ2dceXKFQDA7t270aZNG9H99evXR1BQELZt24bx48dj/fr1AIA7d+5gxIgRiIyMxD/+8Q97hE5ERGRXHFMizS5JiZeXl+W/3dzcoNFoRG0ALElJ/fr1y5y7Z+3atejUqRO+/PJLvPDCC3jvvfeQn5+PBQsW2CNsIiIiu2PxRpqqx5QEBAQgMTER48aNg4uLCxITE5GamgpXV1elQyMiIiIbUzwp6dKlC7Ra8dCW69evW/570qRJ2LJlC/r27YsJEyagR48ecodIRERkMyzfSFN88bSUlBQcO3ZMdNxPo9Hgrbfegtlsxttvv13us0wmEwoLC0XHbaHUjtETERFVjlljm8Oe8vLyMGzYMLi6usLd3R2jRo0SdRhYu37ChAlo1aoVnJyc0KRJE0ycOBEFBQWVeq/iSYmvry+aN28uOh7k4OAg+n8piYmJcHNzEx2fXs8u9x4iIiISGzZsGH799Vfs2rUL27dvx/79+zF27FjJ6y9cuIALFy5g7ty5OHHiBFavXo3U1FSMGjWqUu9VvHxjS3FxcYiNjRW1HWk5XKFoiIiIylL7OiWZmZlITU3FwYMHERQUBABYsmQJ+vbti7lz58LHx6fMPW3btsXGjRstHzdr1gzvvfceIiMjUVJS8tBOhXsUT0quXr0Ko9EoanN3d4ejo2Oln6XX66HX60VttTW6R4qPiIjIlmyVkphMJphMJlGbtd+DlZWeng53d3dLQgIAYWFh0Gq1OHDgAJ577rkKPaegoACurq4VTkgAFZRvwsLC4O3tLTo2b96sdFhERESqZm3IQmJi4iM/12g0olGjRqI2BwcHeHh4lOlEkHLlyhXMmjWr3JKPNXZPSqKiopCfn1+m3c/PD4IgWD2GDBkiurZ79+4QBAHu7u72DpeIiMiuzDY64uLiUFBQIDri4uIk3ztt2rSHrrielZX1yJ9fYWEh+vXrh9atW2PGjBmVulfx8g0REdHjxFZjSipbqnnttdcQFRVV7jX+/v7w8vLCpUuXRO0lJSXIy8uTXOz0nqKiIvTu3RsuLi7YtGkTatWqVeH4ACYlREREslJqmGvDhg3RsGHDh14XGhqK/Px8HD58GJ07dwYA7NmzB2azGSEhIZL3FRYWIiIiAnq9Hlu3bq3S2FDFx5QQERGRegQGBqJ3794YM2YMMjIy8MMPPyAmJgZDhgyxzLw5f/48AgICkJGRAeBuQhIeHo7i4mIYDAYUFhbCaDTCaDSitLTi64Wxp4SIiEhG1WFF188++wwxMTHo2bMntFotBg4ciMWLF1vO37lzB9nZ2bhx4wYA4MiRIzhw4AAAlFlv7OzZs/Dz86vQe5mUEBERyUjt65QAgIeHB9atWyd5/t5klXvuTUh5VCzfEBERkSqwp4SIiEhG6u8nUQ6TEiIiIhlVhzElSmH5hoiIiFSBPSVEREQyEljAkcSkhIiISEYs30hj+YaIiIhUocb3lNR3u6F0CCLXrrkrHYLILTXm7Pq6SkdAlVBirvhqjbJQWzxandIRiGg0GqVDELlpvqN0CLKrDuuUKKXGJyVERERqwpREGpMSIiIiGbGnRBrHlBAREZEqsKeEiIhIRiocyacaTEqIiIhkxHVKpLF8Q0RERKrAnhIiIiIZsXwjjUkJERGRjFi+kcbyDREREakCe0qIiIhkxPKNNLv0lBiNRkyYMAH+/v7Q6/Xw9fVF//79kZaWBgDw8/ODRqOBRqOBk5MT/Pz8MGjQIOzZs0f0nJycHMt1Go0G9evXR3h4OI4ePWqPsImIiOzOLAg2OWoimyclOTk56Ny5M/bs2YMPP/wQv/zyC1JTU9GjRw+MHz/ect3MmTNx8eJFZGdnY+3atXB3d0dYWBjee++9Ms/cvXs3Ll68iJ07d+L69evo06cP8vPzbR06ERERKcjm5ZtXXnkFGo0GGRkZqFv3z43V2rRpg5EjR1o+dnFxgZeXFwCgSZMmePrpp+Ht7Y34+Hi88MILaNWqleXa+vXrw8vLC15eXpg7dy66du2KAwcOICIiwtbhExER2VXN7OOwDZv2lOTl5SE1NRXjx48XJST3uLu7l3v/pEmTIAgCtmzZInmNk5MTAOD27duPFCsREZESzBBsctRENu0pOX36NARBQEBAQJXu9/DwQKNGjZCTk2P1fH5+PmbNmgVnZ2cEBwc/QqRERETK4JRgaTZNSgQbDLwRBAEajUbU1qVLF2i1WhQXF8Pf3x8pKSnw9PQsc6/JZILJZBK13TabUVvLmc9ERERqZ9Pf1i1atIBGo0FWVlaV7r969SouX76Mpk2bitpTUlJw/PhxXLt2Db///jv69u1r9f7ExES4ubmJjhV5v1cpFiIiInsw2+ioiWyalHh4eCAiIgJJSUkoLi4uc/5hM2YWLVoErVaLAQMGiNp9fX3RrFmzh45JiYuLQ0FBgej4t0ezSn4WRERE9sMxJdJsPvsmKSkJXbt2RXBwMGbOnIn27dujpKQEu3btwrJly5CZmQkAKCoqgtFoxJ07d3D27Fn85z//wapVq5CYmIjmzZtX6d16vR56vV7UxtINERFR9WDzpMTf3x9HjhzBe++9h9deew0XL15Ew4YN0blzZyxbtsxyXXx8POLj41G7dm14eXnhr3/9K9LS0tCjRw9bh0RERKQaHOgqzS7LzHt7e2Pp0qVYunSp1fNSs2se5OfnZ5PBs0RERGpRU8eD2AJrG0RERKQK3JCPiIhIRqwASGNSQkREJKOaOnPGFli+ISIiIpG8vDwMGzYMrq6ucHd3x6hRo3D9+vUK3SsIAvr06QONRoPNmzdX6r1MSoiIiGRUHRZPGzZsGH799Vfs2rUL27dvx/79+zF27NgK3btw4cIyK7NXFMs3REREMlL7lODMzEykpqbi4MGDCAoKAgAsWbIEffv2xdy5c+Hj4yN577FjxzBv3jwcOnQI3t7elX43e0qIiIhkZKsVXU0mEwoLC0XHg/u/VUV6ejrc3d0tCQkAhIWFQavV4sCBA5L33bhxAy+++CKSkpLg5eVVpXczKSEiIqqGrO33lpiY+MjPNRqNaNSokajNwcEBHh4eMBqNkvdNnjwZXbp0wT//+c8qv5vlGyIiIhnZakpwXFwcYmNjRW0PbrVyv2nTpmHOnDnlPvPeVjCVtXXrVuzZswdHjx6t0v33MCkhIiKSka0GqVrb7608r732GqKiosq9xt/fH15eXrh06ZKovaSkBHl5eZJlmT179uD3338vs3HuwIED0a1bN+zbt69CMTIpISIiegw0bNgQDRs2fOh1oaGhyM/Px+HDh9G5c2cAd5MOs9mMkJAQq/dMmzYNo0ePFrW1a9cOCxYsQP/+/SscY41PSqbc0CkdgsjcevlKhyDie0t93wJNgkYqHYJIPb2L0iGInM6/oHQIIps8nlY6BBG/VgOUDkGkqlMj7eXsya1KhyBSkmpQOgTZqX32TWBgIHr37o0xY8Zg+fLluHPnDmJiYjBkyBDLzJvz58+jZ8+eWLt2LYKDg+Hl5WW1F6VJkyZo2rRphd/Nga5EREQystXsG3v67LPPEBAQgJ49e6Jv377429/+hpUrV1rO37lzB9nZ2bhx44ZN36u+fyYTERGRojw8PLBu3TrJ835+fg8dsFuVAb1MSoiIiGTEDfmkMSkhIiKSETfkk8YxJURERKQK7CkhIiKSkdpn3yiJSQkREZGMzBxTIolJCRERkYyYkkjjmBIiIiJSBfaUEBERyYizb6QxKSEiIpIRkxJpipRvjEYjJkyYAH9/f+j1evj6+qJ///5IS0uzXPPjjz+ib9++qFevHhwdHdGuXTvMnz8fpaWlSoRMREREdiZ7T0lOTg66du0Kd3d3fPjhh2jXrh3u3LmDnTt3Yvz48cjKysKmTZswaNAgREdHY+/evXB3d8fu3bvxxhtvID09HRs2bFDdJldEREQVwRVdpcmelLzyyivQaDTIyMhA3bp1Le1t2rTByJEjUVxcjDFjxuDZZ58Vbf4zevRoeHp64tlnn8WGDRswePBguUMnIiJ6ZCzfSJO1fJOXl4fU1FSMHz9elJDc4+7ujm+++QZXr17FlClTypzv378/WrZsifXr18sRLhEREclI1p6S06dPQxAEBAQESF5z8uRJAEBgYKDV8wEBAZZrHmQymWAymURtpUIpdBpdFSMmIiKyLa7oKk3WnpLK1NGqUnNLTEyEm5ub6Dhd+Huln0NERGQvgiDY5KiJZE1KWrRoAY1Gg6ysLMlrWrZsCQDIzMy0ej4zM9NyzYPi4uJQUFAgOpq7Nnv0wImIiMjuZE1KPDw8EBERgaSkJBQXF5c5n5+fj/DwcHh4eGDevHllzm/duhWnTp3C0KFDrT5fr9fD1dVVdLB0Q0REamKGYJOjJpJ9nZKkpCSUlpYiODgYGzduxKlTp5CZmYnFixcjNDQUdevWxYoVK7BlyxaMHTsWP//8M3JycmAwGBAVFYUXXngBgwYNkjtsIiIim2D5RprsU4L9/f1x5MgRvPfee3jttddw8eJFNGzYEJ07d8ayZcsAAC+88AL27t2L9957D926dcOtW7fQokULvPXWW3j11Ve5RgkREVVbNbWXwxYUWWbe29sbS5cuxdKlSyWv6datG1JTU2WMioiIiJTEvW+IiIhkxCnB0piUEBERychcQ8eD2IIiG/IRERERPYg9JURERDJi+UYakxIiIiIZsXwjjeUbIiIiUgX2lBAREcmI5RtpTEqIiIhkxPKNNJZviIiISBXYU0JERCQjlm+kMSkhIiKSEcs30mp8UnKi+L9KhyCic/ZSOgSRyzfrKB1CGXVrOSkdgkhrJ2+lQxBx0tZWOgQRvcp+wHZweVLpEERumu8oHYJISapB6RBEHHqPUjoE2VWHnpK8vDxMmDAB27Ztg1arxcCBA7Fo0SI4OzuXe196ejreeustHDhwADqdDh07dsTOnTvh5FSxn+scU0JEREQiw4YNw6+//opdu3Zh+/bt2L9/P8aOHVvuPenp6ejduzfCw8ORkZGBgwcPIiYmBlptxVONGt9TQkREpCaCYFY6hHJlZmYiNTUVBw8eRFBQEABgyZIl6Nu3L+bOnQsfHx+r902ePBkTJ07EtGnTLG2tWrWq1LvZU0JERCQjMwSbHCaTCYWFhaLDZDI9cnzp6elwd3e3JCQAEBYWBq1WiwMHDli959KlSzhw4AAaNWqELl26wNPTE8888wy+//77Sr2bSQkREVE1lJiYCDc3N9GRmJj4yM81Go1o1KiRqM3BwQEeHh4wGo1W7zlz5gwAYMaMGRgzZgxSU1Px1FNPoWfPnjh16lSF382khIiISEaCINjkiIuLQ0FBgeiIi4uTfO+0adOg0WjKPbKysqr0OZnNd0tS//73vxEdHY1OnTphwYIFaNWqFZKTkyv8HI4pISIikpHZRrNv9Ho99Hp9ha9/7bXXEBUVVe41/v7+8PLywqVLl0TtJSUlyMvLg5eX9Rmk3t53Zym2bt1a1B4YGIhz585VOEYmJURERI+Bhg0bomHDhg+9LjQ0FPn5+Th8+DA6d+4MANizZw/MZjNCQkKs3uPn5wcfHx9kZ2eL2k+ePIk+ffpUOEaWb4iIiGRkq/KNvQQGBqJ3794YM2YMMjIy8MMPPyAmJgZDhgyxzLw5f/48AgICkJGRAQDQaDR4/fXXsXjxYnz55Zc4ffo0pk+fjqysLIwaVfG1aNhTQkREJKPqsKLrZ599hpiYGPTs2dOyeNrixYst5+/cuYPs7GzcuHHD0vbqq6/i1q1bmDx5MvLy8tChQwfs2rULzZo1q/B7mZQQERGRiIeHB9atWyd53s/Pz2pvzbRp00TrlFSWzco3UVFR0Gg0eP/990XtmzdvhkajsXxcWlqKBQsWoF27dnB0dES9evXQp08f/PDDD5Zrli1bBnd3d/z3v+Il4idMmICWLVuKMjMiIqLqRLDR/2oim44pcXR0xJw5c3Dt2jWr5wVBwJAhQzBz5kxMmjQJmZmZ2LdvH3x9fdG9e3ds3rwZAPDyyy8jODhYVIdKS0vDsmXLsHr1atSpo779WoiIiCpC7WNKlGTTpCQsLAxeXl6Si7ds2LABX375JdauXYvRo0ejadOm6NChA1auXIlnn30Wo0ePRnFxMTQaDQwGAw4cOIDly5ejsLAQI0eORGxsLLp06WLLkImIiGRlqxVdayKbJiU6nQ6zZ8/GkiVL8L///a/M+XXr1qFly5bo379/mXOvvfYarl69il27dgEAfH19sXDhQrz++uuIjIyEs7MzZs2aZctwiYiISEVsPiX4ueeeQ8eOHZGQkFDm3MmTJxEYGGj1vnvtJ0+etLRFR0ejbdu22LZtGz755JNKLRJDRESkRizfSLPLOiVz5szBmjVrkJmZWeZcZb6Qx48fx5EjR1CnTh189913D73e2uZEat+NkYiIHi9mQbDJURPZJSl5+umnERERUWYN/pYtW1pNVABY2lu2bAkAuH37NoYPH45hw4bho48+wttvv11mpbgHWducKP/mpXLvISIiInWw24qu77//PrZt24b09HRL25AhQ3Dq1Cls27atzPXz5s1D/fr10atXLwDAzJkzkZeXhwULFmDEiBHo1asXoqOjLZv+WGNtcyJ3p0aS1xMREcmN5RtpdktK2rVrh2HDholWgBsyZAiee+45jBgxAgaDATk5Ofj555/x73//G1u3bsWqVatQt25dHDx4EHPmzIHBYICbmxsAYMWKFcjOzsaCBQsk36nX6+Hq6io6NBqupE9EROrB2TfS7Pobe+bMmaKeDY1Ggw0bNuDNN9+0bGncrVs3/PHHH9i3bx8GDBgAk8mEESNGIDo6GuHh4ZZ7vb29sWTJkgqVcYiIiKj60Qg1tQ/o/2tav4PSIYjs8rS+7bNS/rjqrnQIZYwpPfnwi2TUsa6v0iGInDVdVToEkTmCur4+C/WFSocgctN8R+kQRL5+P1jpEEQceld8szY51Grgb/d3uNa1zTsKi8/Y5Dlqwr1viIiIZFRTZ87YAgdcEBERkSqwp4SIiEhGNXUzPVtgUkJERCQjlm+kMSkhIiKSUQ2fX/JIOKaEiIiIVIE9JURERDLimBJpTEqIiIhkxPKNNJZviIiISBXYU0JERCQj9pRIY1JCREQkI6Yk0li+ISIiInUQ6KFu3bolJCQkCLdu3VI6FEEQ1BePIKgvJsZTPsZTPsZTPsZD9lLjdwm2hcLCQri5uaGgoACurq5Kh6O6eAD1xcR4GA/jYTxU/bB8Q0RERKrApISIiIhUgUkJERERqQKTkgrQ6/VISEiAXq9XOhQA6osHUF9MjKd8jKd8jKd8jIfshQNdiYiISBXYU0JERESqwKSEiIiIVIFJCREREakCkxIiks3+/ftRUlKidBgWX3/9tdIhENF9mJRQjWAymVBcXKx0GKqj0+lw6dIlpcOw6NGjB/Ly8pQOw+L555/H2LFjcf36daVDISIwKRHx9/fH1atXlQ6jjMLCQuzatQs7duzA5cuXlQ5HVS5fvow+ffrA2dkZrq6u+Otf/4rTp08rGpOa/rzUNrlObfEcOHAABw8eRPv27bF//36lw7FQ0/fQPV988QWef/55tG3bFm3btsXzzz+PL7/8UumwqIbhlOD7aLVaGI1GNGrUSOlQLI4dO4a+ffsiNzcXgiDAxcUFGzZsQEREhCLxxMbGVui6+fPn2zmSu0aOHIn/+7//w8SJE+Ho6IgVK1bA29sbe/fuleX9D1Lbn5favqe1Wi1yc3PRsGFDpUOxKCkpwbvvvov3338f48ePx1tvvQUHBwfRNXLup6K27yGz2YyhQ4fiiy++QMuWLREQEAAAyMzMxOnTp/Gvf/0L69evh0ajkSWeW7duwdHRsdxrTp06hRYtWsgSD9kWk5L7qO0HOABERETg+vXrmDt3LhwdHTFr1iz88ssvOHXqlCLx9OjRQ/Tx999/j86dO8PJycnSptFosGfPHlni8fX1xapVqyw/sE+dOoXAwEAUFxcrspCS2v68tFot3n33XTg7O5d73cSJE2WLp0+fPg/9s/nqq69kied+33zzDfr27SvqzREEARqNBqWlpbLFobbvoQULFuDdd9/FmjVr8I9//EN0buvWrYiOjsb06dPx6quvyhJPQEAA1qxZg5CQEKvn58+fj+nTp7OcW00xKbmPVqvFmjVr4ObmVu51zz77rEwRAQ0aNMA333yDp556CgCQn58PDw8P5Ofnq2I3TBcXFxw/fhz+/v6KvF+n0+H8+fPw8vKytNWtWxe//vor/Pz8ZI9HbX9eWq0WTzzxBHQ6neQ1Go0GZ86ckS2eQYMGiZJYaz755BNZ4rnnq6++wrhx49CmTRurPSXPPPOMbLGo7Xuoffv2ePXVVzFy5Eir5w0GAxYtWoSff/5ZlngmTJiAFStW4LXXXsPMmTNRq1YtAHf/QRIVFYWTJ09i8eLFGDp0qCzxkI0JZKHRaB56aLVa2WPKzc0VtTk7OwtnzpyRNQ4pzs7Owu+//67Y+7VarXDp0iVRm4uLi2JfH7X9eVmLR0lqi+fatWvC0KFDhTp16ggLFy5UOhxBENT3PeTo6Cj88ccfkudzcnIER0dHGSMShN27dwtPPvmk0LZtW+HgwYPC/PnzBScnJ+HZZ58VLl68KGssZFsOD09bHi9qK98AwG+//Qaj0Wj5WBAEZGZmoqioyNLWvn17JUJTnCAIaNmypaieff36dXTq1Ala7Z/juOWc8aGmPy+56vwVpbZ4WrdujSZNmuDIkSNo1aqV0uFYqOl7yMnJCfn5+WjSpInV84WFhQ8d42FrPXv2xC+//ILIyEiEhISgTp06WLFiBV566SVZ4yDbY1Jyn4r8wDxx4gTatm0rQzR/6tmzZ5lZC/fXduWueauJ3N38FSH156XRaGQfo/BgHEp7WDyZmZkwGAyYO3euLPG88soriIuLK7e8pQQ1fQ+FhoZi2bJlWLZsmdXzSUlJCA0NlSWW+61fvx579+5FSEgIjhw5gv379+O555576PgpUjcmJfeR+oFZVFSE9evXY9WqVTh8+LCsCcDZs2cfes39/3qytwfrxoIgICsrq8w6D3L9K27EiBGyvKeiKvLnJaeEhARV/ZDeu3cvPDw8RG3FxcX4/PPPYTAY8NNPP6F169ayJSXh4eHlnjeZTNiyZQsGDRokSzyA+r6H3nrrLXTv3h1Xr17FlClTEBAQYOm5mTdvHrZs2SLrbLfz589j5MiRyMjIwOLFixEVFYXjx49jxIgRaNOmDZKTk9GzZ0/Z4iHb4kDX+0RHR2Px4sVwcXEBcHf1SYPBgI0bN8LHxwfPP/88Bg4ciL/85S8KR/pnomQwGHDo0CHZEiWtVmv515oUJXpuBEHA4cOHkZOTA41Gg6ZNm6JTp06qKxfIraKDD5Uo//3www8wGAzYsGEDbt68icmTJ2P06NGWKady0Ol0uHjxoqVk6+rqimPHjlkGbufm5sLHx+ex7Ym8Z9OmTRg7dmyZMmi9evWwYsUKDBw4ULZY6tWrh5CQEKxatQpPPPGEpf3OnTt455138MEHH2DUqFGSPTukbkxKHmA0GrF69WoYDAYUFhZi0KBBWL58OY4fP47WrVsrHZ7iidIff/zx0GuKiopkLXHt3bsXo0aNwh9//GFJlu4lJsnJyXj66adli+WDDz7AhAkTLLNLfvjhBwQFBVmmwBYVFWHq1Kn46KOPZImnvCRSiVLApUuXsHr1aiQnJ6OgoABDhw7Fiy++iNDQUEX+jj24DMCDs8lyc3Ph7e0Ns9ksa1zA3dkkW7ZsESXaAwYMUGym240bN7Bz507L1OSWLVsiPDwcderUkTWO5cuX4+WXX5Y8f/DgQURFReHXX3+VMSqyGRkH1areP/7xD8HV1VUYOnSosH37dqGkpEQQBEFwcHAQfv31V8XiunjxopCYmCg0b95caNSokRATE6N4TA8qLCwUVqxYIQQHB8s6Q+nUqVNCnTp1hB49egibN28WsrKyhMzMTGHjxo3CM888I9StW1fW2UFarVY0c8LFxUX0fqPRKOvXJycnp0KHXBwdHYXIyEghNTVVKC0ttbQr9f384EyXB2eTyf3ndc/s2bMFBwcHQavVCl5eXoKnp6eg1WqFWrVqCR9++KGssaSlpQmBgYFCQUFBmXP5+flC69athf3798sakyAIwo0bN4QtW7YIH374ofDhhx8KW7ZsEW7cuCEIgiCYTCbZ4yHbYFJyH51OJ0yePFk4efKkqF3JBECtidI93377rTB8+HChbt26QosWLYSpU6cKGRkZsr1//Pjxwt///ner58xms/D3v/9diImJkS0etf2Se+edd4Ti4mLZ3vcwrVq1Evz8/IQ333xTyMzMtLQzKfnTnj17BK1WKyQkJAh5eXmW9qtXrwrTp08XdDqd8O2338oWT//+/YX58+dLnl+0aJEwYMAA2eIRBEHYsmWL0LBhwzJLNjRs2FDYunWrrLGQbTEpuU96erowevRowcXFRQgODhaWLFkiXL58WdEEQI2Jkpp6btq0aVPuD6GtW7cKbdq0kS0etf2Se7DnRg2+//57ITo6WnB2dhaeeuopYf78+YKDg4Pw22+/yR6LRqMR9u7dKxw/flw4fvy4ULduXWHHjh2Wj9PS0mRPSgYNGiSMHTtW8vyYMWOEIUOGyBZPkyZNyv2zyczMFHx9fWWL54cffhBq1aolDBw4UPjxxx+Fa9euCdeuXRN++OEH4fnnnxdq164tpKenyxYP2RaTEiuuX78uGAwGoWvXrkKtWrUErVYrLFy4UCgsLJQ9FrUlSmrruXFxcRHOnj0ref7MmTOCs7OzbPGoLSlR22Jl9ysqKhJWrlwphIaGChqNRujevbuwcuXKMovh2dO9BRGlFkpUYsFEPz8/4bvvvpM8v3//fsHPz0+2ePR6vXDq1CnJ86dOnZJ18bQ+ffqUm7SNHTtW6NOnj2zxkG1xoOtDZGdnw2Aw4NNPP0V+fj569eqFrVu3yh5HcXExUlJSkJycjIyMDJSWlmL+/PkYOXKkZbaQHBwcHDBx4kSMGzdOtOFVrVq1VDFQ8UFyz554cK+ZqVOn4vXXX0eDBg0A3B3oGh8fL2s8atoAb+bMmZgyZUqZwZH31if59NNPkZeXhzt37sgST0UGbgPAk08+aedI/lSnTh2cPHlSNLPkfv/73//QokUL3Lx5U5Z4mjVrhnnz5mHAgAFWz3/11VeYMmWKbFsVeHh44Ntvv0W7du2snv/555/xzDPP4Nq1a7LEQ7bFpKSCSktLsW3bNiQnJyuSlNxPyUTpp59+gsFgQEpKCgIDA/HSSy9hyJAh8Pb2Viwp2bNnT5m1L+65cuUKevXqJVsS4OfnV6FpyHKtRaHVauHm5vbQmORa8fbBKbgPKikpwdatW/H888/LEo9UkqQktSXaEyZMwL59+3Dw4MEyK7fevHkTwcHB6NGjBxYvXixLPE5OTsjKypJMFP/44w8EBATIlrSRbTEpqcaUTJTU0nOjtimvaqPVarFw4cKHbjIp1yJ0atuJ+2FJkhIetrOz3L1tubm5eOqpp6DT6RATE2NZjj8rKwtJSUkoLS3FkSNH4OnpKUs87du3x+TJkxEdHW31fHJyMhYuXCjbBoFkW0xK6JEp2XOjtu73v//97/jqq6/g7u4uy/seRm1JgNrKSWr7+gDq620D7v49GzduHHbu3ClaCygiIgJJSUlo2rSpbLEsWLAA7777Lj799FP07dtXdG7Hjh0YMWIE3nzzTcTGxsoWE9kOkxKyGTWVuJSitl9yausJUFs5SW1Jktpdu3YNp0+fhiAIaNGiBerVqyd7DGazGYMHD8bGjRvRqlUrBAYGWpa9P3XqFAYMGIAvvvhCtCEnVR9MSqhaGz58OJKSkiwlo3vjWmrVqqVIPGpLStQYj9rKSWpKkgBgz549iImJwU8//QRXV1fRuYKCAnTp0gXLly9Ht27dZItJjVJSUrBu3TrRCrNDhgzBkCFDFI6MHgWTEqrWHrZ3idweNvD2HiX2mlEDJkkP9+yzz6JHjx6YPHmy1fOLFy/G3r17sWnTJtliIpILkxKq1h62d4kS8XDgrTQ1lpPUlCQBd8c/paamIjAw0Or5rKwshIeH49y5czJHpg73/o6VR6PRoKSkRKaIyJYclA6AqKY5cOAAxyhIUNu/gdS4i3Rubm655UcHBwdcvnxZxojUpbweovT0dCxevFiRDRTJNpiUULX322+/wWg0Arj7Sy8rKwvXr18XXSNnuaRJkyaq+pe3mqjtl4XakiQAaNy4MU6cOIHmzZtbPf/zzz/D29tb5qjU45///GeZtuzsbEybNg3btm3DsGHDMHPmTAUiI1tg+YaqtfLKJffIWS5RYzmAqhe1LVamZhcuXEBCQgLWrFmDiIgIJCYmom3btkqHRY+ASQlVaxVZp6SoqEi2H1QhISGYMGECIiMjAQBxcXEwmUyW8zqdDrNmzSrzy4boHrUtVqZGBQUFmD17NpYsWYKOHTtizpw5j/1spJqCSQnVSEVFRVi/fj0MBgMOHTokW0/J8uXLsWPHDmzbtg3A3YG3bdq0gZOTE4C7v1jeeOMNyZkVRIC6FitTmw8++ABz5syBl5cXZs+ebbWcQ9UXkxKqUfbv3w+DwYCNGzfCx8cHzz//PAYOHIi//OUvsry/W7dueOONN9C/f38AZWcD/ec//0FSUhLS09NliYeqNzUsVqY2Wq0WTk5OCAsLg06nk7zuq6++kjEqshUOdKVqz2g0YvXq1TAYDCgsLMSgQYNgMpmwefNm2TcIPHXqlGj3UkdHR9HKksHBwRg/frysMVH1Va9ePdkS6upi+PDhqpw1RbbBpISqtf79+2P//v3o168fFi5ciN69e0On02H58uWKxFNQUCAaQ/Lg1E2z2Sw6T0SVs3r1aqVDIDtiUkLV2v/93/9h4sSJGDduHFq0aKF0OHjiiSdw4sQJy+DEB/3888944oknZI6KiKh64I5FVK19//33KCoqQufOnRESEoKlS5fiypUrisXTt29fxMfH49atW2XO3bx5E++88w769eunQGREROrHga5UIxQXFyMlJQXJycnIyMhAaWkp5s+fj5EjR1o265NDbm4uOnbsiNq1ayMmJgYtW7YEcHdxp6VLl6KkpARHjx59rKdzEhFJYVJCNU52djYMBgM+/fRT5Ofno1evXti6dats7z979izGjRuHXbt2iaZz9urVCx999JFi+/IQEakdkxKqsUpLS7Ft2zYkJyfLmpTck5eXh9OnTwMAmjdv/tCdg4mIHndMSoiIiEgVONCViIiIVIFJCREREakCkxIiIiJSBSYlREREpApMSoiIiEgVmJQQERGRKjApISIiIlVgUkJERESq8P8AnF5iQKqAzcgAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "developed-weapon"
      },
      "source": [
        "### Prepare data structure for spark"
      ],
      "id": "developed-weapon"
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "theoretical-transsexual",
        "outputId": "2f703046-12c6-45b2-8503-bfc8d8da5ebb"
      },
      "source": [
        "input_cols = df.columns\n",
        "input_cols.remove('NOX')\n",
        "input_cols"
      ],
      "id": "theoretical-transsexual",
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['AT', 'AP', 'AH', 'AFDP', 'GTEP', 'TIT', 'TAT', 'TEY', 'CDP', 'CO']"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "binary-consent"
      },
      "source": [
        "vector_assembler = feature.VectorAssembler(inputCols = input_cols,\n",
        "                                           outputCol = 'features')\n",
        "df2 = vector_assembler.transform(dataset = df)"
      ],
      "id": "binary-consent",
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "entertaining-substance",
        "outputId": "e617b978-a939-4459-8b17-e6ca7db9d695"
      },
      "source": [
        "df2.show()"
      ],
      "id": "entertaining-substance",
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+------+------+------+------+------+------+------+------+------+------+--------------------+\n",
            "|      AT|    AP|    AH|  AFDP|  GTEP|   TIT|   TAT|   TEY|   CDP|    CO|   NOX|            features|\n",
            "+--------+------+------+------+------+------+------+------+------+------+------+--------------------+\n",
            "|  1.9532|1020.1|84.985|2.5304|20.116|1048.7|544.92|116.27|10.799|7.4491|113.25|[1.9532,1020.1,84...|\n",
            "|  1.2191|1020.1|87.523|2.3937|18.584|1045.5| 548.5|109.18|10.347|6.4684|112.02|[1.2191,1020.1,87...|\n",
            "| 0.94915|1022.2|78.335|2.7789|22.264|1068.8|549.95|125.88|11.256|3.6335|88.147|[0.94915,1022.2,7...|\n",
            "|  1.0075|1021.7|76.942| 2.817|23.358|1075.2|549.63|132.21|11.702|3.1972|87.078|[1.0075,1021.7,76...|\n",
            "|  1.2858|1021.6|76.732|2.8377|23.483|1076.2|549.68|133.58|11.737|2.3833|82.515|[1.2858,1021.6,76...|\n",
            "|  1.8319|1021.7|76.411| 2.841|23.495|1076.4|549.92|133.58|11.829|2.0812|81.193|[1.8319,1021.7,76...|\n",
            "|   2.074|1022.0|75.974|2.7981|22.945|1073.7|549.98|131.53|11.687|2.2529|83.171|[2.074,1022.0,75....|\n",
            "|  1.7824|1022.6|73.535|2.8327|23.337|1075.7|550.01|133.18|11.745| 3.735|85.749|[1.7824,1022.6,73...|\n",
            "|   1.593|1023.2|72.873|2.8729|23.654|1078.5|550.06|135.38|11.772|3.6398|86.491|[1.593,1023.2,72....|\n",
            "|  1.6819|1023.8|72.441|2.9058|23.463|1077.9|550.12|134.86|11.742|3.5866|86.328|[1.6819,1023.8,72...|\n",
            "|  1.9002|1024.5|71.376|2.9126|23.562|1078.2|550.12|134.98| 11.77|3.5605|84.117|[1.9002,1024.5,71...|\n",
            "|  1.7797|1025.1|68.528|2.8725|23.276|1077.0|550.03|134.21|11.782|3.6902|85.317|[1.7797,1025.1,68...|\n",
            "|  1.1722|1025.3|65.626|  2.85|23.215|1077.3|550.09|134.44|11.873|3.1766|86.431|[1.1722,1025.3,65...|\n",
            "| 0.72327|1025.4|64.393|2.8395|23.101|1076.2|550.15|133.82|11.797| 2.562|84.708|[0.72327,1025.4,6...|\n",
            "| 0.48348|1025.8|64.144|2.8746|23.299|1078.2|549.96|135.36|11.897|2.1854|84.104|[0.48348,1025.8,6...|\n",
            "| 0.42953|1025.9|64.504| 2.866|23.069|1077.0|549.73|134.47|11.805|2.4286|83.869|[0.42953,1025.9,6...|\n",
            "| 0.48238|1026.1|63.519|2.5481|20.185|1037.1| 535.4|115.92| 10.86|12.659|118.27|[0.48238,1026.1,6...|\n",
            "|0.059447|1026.0|52.124|2.4651|18.478|1048.4|548.78| 111.2|10.384|4.6896|104.56|[0.059447,1026.0,...|\n",
            "|-0.21598|1026.7|51.978|2.5301|19.438|1050.4|546.71|115.46|10.767|7.0983|116.96|[-0.21598,1026.7,...|\n",
            "|  0.0167|1026.8|54.805|3.3399|27.408|1085.6|540.72|148.05| 12.79|3.8632| 75.64|[0.0167,1026.8,54...|\n",
            "+--------+------+------+------+------+------+------+------+------+------+------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "covered-posting"
      },
      "source": [
        "### Dataset split"
      ],
      "id": "covered-posting"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "parental-phoenix"
      },
      "source": [
        "splits = df2.randomSplit([0.7, 0.3], seed = 101)"
      ],
      "id": "parental-phoenix",
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "another-massage"
      },
      "source": [
        "df_tr = splits[0]\n",
        "df_tst = splits[1]"
      ],
      "id": "another-massage",
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bizarre-record",
        "outputId": "408c0d5f-8221-4bb4-ee57-0236c1b54554"
      },
      "source": [
        "df_tr.count()"
      ],
      "id": "bizarre-record",
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "5165"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "electrical-significance",
        "outputId": "2ba7d9ff-437a-4a21-8f35-e1417a38cbe0"
      },
      "source": [
        "df_tst.count()"
      ],
      "id": "electrical-significance",
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2219"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "seven-multimedia"
      },
      "source": [
        "### Modeling"
      ],
      "id": "seven-multimedia"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "worth-creativity"
      },
      "source": [
        "from pyspark.ml.regression import LinearRegression, DecisionTreeRegressor, RandomForestRegressor, GBTRegressor\n",
        "from pyspark.ml.evaluation import RegressionEvaluator\n",
        "from pyspark.ml import feature"
      ],
      "id": "worth-creativity",
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "forward-japanese"
      },
      "source": [
        "#### Linear Regression"
      ],
      "id": "forward-japanese"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nominated-motion"
      },
      "source": [
        "lr = LinearRegression(labelCol = 'NOX')\n",
        "lr_model = lr.fit(df_tr)"
      ],
      "id": "nominated-motion",
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "overall-practitioner",
        "outputId": "14a387a3-b593-470f-cdad-25a435f437b2"
      },
      "source": [
        "lr_model.coefficients"
      ],
      "id": "overall-practitioner",
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DenseVector([-1.5642, -0.318, -0.2524, -0.9052, 0.1939, 1.382, -0.8548, -2.4234, 8.7649, 2.6103])"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "israeli-terrain",
        "outputId": "9c22978b-3c67-451b-d4e6-ab9f28cc596e"
      },
      "source": [
        "lr_model.intercept"
      ],
      "id": "israeli-terrain",
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "-388.41563146668244"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "comprehensive-ebony",
        "outputId": "6731fde3-0728-4ed5-b492-d4b18516d66b"
      },
      "source": [
        "lr_model.summary.coefficientStandardErrors"
      ],
      "id": "comprehensive-ebony",
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.04238483535086803,\n",
              " 0.0170199312711927,\n",
              " 0.00843055000449291,\n",
              " 0.6435605927367861,\n",
              " 0.06115870946984003,\n",
              " 0.1248753643373593,\n",
              " 0.1564139359101661,\n",
              " 0.1290394059456014,\n",
              " 1.8178826898724396,\n",
              " 0.06157189407286876,\n",
              " 34.4769353830434]"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hollow-beatles",
        "outputId": "268048a1-6b5d-483a-9149-a6d595414c6b"
      },
      "source": [
        "lr_model.summary.r2"
      ],
      "id": "hollow-beatles",
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.6984970794733001"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bDXiaJW2WBhV",
        "outputId": "305cc124-bd45-4eef-a5f7-4a2a6f03d707"
      },
      "source": [
        "lr_model.summary.r2adj"
      ],
      "id": "bDXiaJW2WBhV",
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.6979120912689409"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "false-commerce",
        "outputId": "be447e6b-3f7d-4ffa-a389-19e3c0a667db"
      },
      "source": [
        "lr_model.summary.rootMeanSquaredError"
      ],
      "id": "false-commerce",
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "6.115640884500553"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "broad-printer",
        "outputId": "1f69a6a1-8bd6-4674-f191-e2a88de84706"
      },
      "source": [
        "lr_model.summary.residuals.show()"
      ],
      "id": "broad-printer",
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------------+\n",
            "|          residuals|\n",
            "+-------------------+\n",
            "|  -8.96839669067878|\n",
            "|-3.8627793536073796|\n",
            "|  2.874119181377779|\n",
            "| -4.370828630213495|\n",
            "|  6.741081005649448|\n",
            "| -4.895551134729466|\n",
            "|  2.935589937784954|\n",
            "| 11.242572535757546|\n",
            "|-13.814809251439783|\n",
            "| -4.204938642836247|\n",
            "| -1.150566718168335|\n",
            "|-3.1309270882355875|\n",
            "| -0.462771358884261|\n",
            "| -1.480480408065901|\n",
            "| -4.368526197491157|\n",
            "| -5.203989916985471|\n",
            "|-1.5870479937222512|\n",
            "| -1.167041978032053|\n",
            "|-3.6254298117720225|\n",
            "|-0.8221764127413067|\n",
            "+-------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "separated-funeral"
      },
      "source": [
        "pred = lr_model.transform(df_tst)"
      ],
      "id": "separated-funeral",
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "widespread-vintage",
        "outputId": "c9e9c5b9-2a7b-4410-9857-1a4cb9348d9c"
      },
      "source": [
        "pred.show()"
      ],
      "id": "widespread-vintage",
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+------+------+------+------+------+------+------+------+------+------+--------------------+------------------+\n",
            "|      AT|    AP|    AH|  AFDP|  GTEP|   TIT|   TAT|   TEY|   CDP|    CO|   NOX|            features|        prediction|\n",
            "+--------+------+------+------+------+------+------+------+------+------+------+--------------------+------------------+\n",
            "| -5.8956|1034.1|70.851|4.3541|36.533|1100.0| 518.3|177.88|15.083|1.9988|56.206|[-5.8956,1034.1,7...|60.673683426599666|\n",
            "| -5.8847|1033.6| 66.92|4.2815|36.218|1100.0|520.24|176.55|14.891|1.2721|56.124|[-5.8847,1033.6,6...| 59.79764158271553|\n",
            "| -5.4839|1034.5|72.581|4.2569| 35.86|1100.0|520.94|176.09|14.838|2.1367|58.693|[-5.4839,1034.5,7...| 59.71699111924727|\n",
            "| -5.4155|1034.6|58.816|4.0521|34.179|1099.9| 525.8|172.89|14.492|2.7617|63.743|[-5.4155,1034.6,5...| 64.97357662014082|\n",
            "| -5.0675|1034.2|56.755|4.1073|34.732|1099.8|524.19|173.67|14.636|2.6749|62.076|[-5.0675,1034.2,5...| 65.51736826194883|\n",
            "| -4.9758|1036.5|70.657|4.2498|35.593|1099.9|521.44|175.41|14.803|1.5928|57.449|[-4.9758,1036.5,7...| 58.08233972300309|\n",
            "|  -4.938|1033.4|55.609|4.2813|36.573|1100.0| 518.3|177.49|15.039|2.6486| 59.26|[-4.938,1033.4,55...| 65.57547593621013|\n",
            "| -4.9279|1034.9|71.525|4.1793|35.044|1100.0|523.31| 174.2|14.728|2.3491|60.118|[-4.9279,1034.9,7...|61.043294075347774|\n",
            "| -3.6358|1030.2|49.327|4.0361|34.743|1099.9|525.95|172.01|14.548|2.5435|61.681|[-3.6358,1030.2,4...| 68.03397166059233|\n",
            "| -3.3038|1025.4|59.547|3.5164| 29.58|1100.2|542.49|159.23|13.375|2.8682|75.366|[-3.3038,1025.4,5...| 73.74381324829568|\n",
            "|  -3.289|1027.1|51.789|3.3043|28.014|1100.0|547.95|155.48|13.083|3.4726|85.367|[-3.289,1027.1,51...| 78.18919784354517|\n",
            "| -3.0217|1036.3|57.921| 4.233|35.467|1099.9|522.06|173.94|14.762|3.1926| 66.41|[-3.0217,1036.3,5...| 65.14428227095448|\n",
            "| -1.9402|1036.0|54.925|4.3121|35.893|1099.9|521.15|174.22|14.836|1.7114|57.337|[-1.9402,1036.0,5...|61.196885449012484|\n",
            "| -1.4086|1035.8|51.721|4.2906|35.677|1099.9|521.52|173.57|14.842|3.0832|61.519|[-1.4086,1035.8,5...| 66.10771226022291|\n",
            "| -1.4042|1021.1|68.896|2.4012|18.991|1040.7|543.56|112.05|10.534|9.1157|106.53|[-1.4042,1021.1,6...| 91.33655260238879|\n",
            "| -1.1782|1022.1|71.123|2.5283|19.552|1034.3|536.35|112.69|10.667|3.4947|110.99|[-1.1782,1022.1,7...| 72.35735431947865|\n",
            "| -0.9986|1022.8| 70.71|2.9023|23.462|1075.1| 550.3|133.03| 11.71|1.8773|84.164|[-0.9986,1022.8,7...| 72.46563754717533|\n",
            "|-0.51781|1023.3|69.287| 2.867|23.692|1075.1|550.34|133.77|11.734|3.3021|85.342|[-0.51781,1023.3,...| 74.09236123483703|\n",
            "| -0.0821|1023.7|64.949| 2.872|23.898|1075.4|550.21|133.77|11.737|3.7902|85.485|[-0.0821,1023.7,6...| 76.24024679735919|\n",
            "|  0.0291|1020.7|78.937|2.6741|18.752|1044.0|547.45|109.59|10.446|9.8027|113.29|[0.0291,1020.7,78...| 94.61249911692965|\n",
            "+--------+------+------+------+------+------+------+------+------+------+------+--------------------+------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "confidential-rwanda",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f5e08732-e9b1-4a9a-f26e-b4e751e0c767"
      },
      "source": [
        "evaluator = RegressionEvaluator(labelCol = 'NOX', metricName = 'r2')\n",
        "evaluator.evaluate(pred)"
      ],
      "id": "confidential-rwanda",
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.704573645921791"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "verified-african"
      },
      "source": [
        "#### Decision Tree Regression"
      ],
      "id": "verified-african"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "suitable-craft"
      },
      "source": [
        "dt = DecisionTreeRegressor(labelCol = 'NOX')\n",
        "dtr_model = dt.fit(df_tr)"
      ],
      "id": "suitable-craft",
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t5iguf3HXOxp",
        "outputId": "3852ce2a-a191-442f-f591-de6668f72e14"
      },
      "source": [
        "df_tr.columns[:-2]"
      ],
      "id": "t5iguf3HXOxp",
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['AT', 'AP', 'AH', 'AFDP', 'GTEP', 'TIT', 'TAT', 'TEY', 'CDP', 'CO']"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MYwXvWt_YJN0",
        "outputId": "1a5fbbf6-6efc-4912-8336-5dcb0d32f701"
      },
      "source": [
        "dtr_model.featureImportances"
      ],
      "id": "MYwXvWt_YJN0",
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "SparseVector(10, {0: 0.5386, 1: 0.0018, 2: 0.0106, 3: 0.2779, 4: 0.0103, 5: 0.0342, 6: 0.017, 7: 0.0085, 9: 0.1012})"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q3VUD9rpYJpW",
        "outputId": "cf355dcf-ec03-4ef0-a932-5587315c4846"
      },
      "source": [
        "dtr_model.featureImportances.toArray()"
      ],
      "id": "Q3VUD9rpYJpW",
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.53859252, 0.00178208, 0.01059783, 0.27789723, 0.01032851,\n",
              "       0.03415561, 0.01698441, 0.00846611, 0.        , 0.1011957 ])"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wired-kennedy",
        "outputId": "4c599cfe-6ff6-448c-b4af-fa3df823b10c"
      },
      "source": [
        "dtr_model.featureImportances.toArray().tolist()"
      ],
      "id": "wired-kennedy",
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.5385925239233093,\n",
              " 0.0017820820835477216,\n",
              " 0.010597825028970383,\n",
              " 0.27789722914102,\n",
              " 0.010328510448363847,\n",
              " 0.034155614264764385,\n",
              " 0.016984408270299414,\n",
              " 0.008466107120362264,\n",
              " 0.0,\n",
              " 0.10119569971936287]"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd"
      ],
      "metadata": {
        "id": "ioZn272DhACe"
      },
      "id": "ioZn272DhACe",
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "RERf2pAmXxga",
        "outputId": "dcdf6064-0c22-4c06-93f5-ad8101b182c6"
      },
      "source": [
        "pd.DataFrame(zip(df_tr.columns[:-2], dtr_model.featureImportances.toArray()))"
      ],
      "id": "RERf2pAmXxga",
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "      0         1\n",
              "0    AT  0.538593\n",
              "1    AP  0.001782\n",
              "2    AH  0.010598\n",
              "3  AFDP  0.277897\n",
              "4  GTEP  0.010329\n",
              "5   TIT  0.034156\n",
              "6   TAT  0.016984\n",
              "7   TEY  0.008466\n",
              "8   CDP  0.000000\n",
              "9    CO  0.101196"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a3def364-d77b-4366-8476-31c9fa002d21\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>AT</td>\n",
              "      <td>0.538593</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>AP</td>\n",
              "      <td>0.001782</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>AH</td>\n",
              "      <td>0.010598</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>AFDP</td>\n",
              "      <td>0.277897</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>GTEP</td>\n",
              "      <td>0.010329</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>TIT</td>\n",
              "      <td>0.034156</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>TAT</td>\n",
              "      <td>0.016984</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>TEY</td>\n",
              "      <td>0.008466</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>CDP</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>CO</td>\n",
              "      <td>0.101196</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a3def364-d77b-4366-8476-31c9fa002d21')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-a3def364-d77b-4366-8476-31c9fa002d21 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-a3def364-d77b-4366-8476-31c9fa002d21');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-a0da8c4b-a3ca-4f61-9344-447e192bec8a\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-a0da8c4b-a3ca-4f61-9344-447e192bec8a')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-a0da8c4b-a3ca-4f61-9344-447e192bec8a button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"pd\",\n  \"rows\": 10,\n  \"fields\": [\n    {\n      \"column\": 0,\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"CDP\",\n          \"AP\",\n          \"TIT\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": 1,\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.17616225644176156,\n        \"min\": 0.0,\n        \"max\": 0.5385925239233093,\n        \"num_unique_values\": 10,\n        \"samples\": [\n          0.0,\n          0.0017820820835477216,\n          0.034155614264764385\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iuIej8tQXKUq",
        "outputId": "fefccffd-0a61-471f-aaad-100e0254cf38"
      },
      "source": [
        "spark.createDataFrame(data = zip(df_tr.columns[:-2], dtr_model.featureImportances.toArray().tolist()),\n",
        "                      schema = ['variables', 'importance']).show()"
      ],
      "id": "iuIej8tQXKUq",
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+--------------------+\n",
            "|variables|          importance|\n",
            "+---------+--------------------+\n",
            "|       AT|  0.5385925239233093|\n",
            "|       AP|0.001782082083547...|\n",
            "|       AH|0.010597825028970383|\n",
            "|     AFDP|    0.27789722914102|\n",
            "|     GTEP|0.010328510448363847|\n",
            "|      TIT|0.034155614264764385|\n",
            "|      TAT|0.016984408270299414|\n",
            "|      TEY|0.008466107120362264|\n",
            "|      CDP|                 0.0|\n",
            "|       CO| 0.10119569971936287|\n",
            "+---------+--------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "handed-transport"
      },
      "source": [
        "pred = dtr_model.transform(df_tst)"
      ],
      "id": "handed-transport",
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "boolean-concentration",
        "outputId": "e9ecfc75-4d37-4bd8-83c2-7003e2117a55"
      },
      "source": [
        "evaluator.evaluate(pred)"
      ],
      "id": "boolean-concentration",
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7990927688414038"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "liberal-notice"
      },
      "source": [
        "#### Random Forest Regression"
      ],
      "id": "liberal-notice"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "opposite-prime"
      },
      "source": [
        "rfr = RandomForestRegressor(labelCol = 'NOX')\n",
        "rfr_model = rfr.fit(df_tr)\n",
        "pred = rfr_model.transform(df_tst)"
      ],
      "id": "opposite-prime",
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "evaluator.evaluate(pred)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W7GntxKkh0WR",
        "outputId": "258b492c-da41-4fc5-ec8c-b35a8989c287"
      },
      "id": "W7GntxKkh0WR",
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8266529856781204"
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "short-sailing"
      },
      "source": [
        "#### Gradient Boosted Trees Regression"
      ],
      "id": "short-sailing"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hired-sleeve"
      },
      "source": [
        "gbtr = GBTRegressor(labelCol = 'NOX')\n",
        "gbtr_model = gbtr.fit(df_tr)\n",
        "pred = gbtr_model.transform(df_tst)"
      ],
      "id": "hired-sleeve",
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "evaluator.evaluate(pred)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9dlvEiY9iy0Z",
        "outputId": "1cd7f3a1-204b-4678-dad6-a5fedc625cb5"
      },
      "id": "9dlvEiY9iy0Z",
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8637498037880095"
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    }
  ]
}