4.4.outlier_treatement.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1220ce71",
   "metadata": {},
   "source": [
    "<img src=\"./images/shouke_logo.png\"\n",
    "     style=\"float: right\"\n",
    "     width=100\n",
    "     style=\"padding-bottom:100px;\"/>\n",
    "<br>\n",
    "<br>\n",
    "\n",
    "<table style=\"float:center;\">\n",
    "    <tr>\n",
    "        <td>\n",
    "            <img src='./images/python-logo.png'width=130>\n",
    "        </td>\n",
    "        <td>\n",
    "            <img src='./images/pandas-logo.png'width=150>\n",
    "        </td>\n",
    "    </tr>\n",
    "</table>\n",
    "\n",
    "<h1 style='text-align: center;'>Treating Outliers of DataFrame</h1>\n",
    "<h3 style='text-align: center;'>Shouke Wei, Ph.D. Professor</h3>\n",
    "<h4 style='text-align: center;'>Email: shouke.wei@gmail.com</h4>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6cef670f",
   "metadata": {},
   "source": [
    "## Objective\n",
    "- learn how to treat outliers in dataframe with some handy methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "830ee264",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prov</th>\n",
       "      <th>gdpr</th>\n",
       "      <th>year</th>\n",
       "      <th>gdp</th>\n",
       "      <th>pop</th>\n",
       "      <th>finv</th>\n",
       "      <th>trade</th>\n",
       "      <th>fexpen</th>\n",
       "      <th>uinc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2000</td>\n",
       "      <td>1.074125</td>\n",
       "      <td>8.650000</td>\n",
       "      <td>0.314513</td>\n",
       "      <td>1.408147</td>\n",
       "      <td>0.108032</td>\n",
       "      <td>0.976157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2001</td>\n",
       "      <td>1.203925</td>\n",
       "      <td>8.733000</td>\n",
       "      <td>0.348443</td>\n",
       "      <td>14.609701</td>\n",
       "      <td>0.132133</td>\n",
       "      <td>1.041519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2002</td>\n",
       "      <td>1.350242</td>\n",
       "      <td>8.842000</td>\n",
       "      <td>0.385078</td>\n",
       "      <td>1.830169</td>\n",
       "      <td>0.152108</td>\n",
       "      <td>1.113720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2003</td>\n",
       "      <td>1.584464</td>\n",
       "      <td>8.963000</td>\n",
       "      <td>0.481320</td>\n",
       "      <td>2.346735</td>\n",
       "      <td>0.169563</td>\n",
       "      <td>1.238043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2004</td>\n",
       "      <td>1.886462</td>\n",
       "      <td>9.052298</td>\n",
       "      <td>0.587002</td>\n",
       "      <td>2.955899</td>\n",
       "      <td>0.185295</td>\n",
       "      <td>1.362765</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        prov   gdpr  year       gdp       pop      finv      trade    fexpen  \\\n",
       "0  Guangdong  First  2000  1.074125  8.650000  0.314513   1.408147  0.108032   \n",
       "1  Guangdong  First  2001  1.203925  8.733000  0.348443  14.609701  0.132133   \n",
       "2  Guangdong  First  2002  1.350242  8.842000  0.385078   1.830169  0.152108   \n",
       "3  Guangdong  First  2003  1.584464  8.963000  0.481320   2.346735  0.169563   \n",
       "4  Guangdong  First  2004  1.886462  9.052298  0.587002   2.955899  0.185295   \n",
       "\n",
       "       uinc  \n",
       "0  0.976157  \n",
       "1  1.041519  \n",
       "2  1.113720  \n",
       "3  1.238043  \n",
       "4  1.362765  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import required packages\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# read data\n",
    "df = pd.read_csv('./data/gdp_china_mis_cl.csv')\n",
    "\n",
    "# display the first 5 rows\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c91301f",
   "metadata": {},
   "source": [
    "## The results of outliner detection using quantile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c910c597",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.017796237800000003, 7.607179460000017)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min_threshold, max_threshold = df['trade'].quantile([0.01,0.99])\n",
    "min_threshold, max_threshold                                               "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "14b47bcf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prov</th>\n",
       "      <th>gdpr</th>\n",
       "      <th>year</th>\n",
       "      <th>gdp</th>\n",
       "      <th>pop</th>\n",
       "      <th>finv</th>\n",
       "      <th>trade</th>\n",
       "      <th>fexpen</th>\n",
       "      <th>uinc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2001</td>\n",
       "      <td>1.203925</td>\n",
       "      <td>8.733</td>\n",
       "      <td>0.348443</td>\n",
       "      <td>14.609701</td>\n",
       "      <td>0.132133</td>\n",
       "      <td>1.041519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2017</td>\n",
       "      <td>4.455283</td>\n",
       "      <td>9.392</td>\n",
       "      <td>4.449690</td>\n",
       "      <td>0.000523</td>\n",
       "      <td>0.821552</td>\n",
       "      <td>2.955790</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         prov   gdpr  year       gdp    pop      finv      trade    fexpen  \\\n",
       "1   Guangdong  First  2001  1.203925  8.733  0.348443  14.609701  0.132133   \n",
       "93      Henan  Fifth  2017  4.455283  9.392  4.449690   0.000523  0.821552   \n",
       "\n",
       "        uinc  \n",
       "1   1.041519  \n",
       "93  2.955790  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# display the rows with outliers\n",
    "df[(df['trade'] > max_threshold)|(df['trade'] < min_threshold)]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64e7dd4b",
   "metadata": {},
   "source": [
    "## 1. Remove the outliers\n",
    "### (1) keep the non-outlier rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a7647e4e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prov</th>\n",
       "      <th>gdpr</th>\n",
       "      <th>year</th>\n",
       "      <th>gdp</th>\n",
       "      <th>pop</th>\n",
       "      <th>finv</th>\n",
       "      <th>trade</th>\n",
       "      <th>fexpen</th>\n",
       "      <th>uinc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2000</td>\n",
       "      <td>1.074125</td>\n",
       "      <td>8.650000</td>\n",
       "      <td>0.314513</td>\n",
       "      <td>1.408147</td>\n",
       "      <td>0.108032</td>\n",
       "      <td>0.976157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2002</td>\n",
       "      <td>1.350242</td>\n",
       "      <td>8.842000</td>\n",
       "      <td>0.385078</td>\n",
       "      <td>1.830169</td>\n",
       "      <td>0.152108</td>\n",
       "      <td>1.113720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2003</td>\n",
       "      <td>1.584464</td>\n",
       "      <td>8.963000</td>\n",
       "      <td>0.481320</td>\n",
       "      <td>2.346735</td>\n",
       "      <td>0.169563</td>\n",
       "      <td>1.238043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2004</td>\n",
       "      <td>1.886462</td>\n",
       "      <td>9.052298</td>\n",
       "      <td>0.587002</td>\n",
       "      <td>2.955899</td>\n",
       "      <td>0.185295</td>\n",
       "      <td>1.362765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2005</td>\n",
       "      <td>2.255737</td>\n",
       "      <td>9.194000</td>\n",
       "      <td>0.697793</td>\n",
       "      <td>3.505761</td>\n",
       "      <td>0.228907</td>\n",
       "      <td>1.476994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2013</td>\n",
       "      <td>3.219130</td>\n",
       "      <td>9.413000</td>\n",
       "      <td>2.608746</td>\n",
       "      <td>0.371325</td>\n",
       "      <td>0.558231</td>\n",
       "      <td>2.174067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2014</td>\n",
       "      <td>3.493824</td>\n",
       "      <td>9.436000</td>\n",
       "      <td>3.078217</td>\n",
       "      <td>0.399111</td>\n",
       "      <td>0.602869</td>\n",
       "      <td>2.367206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2015</td>\n",
       "      <td>3.700216</td>\n",
       "      <td>9.480000</td>\n",
       "      <td>3.566035</td>\n",
       "      <td>0.459535</td>\n",
       "      <td>0.679935</td>\n",
       "      <td>2.557561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2016</td>\n",
       "      <td>4.047179</td>\n",
       "      <td>9.532000</td>\n",
       "      <td>4.041509</td>\n",
       "      <td>0.471385</td>\n",
       "      <td>0.745374</td>\n",
       "      <td>2.723292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2018</td>\n",
       "      <td>4.805586</td>\n",
       "      <td>9.360000</td>\n",
       "      <td>4.450960</td>\n",
       "      <td>0.551170</td>\n",
       "      <td>0.921773</td>\n",
       "      <td>3.187420</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>93 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         prov   gdpr  year       gdp       pop      finv     trade    fexpen  \\\n",
       "0   Guangdong  First  2000  1.074125  8.650000  0.314513  1.408147  0.108032   \n",
       "2   Guangdong  First  2002  1.350242  8.842000  0.385078  1.830169  0.152108   \n",
       "3   Guangdong  First  2003  1.584464  8.963000  0.481320  2.346735  0.169563   \n",
       "4   Guangdong  First  2004  1.886462  9.052298  0.587002  2.955899  0.185295   \n",
       "5   Guangdong  First  2005  2.255737  9.194000  0.697793  3.505761  0.228907   \n",
       "..        ...    ...   ...       ...       ...       ...       ...       ...   \n",
       "89      Henan  Fifth  2013  3.219130  9.413000  2.608746  0.371325  0.558231   \n",
       "90      Henan  Fifth  2014  3.493824  9.436000  3.078217  0.399111  0.602869   \n",
       "91      Henan  Fifth  2015  3.700216  9.480000  3.566035  0.459535  0.679935   \n",
       "92      Henan  Fifth  2016  4.047179  9.532000  4.041509  0.471385  0.745374   \n",
       "94      Henan  Fifth  2018  4.805586  9.360000  4.450960  0.551170  0.921773   \n",
       "\n",
       "        uinc  \n",
       "0   0.976157  \n",
       "2   1.113720  \n",
       "3   1.238043  \n",
       "4   1.362765  \n",
       "5   1.476994  \n",
       "..       ...  \n",
       "89  2.174067  \n",
       "90  2.367206  \n",
       "91  2.557561  \n",
       "92  2.723292  \n",
       "94  3.187420  \n",
       "\n",
       "[93 rows x 9 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a new dataframe excluding the outlier rows\n",
    "df_new = df[(df['trade'] < max_threshold)&(df['trade'] > min_threshold)]\n",
    "df_new"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63db579a",
   "metadata": {},
   "source": [
    "### (2) reomove the rows with outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8ba6a6a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prov</th>\n",
       "      <th>gdpr</th>\n",
       "      <th>year</th>\n",
       "      <th>gdp</th>\n",
       "      <th>pop</th>\n",
       "      <th>finv</th>\n",
       "      <th>trade</th>\n",
       "      <th>fexpen</th>\n",
       "      <th>uinc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2000</td>\n",
       "      <td>1.074125</td>\n",
       "      <td>8.650000</td>\n",
       "      <td>0.314513</td>\n",
       "      <td>1.408147</td>\n",
       "      <td>0.108032</td>\n",
       "      <td>0.976157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2002</td>\n",
       "      <td>1.350242</td>\n",
       "      <td>8.842000</td>\n",
       "      <td>0.385078</td>\n",
       "      <td>1.830169</td>\n",
       "      <td>0.152108</td>\n",
       "      <td>1.113720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2003</td>\n",
       "      <td>1.584464</td>\n",
       "      <td>8.963000</td>\n",
       "      <td>0.481320</td>\n",
       "      <td>2.346735</td>\n",
       "      <td>0.169563</td>\n",
       "      <td>1.238043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2004</td>\n",
       "      <td>1.886462</td>\n",
       "      <td>9.052298</td>\n",
       "      <td>0.587002</td>\n",
       "      <td>2.955899</td>\n",
       "      <td>0.185295</td>\n",
       "      <td>1.362765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2005</td>\n",
       "      <td>2.255737</td>\n",
       "      <td>9.194000</td>\n",
       "      <td>0.697793</td>\n",
       "      <td>3.505761</td>\n",
       "      <td>0.228907</td>\n",
       "      <td>1.476994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2013</td>\n",
       "      <td>3.219130</td>\n",
       "      <td>9.413000</td>\n",
       "      <td>2.608746</td>\n",
       "      <td>0.371325</td>\n",
       "      <td>0.558231</td>\n",
       "      <td>2.174067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2014</td>\n",
       "      <td>3.493824</td>\n",
       "      <td>9.436000</td>\n",
       "      <td>3.078217</td>\n",
       "      <td>0.399111</td>\n",
       "      <td>0.602869</td>\n",
       "      <td>2.367206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2015</td>\n",
       "      <td>3.700216</td>\n",
       "      <td>9.480000</td>\n",
       "      <td>3.566035</td>\n",
       "      <td>0.459535</td>\n",
       "      <td>0.679935</td>\n",
       "      <td>2.557561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2016</td>\n",
       "      <td>4.047179</td>\n",
       "      <td>9.532000</td>\n",
       "      <td>4.041509</td>\n",
       "      <td>0.471385</td>\n",
       "      <td>0.745374</td>\n",
       "      <td>2.723292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2018</td>\n",
       "      <td>4.805586</td>\n",
       "      <td>9.360000</td>\n",
       "      <td>4.450960</td>\n",
       "      <td>0.551170</td>\n",
       "      <td>0.921773</td>\n",
       "      <td>3.187420</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>93 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         prov   gdpr  year       gdp       pop      finv     trade    fexpen  \\\n",
       "0   Guangdong  First  2000  1.074125  8.650000  0.314513  1.408147  0.108032   \n",
       "2   Guangdong  First  2002  1.350242  8.842000  0.385078  1.830169  0.152108   \n",
       "3   Guangdong  First  2003  1.584464  8.963000  0.481320  2.346735  0.169563   \n",
       "4   Guangdong  First  2004  1.886462  9.052298  0.587002  2.955899  0.185295   \n",
       "5   Guangdong  First  2005  2.255737  9.194000  0.697793  3.505761  0.228907   \n",
       "..        ...    ...   ...       ...       ...       ...       ...       ...   \n",
       "89      Henan  Fifth  2013  3.219130  9.413000  2.608746  0.371325  0.558231   \n",
       "90      Henan  Fifth  2014  3.493824  9.436000  3.078217  0.399111  0.602869   \n",
       "91      Henan  Fifth  2015  3.700216  9.480000  3.566035  0.459535  0.679935   \n",
       "92      Henan  Fifth  2016  4.047179  9.532000  4.041509  0.471385  0.745374   \n",
       "94      Henan  Fifth  2018  4.805586  9.360000  4.450960  0.551170  0.921773   \n",
       "\n",
       "        uinc  \n",
       "0   0.976157  \n",
       "2   1.113720  \n",
       "3   1.238043  \n",
       "4   1.362765  \n",
       "5   1.476994  \n",
       "..       ...  \n",
       "89  2.174067  \n",
       "90  2.367206  \n",
       "91  2.557561  \n",
       "92  2.723292  \n",
       "94  3.187420  \n",
       "\n",
       "[93 rows x 9 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new =df.drop([1,93])\n",
    "df_new"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fab1c34",
   "metadata": {},
   "source": [
    "## 2. Regard outliers as NaNs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "966b376d",
   "metadata": {},
   "source": [
    "### (1) Replace them with NaN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "635ba00f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_copy = df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f889c0bb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prov</th>\n",
       "      <th>gdpr</th>\n",
       "      <th>year</th>\n",
       "      <th>gdp</th>\n",
       "      <th>pop</th>\n",
       "      <th>finv</th>\n",
       "      <th>trade</th>\n",
       "      <th>fexpen</th>\n",
       "      <th>uinc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2000</td>\n",
       "      <td>1.074125</td>\n",
       "      <td>8.650000</td>\n",
       "      <td>0.314513</td>\n",
       "      <td>1.408147</td>\n",
       "      <td>0.108032</td>\n",
       "      <td>0.976157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2001</td>\n",
       "      <td>1.203925</td>\n",
       "      <td>8.733000</td>\n",
       "      <td>0.348443</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.132133</td>\n",
       "      <td>1.041519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2002</td>\n",
       "      <td>1.350242</td>\n",
       "      <td>8.842000</td>\n",
       "      <td>0.385078</td>\n",
       "      <td>1.830169</td>\n",
       "      <td>0.152108</td>\n",
       "      <td>1.113720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2003</td>\n",
       "      <td>1.584464</td>\n",
       "      <td>8.963000</td>\n",
       "      <td>0.481320</td>\n",
       "      <td>2.346735</td>\n",
       "      <td>0.169563</td>\n",
       "      <td>1.238043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2004</td>\n",
       "      <td>1.886462</td>\n",
       "      <td>9.052298</td>\n",
       "      <td>0.587002</td>\n",
       "      <td>2.955899</td>\n",
       "      <td>0.185295</td>\n",
       "      <td>1.362765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2014</td>\n",
       "      <td>3.493824</td>\n",
       "      <td>9.436000</td>\n",
       "      <td>3.078217</td>\n",
       "      <td>0.399111</td>\n",
       "      <td>0.602869</td>\n",
       "      <td>2.367206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2015</td>\n",
       "      <td>3.700216</td>\n",
       "      <td>9.480000</td>\n",
       "      <td>3.566035</td>\n",
       "      <td>0.459535</td>\n",
       "      <td>0.679935</td>\n",
       "      <td>2.557561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2016</td>\n",
       "      <td>4.047179</td>\n",
       "      <td>9.532000</td>\n",
       "      <td>4.041509</td>\n",
       "      <td>0.471385</td>\n",
       "      <td>0.745374</td>\n",
       "      <td>2.723292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2017</td>\n",
       "      <td>4.455283</td>\n",
       "      <td>9.392000</td>\n",
       "      <td>4.449690</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.821552</td>\n",
       "      <td>2.955790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2018</td>\n",
       "      <td>4.805586</td>\n",
       "      <td>9.360000</td>\n",
       "      <td>4.450960</td>\n",
       "      <td>0.551170</td>\n",
       "      <td>0.921773</td>\n",
       "      <td>3.187420</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>95 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         prov   gdpr  year       gdp       pop      finv     trade    fexpen  \\\n",
       "0   Guangdong  First  2000  1.074125  8.650000  0.314513  1.408147  0.108032   \n",
       "1   Guangdong  First  2001  1.203925  8.733000  0.348443       NaN  0.132133   \n",
       "2   Guangdong  First  2002  1.350242  8.842000  0.385078  1.830169  0.152108   \n",
       "3   Guangdong  First  2003  1.584464  8.963000  0.481320  2.346735  0.169563   \n",
       "4   Guangdong  First  2004  1.886462  9.052298  0.587002  2.955899  0.185295   \n",
       "..        ...    ...   ...       ...       ...       ...       ...       ...   \n",
       "90      Henan  Fifth  2014  3.493824  9.436000  3.078217  0.399111  0.602869   \n",
       "91      Henan  Fifth  2015  3.700216  9.480000  3.566035  0.459535  0.679935   \n",
       "92      Henan  Fifth  2016  4.047179  9.532000  4.041509  0.471385  0.745374   \n",
       "93      Henan  Fifth  2017  4.455283  9.392000  4.449690       NaN  0.821552   \n",
       "94      Henan  Fifth  2018  4.805586  9.360000  4.450960  0.551170  0.921773   \n",
       "\n",
       "        uinc  \n",
       "0   0.976157  \n",
       "1   1.041519  \n",
       "2   1.113720  \n",
       "3   1.238043  \n",
       "4   1.362765  \n",
       "..       ...  \n",
       "90  2.367206  \n",
       "91  2.557561  \n",
       "92  2.723292  \n",
       "93  2.955790  \n",
       "94  3.187420  \n",
       "\n",
       "[95 rows x 9 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# change the outliers with 'np.nan'\n",
    "df_copy.loc[[1,93],['trade']]=np.nan\n",
    "df_copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "66aaceeb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prov</th>\n",
       "      <th>gdpr</th>\n",
       "      <th>year</th>\n",
       "      <th>gdp</th>\n",
       "      <th>pop</th>\n",
       "      <th>finv</th>\n",
       "      <th>trade</th>\n",
       "      <th>fexpen</th>\n",
       "      <th>uinc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2000</td>\n",
       "      <td>1.074125</td>\n",
       "      <td>8.650000</td>\n",
       "      <td>0.314513</td>\n",
       "      <td>1.408147</td>\n",
       "      <td>0.108032</td>\n",
       "      <td>0.976157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2001</td>\n",
       "      <td>1.203925</td>\n",
       "      <td>8.733000</td>\n",
       "      <td>0.348443</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.132133</td>\n",
       "      <td>1.041519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2002</td>\n",
       "      <td>1.350242</td>\n",
       "      <td>8.842000</td>\n",
       "      <td>0.385078</td>\n",
       "      <td>1.830169</td>\n",
       "      <td>0.152108</td>\n",
       "      <td>1.113720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2003</td>\n",
       "      <td>1.584464</td>\n",
       "      <td>8.963000</td>\n",
       "      <td>0.481320</td>\n",
       "      <td>2.346735</td>\n",
       "      <td>0.169563</td>\n",
       "      <td>1.238043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2004</td>\n",
       "      <td>1.886462</td>\n",
       "      <td>9.052298</td>\n",
       "      <td>0.587002</td>\n",
       "      <td>2.955899</td>\n",
       "      <td>0.185295</td>\n",
       "      <td>1.362765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2014</td>\n",
       "      <td>3.493824</td>\n",
       "      <td>9.436000</td>\n",
       "      <td>3.078217</td>\n",
       "      <td>0.399111</td>\n",
       "      <td>0.602869</td>\n",
       "      <td>2.367206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2015</td>\n",
       "      <td>3.700216</td>\n",
       "      <td>9.480000</td>\n",
       "      <td>3.566035</td>\n",
       "      <td>0.459535</td>\n",
       "      <td>0.679935</td>\n",
       "      <td>2.557561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2016</td>\n",
       "      <td>4.047179</td>\n",
       "      <td>9.532000</td>\n",
       "      <td>4.041509</td>\n",
       "      <td>0.471385</td>\n",
       "      <td>0.745374</td>\n",
       "      <td>2.723292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2017</td>\n",
       "      <td>4.455283</td>\n",
       "      <td>9.392000</td>\n",
       "      <td>4.449690</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.821552</td>\n",
       "      <td>2.955790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2018</td>\n",
       "      <td>4.805586</td>\n",
       "      <td>9.360000</td>\n",
       "      <td>4.450960</td>\n",
       "      <td>0.551170</td>\n",
       "      <td>0.921773</td>\n",
       "      <td>3.187420</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>95 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         prov   gdpr  year       gdp       pop      finv     trade    fexpen  \\\n",
       "0   Guangdong  First  2000  1.074125  8.650000  0.314513  1.408147  0.108032   \n",
       "1   Guangdong  First  2001  1.203925  8.733000  0.348443       NaN  0.132133   \n",
       "2   Guangdong  First  2002  1.350242  8.842000  0.385078  1.830169  0.152108   \n",
       "3   Guangdong  First  2003  1.584464  8.963000  0.481320  2.346735  0.169563   \n",
       "4   Guangdong  First  2004  1.886462  9.052298  0.587002  2.955899  0.185295   \n",
       "..        ...    ...   ...       ...       ...       ...       ...       ...   \n",
       "90      Henan  Fifth  2014  3.493824  9.436000  3.078217  0.399111  0.602869   \n",
       "91      Henan  Fifth  2015  3.700216  9.480000  3.566035  0.459535  0.679935   \n",
       "92      Henan  Fifth  2016  4.047179  9.532000  4.041509  0.471385  0.745374   \n",
       "93      Henan  Fifth  2017  4.455283  9.392000  4.449690       NaN  0.821552   \n",
       "94      Henan  Fifth  2018  4.805586  9.360000  4.450960  0.551170  0.921773   \n",
       "\n",
       "        uinc  \n",
       "0   0.976157  \n",
       "1   1.041519  \n",
       "2   1.113720  \n",
       "3   1.238043  \n",
       "4   1.362765  \n",
       "..       ...  \n",
       "90  2.367206  \n",
       "91  2.557561  \n",
       "92  2.723292  \n",
       "93  2.955790  \n",
       "94  3.187420  \n",
       "\n",
       "[95 rows x 9 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# replace the outlier values with 'np.nan'\n",
    "df_copy.replace({\n",
    "    'trade':{14.609701:np.nan,0.000523:np.nan}\n",
    "},inplace=True)\n",
    "df_copy"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef6851d3",
   "metadata": {},
   "source": [
    "## (2) Apply methods for filling in missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ea392c00",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prov</th>\n",
       "      <th>gdpr</th>\n",
       "      <th>year</th>\n",
       "      <th>gdp</th>\n",
       "      <th>pop</th>\n",
       "      <th>finv</th>\n",
       "      <th>trade</th>\n",
       "      <th>fexpen</th>\n",
       "      <th>uinc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2000</td>\n",
       "      <td>1.074125</td>\n",
       "      <td>8.650000</td>\n",
       "      <td>0.314513</td>\n",
       "      <td>1.408147</td>\n",
       "      <td>0.108032</td>\n",
       "      <td>0.976157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2001</td>\n",
       "      <td>1.203925</td>\n",
       "      <td>8.733000</td>\n",
       "      <td>0.348443</td>\n",
       "      <td>1.501391</td>\n",
       "      <td>0.132133</td>\n",
       "      <td>1.041519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2002</td>\n",
       "      <td>1.350242</td>\n",
       "      <td>8.842000</td>\n",
       "      <td>0.385078</td>\n",
       "      <td>1.830169</td>\n",
       "      <td>0.152108</td>\n",
       "      <td>1.113720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2003</td>\n",
       "      <td>1.584464</td>\n",
       "      <td>8.963000</td>\n",
       "      <td>0.481320</td>\n",
       "      <td>2.346735</td>\n",
       "      <td>0.169563</td>\n",
       "      <td>1.238043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Guangdong</td>\n",
       "      <td>First</td>\n",
       "      <td>2004</td>\n",
       "      <td>1.886462</td>\n",
       "      <td>9.052298</td>\n",
       "      <td>0.587002</td>\n",
       "      <td>2.955899</td>\n",
       "      <td>0.185295</td>\n",
       "      <td>1.362765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2014</td>\n",
       "      <td>3.493824</td>\n",
       "      <td>9.436000</td>\n",
       "      <td>3.078217</td>\n",
       "      <td>0.399111</td>\n",
       "      <td>0.602869</td>\n",
       "      <td>2.367206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2015</td>\n",
       "      <td>3.700216</td>\n",
       "      <td>9.480000</td>\n",
       "      <td>3.566035</td>\n",
       "      <td>0.459535</td>\n",
       "      <td>0.679935</td>\n",
       "      <td>2.557561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2016</td>\n",
       "      <td>4.047179</td>\n",
       "      <td>9.532000</td>\n",
       "      <td>4.041509</td>\n",
       "      <td>0.471385</td>\n",
       "      <td>0.745374</td>\n",
       "      <td>2.723292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2017</td>\n",
       "      <td>4.455283</td>\n",
       "      <td>9.392000</td>\n",
       "      <td>4.449690</td>\n",
       "      <td>0.474870</td>\n",
       "      <td>0.821552</td>\n",
       "      <td>2.955790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>Henan</td>\n",
       "      <td>Fifth</td>\n",
       "      <td>2018</td>\n",
       "      <td>4.805586</td>\n",
       "      <td>9.360000</td>\n",
       "      <td>4.450960</td>\n",
       "      <td>0.551170</td>\n",
       "      <td>0.921773</td>\n",
       "      <td>3.187420</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>95 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         prov   gdpr  year       gdp       pop      finv     trade    fexpen  \\\n",
       "0   Guangdong  First  2000  1.074125  8.650000  0.314513  1.408147  0.108032   \n",
       "1   Guangdong  First  2001  1.203925  8.733000  0.348443  1.501391  0.132133   \n",
       "2   Guangdong  First  2002  1.350242  8.842000  0.385078  1.830169  0.152108   \n",
       "3   Guangdong  First  2003  1.584464  8.963000  0.481320  2.346735  0.169563   \n",
       "4   Guangdong  First  2004  1.886462  9.052298  0.587002  2.955899  0.185295   \n",
       "..        ...    ...   ...       ...       ...       ...       ...       ...   \n",
       "90      Henan  Fifth  2014  3.493824  9.436000  3.078217  0.399111  0.602869   \n",
       "91      Henan  Fifth  2015  3.700216  9.480000  3.566035  0.459535  0.679935   \n",
       "92      Henan  Fifth  2016  4.047179  9.532000  4.041509  0.471385  0.745374   \n",
       "93      Henan  Fifth  2017  4.455283  9.392000  4.449690  0.474870  0.821552   \n",
       "94      Henan  Fifth  2018  4.805586  9.360000  4.450960  0.551170  0.921773   \n",
       "\n",
       "        uinc  \n",
       "0   0.976157  \n",
       "1   1.041519  \n",
       "2   1.113720  \n",
       "3   1.238043  \n",
       "4   1.362765  \n",
       "..       ...  \n",
       "90  2.367206  \n",
       "91  2.557561  \n",
       "92  2.723292  \n",
       "93  2.955790  \n",
       "94  3.187420  \n",
       "\n",
       "[95 rows x 9 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new = df_copy.interpolate(method='cubicspline',order=2)\n",
    "df_new"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec036600",
   "metadata": {},
   "source": [
    "## 3. Save the treated data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0cf64c88",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_new.to_csv('./data/gdp_china_outlier_cl.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}