clean_mini_dataset_neat.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%store -r PATH_TO_DATA_FOLDER\n",
    "\n",
    "data = pd.read_csv(PATH_TO_DATA_FOLDER+\"/data_chunk_1.csv\")\n",
    "#generalise this file name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count    400.000000\n",
      "mean      13.121761\n",
      "std        5.695504\n",
      "min        0.000000\n",
      "25%       12.919502\n",
      "50%       12.966616\n",
      "75%       13.018760\n",
      "max      100.000000\n",
      "Name: lat, dtype: float64\n",
      "count     400.000000\n",
      "mean       80.554366\n",
      "std        62.058616\n",
      "min         0.000000\n",
      "25%        77.536640\n",
      "50%        77.585838\n",
      "75%        77.634634\n",
      "max      1000.000000\n",
      "Name: long, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print((data.lat.describe()))\n",
    "print((data.long.describe()))\n",
    "#clearly there are zero values and values very far from the mean\n",
    "#these must be device errors since the bus can't deviate by 80 units\n",
    "#these values are hence dropped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             bus_id         lat         long\n",
      "count  3.930000e+02  393.000000   393.000000\n",
      "mean   1.504179e+08   13.355482    81.989176\n",
      "std    2.794590e+05    5.467048    61.661215\n",
      "min    1.502181e+08   12.680088    77.394203\n",
      "25%    1.502201e+08   12.920876    77.537109\n",
      "50%    1.502218e+08   12.976646    77.586082\n",
      "75%    1.508114e+08   13.018923    77.634689\n",
      "max    1.508148e+08  100.000000  1000.000000\n"
     ]
    }
   ],
   "source": [
    "#remove rows which have 0 lat values(by picking a subset of rows that don't have a zero lat value)\n",
    "lat_nullfree = data[data.lat != 0.0]\n",
    "\n",
    "#now remove the rows which have 0 long values\n",
    "lat_long_nullfree = data[data.long != 0.0]\n",
    "\n",
    "#min value is no longer zero\n",
    "print(lat_long_nullfree.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#collect mean and define a safe error margin\n",
    "lat_mean = lat_long_nullfree.mean()[\"lat\"]\n",
    "long_mean = lat_long_nullfree.mean()[\"long\"]\n",
    "\n",
    "lat_error = 10\n",
    "long_error = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             bus_id         lat        long\n",
      "count  3.910000e+02  391.000000  391.000000\n",
      "mean   1.504174e+08   12.969171   77.595080\n",
      "std    2.792901e+05    0.086997    0.082799\n",
      "min    1.502181e+08   12.680088   77.394203\n",
      "25%    1.502201e+08   12.920704   77.537064\n",
      "50%    1.502218e+08   12.974773   77.585938\n",
      "75%    1.508114e+08   13.018783   77.634640\n",
      "max    1.508148e+08   13.300202   77.862457\n"
     ]
    }
   ],
   "source": [
    "#remove rows where the mean value and actual value differ by more than the error\n",
    "\n",
    "#first for lat\n",
    "lat_proper = lat_long_nullfree[abs(lat_long_nullfree.lat - lat_mean) < lat_error]\n",
    "\n",
    "#now for long on top of this\n",
    "lat_long_proper = lat_proper[abs(lat_proper.long - long_mean) < long_error]\n",
    "\n",
    "print(lat_long_proper.describe())\n",
    "\n",
    "#the max value is now well within the error, so it was safe\n",
    "#using the fact that the lat, long deviation of buses can't be very high\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#now write that data into a csv file\n",
    "\n",
    "lat_long_proper.to_csv(PATH_TO_DATA_FOLDER + \"/cleaned_data_chunk_1.csv\",columns = ['bus_id', 'lat', 'long', 'time'],index=None)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}