initial_cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Basic Library Tools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Installed autotime.py. To use it, type:\n",
      "  %load_ext autotime\n"
     ]
    }
   ],
   "source": [
    "%matplotlib notebook\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import warnings\n",
    "#Notebook Specifics\n",
    "\n",
    "pd.set_option(\"max_rows\", 20)\n",
    "np.set_printoptions(suppress = True)\n",
    "#Pretty Graphs\n",
    "\n",
    "from seaborn import set_style\n",
    "set_style(\"darkgrid\")\n",
    "import seaborn as sns\n",
    "\n",
    "\n",
    "import time\n",
    "from datetime import datetime\n",
    "import calendar\n",
    "\n",
    "# This package will output the execution time of each cell. Pretty neat!\n",
    "warnings.filterwarnings('ignore')\n",
    "%install_ext https://raw.github.com/cpcloud/ipython-autotime/master/autotime.py\n",
    "warnings.filterwarnings('default')\n",
    "%load_ext autotime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading Data into a Test and a Training set\n",
    "#### Testing Data = Summer 2016 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "testing_csv_files = [\"201606-citibike-tripdata.csv\", \"201608-citibike-tripdata.csv\", \"201607-citibike-tripdata.csv\"]\n",
    "testing = pd.DataFrame()\n",
    "for filename in testing_csv_files:\n",
    "    testing = testing.append(pd.read_csv(filename))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Training Data = Summer 2014 and Summer 2015"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "training_csv_files = [\"2014-06 - Citi Bike trip data.csv\",  \"201506-citibike-tripdata.csv\", \"2014-07 - Citi Bike trip data.csv\", \"201507-citibike-tripdata.csv\", \"2014-08 - Citi Bike trip data.csv\", \"201508-citibike-tripdata.csv\"]\n",
    "training = pd.DataFrame()\n",
    "for filename in training_csv_files:\n",
    "    training = training.append(pd.read_csv(filename))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sanity checks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([], dtype='object')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training.columns.difference(testing.columns) #To check that the heading of our test Data and Training Data are the same\n",
    "                                             # Since object return is empty, this implies that they have the same headings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training.columns.equals(testing.columns)      #Object returned is true, so both have the same headings. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Some Data Mangling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 1 Take only the Subscribers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### First lets get the ratio of people who are subscribers  vs people who are customers "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>counts</th>\n",
       "      <th>freqs</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>categories</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Customer</th>\n",
       "      <td>904882</td>\n",
       "      <td>0.148948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Subscriber</th>\n",
       "      <td>5170268</td>\n",
       "      <td>0.851052</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             counts     freqs\n",
       "categories                   \n",
       "Customer     904882  0.148948\n",
       "Subscriber  5170268  0.851052"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "usertype_ratio = pd.Categorical(training[\"usertype\"])\n",
    "usertype_ratio.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Thus the number of Subscribers is 85% of the original data we had. \n",
    "##### This leaves us with 5.17 million trips"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# To clean out the customers detail\n",
    "training[training.usertype == \"Subscriber\"]\n",
    "del training['usertype']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Now to Drop NAN entries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "training = training.dropna(how='any')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### This drops another 370k entries.  Reducing our total data size to "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 2. Calculate age of User "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "training['birth year'] = pd.to_numeric(training['birth year'], errors='coerce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "training = training.dropna(how='any')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "training['Age'] = 2015 - training['birth year']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "del training['birth year'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del training['bikeid']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tripduration</th>\n",
       "      <th>starttime</th>\n",
       "      <th>stoptime</th>\n",
       "      <th>start station id</th>\n",
       "      <th>start station name</th>\n",
       "      <th>start station latitude</th>\n",
       "      <th>start station longitude</th>\n",
       "      <th>end station id</th>\n",
       "      <th>end station name</th>\n",
       "      <th>end station latitude</th>\n",
       "      <th>end station longitude</th>\n",
       "      <th>gender</th>\n",
       "      <th>Age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>520</td>\n",
       "      <td>2014-06-01 00:00:02</td>\n",
       "      <td>2014-06-01 00:08:42</td>\n",
       "      <td>358</td>\n",
       "      <td>Christopher St &amp; Greenwich St</td>\n",
       "      <td>40.732916</td>\n",
       "      <td>-74.007114</td>\n",
       "      <td>426</td>\n",
       "      <td>West St &amp; Chambers St</td>\n",
       "      <td>40.717548</td>\n",
       "      <td>-74.013221</td>\n",
       "      <td>1</td>\n",
       "      <td>36.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>414</td>\n",
       "      <td>2014-06-01 00:00:32</td>\n",
       "      <td>2014-06-01 00:07:26</td>\n",
       "      <td>439</td>\n",
       "      <td>E 4 St &amp; 2 Ave</td>\n",
       "      <td>40.726281</td>\n",
       "      <td>-73.989780</td>\n",
       "      <td>368</td>\n",
       "      <td>Carmine St &amp; 6 Ave</td>\n",
       "      <td>40.730386</td>\n",
       "      <td>-74.002150</td>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>310</td>\n",
       "      <td>2014-06-01 00:00:34</td>\n",
       "      <td>2014-06-01 00:05:44</td>\n",
       "      <td>463</td>\n",
       "      <td>9 Ave &amp; W 16 St</td>\n",
       "      <td>40.742065</td>\n",
       "      <td>-74.004432</td>\n",
       "      <td>380</td>\n",
       "      <td>W 4 St &amp; 7 Ave S</td>\n",
       "      <td>40.734011</td>\n",
       "      <td>-74.002939</td>\n",
       "      <td>1</td>\n",
       "      <td>31.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>457</td>\n",
       "      <td>2014-06-01 00:00:35</td>\n",
       "      <td>2014-06-01 00:08:12</td>\n",
       "      <td>352</td>\n",
       "      <td>W 56 St &amp; 6 Ave</td>\n",
       "      <td>40.763406</td>\n",
       "      <td>-73.977225</td>\n",
       "      <td>305</td>\n",
       "      <td>E 58 St &amp; 3 Ave</td>\n",
       "      <td>40.760958</td>\n",
       "      <td>-73.967245</td>\n",
       "      <td>1</td>\n",
       "      <td>46.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>399</td>\n",
       "      <td>2014-06-01 00:00:43</td>\n",
       "      <td>2014-06-01 00:07:22</td>\n",
       "      <td>293</td>\n",
       "      <td>Lafayette St &amp; E 8 St</td>\n",
       "      <td>40.730287</td>\n",
       "      <td>-73.990765</td>\n",
       "      <td>247</td>\n",
       "      <td>Perry St &amp; Bleecker St</td>\n",
       "      <td>40.735354</td>\n",
       "      <td>-74.004831</td>\n",
       "      <td>1</td>\n",
       "      <td>45.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tripduration            starttime             stoptime  start station id  \\\n",
       "0           520  2014-06-01 00:00:02  2014-06-01 00:08:42               358   \n",
       "2           414  2014-06-01 00:00:32  2014-06-01 00:07:26               439   \n",
       "3           310  2014-06-01 00:00:34  2014-06-01 00:05:44               463   \n",
       "4           457  2014-06-01 00:00:35  2014-06-01 00:08:12               352   \n",
       "5           399  2014-06-01 00:00:43  2014-06-01 00:07:22               293   \n",
       "\n",
       "              start station name  start station latitude  \\\n",
       "0  Christopher St & Greenwich St               40.732916   \n",
       "2                 E 4 St & 2 Ave               40.726281   \n",
       "3                9 Ave & W 16 St               40.742065   \n",
       "4                W 56 St & 6 Ave               40.763406   \n",
       "5          Lafayette St & E 8 St               40.730287   \n",
       "\n",
       "   start station longitude  end station id        end station name  \\\n",
       "0               -74.007114             426   West St & Chambers St   \n",
       "2               -73.989780             368      Carmine St & 6 Ave   \n",
       "3               -74.004432             380        W 4 St & 7 Ave S   \n",
       "4               -73.977225             305         E 58 St & 3 Ave   \n",
       "5               -73.990765             247  Perry St & Bleecker St   \n",
       "\n",
       "   end station latitude  end station longitude  gender   Age  \n",
       "0             40.717548             -74.013221       1  36.0  \n",
       "2             40.730386             -74.002150       1  35.0  \n",
       "3             40.734011             -74.002939       1  31.0  \n",
       "4             40.760958             -73.967245       1  46.0  \n",
       "5             40.735354             -74.004831       1  45.0  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Step 3. Add day of week "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "training.to_csv('partiall_cleaned_training_data.csv', sep=',', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 34.1 s\n"
     ]
    }
   ],
   "source": [
    "training = pd.read_csv('partiall_cleaned_training_data.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "###### Since our data follows multiple date formats, we need to clean it to get data out of it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 8.31 ms\n"
     ]
    }
   ],
   "source": [
    "weekend = []\n",
    "\n",
    "def try_parsing_date(text):\n",
    "    for fmt in ('%Y-%m-%d %H:%M:%S', '%m/%d/%Y %H:%M:%S', '%m/%d/%Y %H:%M'):\n",
    "        try:\n",
    "            return datetime.strptime(text, fmt)\n",
    "        except ValueError:\n",
    "            pass\n",
    "    raise ValueError('no valid date format found')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 12min 54s\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(training)):\n",
    "    weekend.append(calendar.day_name[try_parsing_date(training[\"starttime\"].values[i]).weekday()])\n",
    "training[\"day of week\"] = weekend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 5170103 entries, 0 to 5170102\n",
      "Data columns (total 15 columns):\n",
      "Unnamed: 0                 int64\n",
      "tripduration               int64\n",
      "starttime                  object\n",
      "stoptime                   object\n",
      "start station id           int64\n",
      "start station name         object\n",
      "start station latitude     float64\n",
      "start station longitude    float64\n",
      "end station id             int64\n",
      "end station name           object\n",
      "end station latitude       float64\n",
      "end station longitude      float64\n",
      "gender                     int64\n",
      "Age                        float64\n",
      "day of week                object\n",
      "dtypes: float64(5), int64(5), object(5)\n",
      "memory usage: 591.7+ MB\n",
      "time: 1.39 s\n"
     ]
    }
   ],
   "source": [
    "training.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/haris/anaconda3/lib/python3.5/site-packages/IPython/core/formatters.py:669: DeprecationWarning: PlainTextFormatter._singleton_printers_default is deprecated: use @default decorator instead.\n",
      "  def _singleton_printers_default(self):\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>tripduration</th>\n",
       "      <th>starttime</th>\n",
       "      <th>stoptime</th>\n",
       "      <th>start station id</th>\n",
       "      <th>start station name</th>\n",
       "      <th>start station latitude</th>\n",
       "      <th>start station longitude</th>\n",
       "      <th>end station id</th>\n",
       "      <th>end station name</th>\n",
       "      <th>end station latitude</th>\n",
       "      <th>end station longitude</th>\n",
       "      <th>gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>day of week</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>520</td>\n",
       "      <td>2014-06-01 00:00:02</td>\n",
       "      <td>2014-06-01 00:08:42</td>\n",
       "      <td>358</td>\n",
       "      <td>Christopher St &amp; Greenwich St</td>\n",
       "      <td>40.732916</td>\n",
       "      <td>-74.007114</td>\n",
       "      <td>426</td>\n",
       "      <td>West St &amp; Chambers St</td>\n",
       "      <td>40.717548</td>\n",
       "      <td>-74.013221</td>\n",
       "      <td>1</td>\n",
       "      <td>36.0</td>\n",
       "      <td>Sunday</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>414</td>\n",
       "      <td>2014-06-01 00:00:32</td>\n",
       "      <td>2014-06-01 00:07:26</td>\n",
       "      <td>439</td>\n",
       "      <td>E 4 St &amp; 2 Ave</td>\n",
       "      <td>40.726281</td>\n",
       "      <td>-73.989780</td>\n",
       "      <td>368</td>\n",
       "      <td>Carmine St &amp; 6 Ave</td>\n",
       "      <td>40.730386</td>\n",
       "      <td>-74.002150</td>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>Sunday</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>310</td>\n",
       "      <td>2014-06-01 00:00:34</td>\n",
       "      <td>2014-06-01 00:05:44</td>\n",
       "      <td>463</td>\n",
       "      <td>9 Ave &amp; W 16 St</td>\n",
       "      <td>40.742065</td>\n",
       "      <td>-74.004432</td>\n",
       "      <td>380</td>\n",
       "      <td>W 4 St &amp; 7 Ave S</td>\n",
       "      <td>40.734011</td>\n",
       "      <td>-74.002939</td>\n",
       "      <td>1</td>\n",
       "      <td>31.0</td>\n",
       "      <td>Sunday</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>457</td>\n",
       "      <td>2014-06-01 00:00:35</td>\n",
       "      <td>2014-06-01 00:08:12</td>\n",
       "      <td>352</td>\n",
       "      <td>W 56 St &amp; 6 Ave</td>\n",
       "      <td>40.763406</td>\n",
       "      <td>-73.977225</td>\n",
       "      <td>305</td>\n",
       "      <td>E 58 St &amp; 3 Ave</td>\n",
       "      <td>40.760958</td>\n",
       "      <td>-73.967245</td>\n",
       "      <td>1</td>\n",
       "      <td>46.0</td>\n",
       "      <td>Sunday</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>399</td>\n",
       "      <td>2014-06-01 00:00:43</td>\n",
       "      <td>2014-06-01 00:07:22</td>\n",
       "      <td>293</td>\n",
       "      <td>Lafayette St &amp; E 8 St</td>\n",
       "      <td>40.730287</td>\n",
       "      <td>-73.990765</td>\n",
       "      <td>247</td>\n",
       "      <td>Perry St &amp; Bleecker St</td>\n",
       "      <td>40.735354</td>\n",
       "      <td>-74.004831</td>\n",
       "      <td>1</td>\n",
       "      <td>45.0</td>\n",
       "      <td>Sunday</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  tripduration            starttime             stoptime  \\\n",
       "0           0           520  2014-06-01 00:00:02  2014-06-01 00:08:42   \n",
       "1           2           414  2014-06-01 00:00:32  2014-06-01 00:07:26   \n",
       "2           3           310  2014-06-01 00:00:34  2014-06-01 00:05:44   \n",
       "3           4           457  2014-06-01 00:00:35  2014-06-01 00:08:12   \n",
       "4           5           399  2014-06-01 00:00:43  2014-06-01 00:07:22   \n",
       "\n",
       "   start station id             start station name  start station latitude  \\\n",
       "0               358  Christopher St & Greenwich St               40.732916   \n",
       "1               439                 E 4 St & 2 Ave               40.726281   \n",
       "2               463                9 Ave & W 16 St               40.742065   \n",
       "3               352                W 56 St & 6 Ave               40.763406   \n",
       "4               293          Lafayette St & E 8 St               40.730287   \n",
       "\n",
       "   start station longitude  end station id        end station name  \\\n",
       "0               -74.007114             426   West St & Chambers St   \n",
       "1               -73.989780             368      Carmine St & 6 Ave   \n",
       "2               -74.004432             380        W 4 St & 7 Ave S   \n",
       "3               -73.977225             305         E 58 St & 3 Ave   \n",
       "4               -73.990765             247  Perry St & Bleecker St   \n",
       "\n",
       "   end station latitude  end station longitude  gender   Age day of week  \n",
       "0             40.717548             -74.013221       1  36.0      Sunday  \n",
       "1             40.730386             -74.002150       1  35.0      Sunday  \n",
       "2             40.734011             -74.002939       1  31.0      Sunday  \n",
       "3             40.760958             -73.967245       1  46.0      Sunday  \n",
       "4             40.735354             -74.004831       1  45.0      Sunday  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 74.4 ms\n"
     ]
    }
   ],
   "source": [
    "training.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### Now to change our day of the week into an integer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 9.71 s\n"
     ]
    }
   ],
   "source": [
    "training['day of week'].replace({\"Sunday\": 0, \"Monday\":1, \"Tuesday\": 2, \"Wednesday\":3, \"Thursday\": 4, \"Friday\":5, \"Saturday\": 6 }, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>tripduration</th>\n",
       "      <th>starttime</th>\n",
       "      <th>stoptime</th>\n",
       "      <th>start station id</th>\n",
       "      <th>start station name</th>\n",
       "      <th>start station latitude</th>\n",
       "      <th>start station longitude</th>\n",
       "      <th>end station id</th>\n",
       "      <th>end station name</th>\n",
       "      <th>end station latitude</th>\n",
       "      <th>end station longitude</th>\n",
       "      <th>gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>day of week</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>520</td>\n",
       "      <td>2014-06-01 00:00:02</td>\n",
       "      <td>2014-06-01 00:08:42</td>\n",
       "      <td>358</td>\n",
       "      <td>Christopher St &amp; Greenwich St</td>\n",
       "      <td>40.732916</td>\n",
       "      <td>-74.007114</td>\n",
       "      <td>426</td>\n",
       "      <td>West St &amp; Chambers St</td>\n",
       "      <td>40.717548</td>\n",
       "      <td>-74.013221</td>\n",
       "      <td>1</td>\n",
       "      <td>36.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>414</td>\n",
       "      <td>2014-06-01 00:00:32</td>\n",
       "      <td>2014-06-01 00:07:26</td>\n",
       "      <td>439</td>\n",
       "      <td>E 4 St &amp; 2 Ave</td>\n",
       "      <td>40.726281</td>\n",
       "      <td>-73.989780</td>\n",
       "      <td>368</td>\n",
       "      <td>Carmine St &amp; 6 Ave</td>\n",
       "      <td>40.730386</td>\n",
       "      <td>-74.002150</td>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>310</td>\n",
       "      <td>2014-06-01 00:00:34</td>\n",
       "      <td>2014-06-01 00:05:44</td>\n",
       "      <td>463</td>\n",
       "      <td>9 Ave &amp; W 16 St</td>\n",
       "      <td>40.742065</td>\n",
       "      <td>-74.004432</td>\n",
       "      <td>380</td>\n",
       "      <td>W 4 St &amp; 7 Ave S</td>\n",
       "      <td>40.734011</td>\n",
       "      <td>-74.002939</td>\n",
       "      <td>1</td>\n",
       "      <td>31.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>457</td>\n",
       "      <td>2014-06-01 00:00:35</td>\n",
       "      <td>2014-06-01 00:08:12</td>\n",
       "      <td>352</td>\n",
       "      <td>W 56 St &amp; 6 Ave</td>\n",
       "      <td>40.763406</td>\n",
       "      <td>-73.977225</td>\n",
       "      <td>305</td>\n",
       "      <td>E 58 St &amp; 3 Ave</td>\n",
       "      <td>40.760958</td>\n",
       "      <td>-73.967245</td>\n",
       "      <td>1</td>\n",
       "      <td>46.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>399</td>\n",
       "      <td>2014-06-01 00:00:43</td>\n",
       "      <td>2014-06-01 00:07:22</td>\n",
       "      <td>293</td>\n",
       "      <td>Lafayette St &amp; E 8 St</td>\n",
       "      <td>40.730287</td>\n",
       "      <td>-73.990765</td>\n",
       "      <td>247</td>\n",
       "      <td>Perry St &amp; Bleecker St</td>\n",
       "      <td>40.735354</td>\n",
       "      <td>-74.004831</td>\n",
       "      <td>1</td>\n",
       "      <td>45.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  tripduration            starttime             stoptime  \\\n",
       "0           0           520  2014-06-01 00:00:02  2014-06-01 00:08:42   \n",
       "1           2           414  2014-06-01 00:00:32  2014-06-01 00:07:26   \n",
       "2           3           310  2014-06-01 00:00:34  2014-06-01 00:05:44   \n",
       "3           4           457  2014-06-01 00:00:35  2014-06-01 00:08:12   \n",
       "4           5           399  2014-06-01 00:00:43  2014-06-01 00:07:22   \n",
       "\n",
       "   start station id             start station name  start station latitude  \\\n",
       "0               358  Christopher St & Greenwich St               40.732916   \n",
       "1               439                 E 4 St & 2 Ave               40.726281   \n",
       "2               463                9 Ave & W 16 St               40.742065   \n",
       "3               352                W 56 St & 6 Ave               40.763406   \n",
       "4               293          Lafayette St & E 8 St               40.730287   \n",
       "\n",
       "   start station longitude  end station id        end station name  \\\n",
       "0               -74.007114             426   West St & Chambers St   \n",
       "1               -73.989780             368      Carmine St & 6 Ave   \n",
       "2               -74.004432             380        W 4 St & 7 Ave S   \n",
       "3               -73.977225             305         E 58 St & 3 Ave   \n",
       "4               -73.990765             247  Perry St & Bleecker St   \n",
       "\n",
       "   end station latitude  end station longitude  gender   Age  day of week  \n",
       "0             40.717548             -74.013221       1  36.0            0  \n",
       "1             40.730386             -74.002150       1  35.0            0  \n",
       "2             40.734011             -74.002939       1  31.0            0  \n",
       "3             40.760958             -73.967245       1  46.0            0  \n",
       "4             40.735354             -74.004831       1  45.0            0  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 61.7 ms\n"
     ]
    }
   ],
   "source": [
    "training.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 5170103 entries, 0 to 5170102\n",
      "Data columns (total 15 columns):\n",
      "Unnamed: 0                 int64\n",
      "tripduration               int64\n",
      "starttime                  object\n",
      "stoptime                   object\n",
      "start station id           int64\n",
      "start station name         object\n",
      "start station latitude     float64\n",
      "start station longitude    float64\n",
      "end station id             int64\n",
      "end station name           object\n",
      "end station latitude       float64\n",
      "end station longitude      float64\n",
      "gender                     int64\n",
      "Age                        float64\n",
      "day of week                int64\n",
      "dtypes: float64(5), int64(6), object(4)\n",
      "memory usage: 591.7+ MB\n",
      "time: 363 ms\n"
     ]
    }
   ],
   "source": [
    "training.info()"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### Now That we were able to add the weekday to the data. The next component is to remove the dates that we have and keep just the start time and stop time. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 5min 48s\n"
     ]
    }
   ],
   "source": [
    "training.to_csv('semi_cleaned_training_data.csv', sep=',', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}