Week3_Toronto_Neighborhoods.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Segmenting and Clustering Neighborhoods in Toronto"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set up dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Libraries imported.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # library to handle data in a vectorized manner\n",
    "\n",
    "import pandas as pd # library for data analysis\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', None)\n",
    "\n",
    "import json # library to handle JSON files\n",
    "\n",
    "#!conda install -c conda-forge geopy --yes \n",
    "from geopy.geocoders import Nominatim # convert an address into latitude and longitude values\n",
    "\n",
    "import requests # library to handle requests\n",
    "from pandas.io.json import json_normalize # tranform JSON file into a pandas dataframe\n",
    "\n",
    "# Matplotlib and associated plotting modules\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib.colors as colors\n",
    "\n",
    "# import k-means from clustering stage\n",
    "from sklearn.cluster import KMeans\n",
    "\n",
    "#!conda install -c conda-forge folium=0.5.0 --yes\n",
    "import folium # map rendering library\n",
    "\n",
    "import requests # library to fetch URLs\n",
    "\n",
    "from bs4 import BeautifulSoup # library to scrape data from a webpage\n",
    "\n",
    "print('Libraries imported.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scrape Postal Codes from Wikipedia\n",
    "Scrape the postal codes table from this Wikipedia page:\n",
    "https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M\n",
    "\n",
    "<i>\"This is a list of postal codes in Canada where the first letter is M. Postal codes beginning with M are located within the city of Toronto in the province of Ontario. Only the first three characters are listed, corresponding to the Forward Sortation Area.\"</i>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data downloaded from: https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M\n"
     ]
    }
   ],
   "source": [
    "url = 'https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M'\n",
    "\n",
    "response = requests.get(url)\n",
    "if response.status_code != 200:\n",
    "    print('Failed to get data:', response.status_code)\n",
    "else:\n",
    "    print(f'Data downloaded from: {url}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Postcode</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighbourhood</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>M1A</td>\n",
       "      <td>Not assigned</td>\n",
       "      <td>Not assigned</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>M2A</td>\n",
       "      <td>Not assigned</td>\n",
       "      <td>Not assigned</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>M3A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Parkwoods</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>M4A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Victoria Village</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>M5A</td>\n",
       "      <td>Downtown Toronto</td>\n",
       "      <td>Harbourfront</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Postcode           Borough     Neighbourhood\n",
       "0      M1A      Not assigned      Not assigned\n",
       "1      M2A      Not assigned      Not assigned\n",
       "2      M3A        North York         Parkwoods\n",
       "3      M4A        North York  Victoria Village\n",
       "4      M5A  Downtown Toronto      Harbourfront"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup = BeautifulSoup(response.text, 'html.parser')\n",
    "table = soup.find(\"table\", {\"class\":\"wikitable sortable\"})\n",
    "df = pd.read_html(str(table),header=0)[0]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean up the postal code dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Removed 0 rows with unassigned boroughs.\n",
      "As expected, there is only one borough in each postcode. :)\n",
      "0 rows with unassigned neighbourhoods (now assigned to borough).\n",
      "Collapsed 0 duplicated postcodes into single rows.\n",
      "\n",
      "\n",
      "  Postcode      Borough                           Neighbourhood\n",
      "0      M1B  Scarborough                          Rouge, Malvern\n",
      "1      M1C  Scarborough  Highland Creek, Rouge Hill, Port Union\n",
      "2      M1E  Scarborough       Guildwood, Morningside, West Hill\n",
      "3      M1G  Scarborough                                  Woburn\n",
      "4      M1H  Scarborough                               Cedarbrae\n"
     ]
    }
   ],
   "source": [
    "# Remove unassigned boroughs\n",
    "n = df.shape[0]\n",
    "df = df.loc[df['Borough']!='Not assigned',:]\n",
    "print(f'Removed {df.shape[0]-n:,} rows with unassigned boroughs.')\n",
    "del n\n",
    "\n",
    "# Check if a postcode can have more than one borough:\n",
    "if np.max(df.groupby('Postcode')['Borough'].nunique())==1:\n",
    "    print('As expected, there is only one borough in each postcode. :)')\n",
    "else:\n",
    "    print('Watch out! There is at least one postcode with multiple distinct boroughs...')\n",
    "\n",
    "# Give unassigned neighborhoods the name of their borough\n",
    "n = sum(df['Neighbourhood']=='Not assigned')\n",
    "print(f'{n:,} rows with unassigned neighbourhoods (now assigned to borough).')\n",
    "del n\n",
    "\n",
    "df.loc[df['Neighbourhood']=='Not assigned','Neighbourhood']=df['Borough']\n",
    "\n",
    "# Combine multiple rows for a given postcode into one comma-separated list of neighbourhoods\n",
    "n = df.shape[0]\n",
    "df=df.groupby(['Postcode','Borough'])['Neighbourhood'].apply(', '.join).reset_index()\n",
    "print(f'Collapsed {n-df.shape[0]:,} duplicated postcodes into single rows.')\n",
    "del n\n",
    "print('\\n')\n",
    "print(df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The cleaned dataframe has 103 total rows, making up the following buroughs:\n",
      "North York          24\n",
      "Downtown Toronto    18\n",
      "Scarborough         17\n",
      "Etobicoke           12\n",
      "Central Toronto      9\n",
      "West Toronto         6\n",
      "East York            5\n",
      "York                 5\n",
      "East Toronto         5\n",
      "Queen's Park         1\n",
      "Mississauga          1\n",
      "Name: Borough, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(f'The cleaned dataframe has {df.shape[0]:,} total rows, making up the following buroughs:')\n",
    "print(df['Borough'].value_counts())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>In the last cell of the notebook, use the .shape method to print the number of rows in the dataframe:</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "103\n"
     ]
    }
   ],
   "source": [
    "print(df.shape[0])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}