app_jupyter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from etl import fetcher, parser, extractor, loader, utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "CSV = '.csv'  # comma separated values\n",
    "DBF = '.dbf'  # dbase\n",
    "MDB = '.mdb'  # microsoft access database (jet, access, etc.)\n",
    "SBN = '.sbn'  # .shp support file\n",
    "SBX = '.sbx'  # .shp support file\n",
    "SHP = '.shp'  # shapes\n",
    "SHX = '.shx'  # .shp support file\n",
    "\n",
    "# ROOT_DIR = os.path.dirname(os.path.abspath(__file__))\n",
    "SUPPORTED_FILE_EXT = [CSV, DBF, MDB, SBN, SBX, SHP, SHX]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fetcher\n",
    "fetcher = fetcher.Fetcher()\n",
    "src_yaml = utils.get_yaml('data/sources/sources.yml')\n",
    "responses = fetcher.fetch_all(src_yaml)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parser\n",
    "parser = parser.Parser()\n",
    "for response in responses:\n",
    "    try:\n",
    "        response.payload = parser.flatten(response, SUPPORTED_FILE_EXT)\n",
    "    except Exception as err:\n",
    "        print(err)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extractor\n",
    "extractor = extractor.Extractor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".dbf\n",
      ".shp\n",
      "{'name': 'prcl_shape', 'payload': [{'filename': 'prcl.shx', 'data': <_io.BytesIO object at 0x10d8d8fc0>}, {'filename': 'prcl.dbf', 'data': <_io.BytesIO object at 0x118ccfe60>}, {'filename': 'prcl.sbn', 'data': <_io.BytesIO object at 0x118ccfeb8>}, {'filename': 'prcl.sbx', 'data': <_io.BytesIO object at 0x118ccff10>}, {'filename': 'prcl.shp', 'data': <_io.BytesIO object at 0x118ccff68>}], 'source': 'https://www.stlouis-mo.gov/data/upload/data-files/prcl_shape.zip', 'error': None}\n",
      "{'filename': 'prcl.shp', 'data': <_io.BytesIO object at 0x118ccff68>}\n",
      "done\n",
      ".mdb\n",
      ".dbf\n",
      ".mdb\n",
      ".mdb\n",
      ".mdb\n",
      ".mdb\n",
      ".csv\n"
     ]
    }
   ],
   "source": [
    "# Master entity list\n",
    "entity_dict = dict()\n",
    "entities = []\n",
    "# Populate dictionary\n",
    "for response in responses:\n",
    "    for payload in response.payload:\n",
    "        if utils.get_file_ext(payload.filename) == CSV:\n",
    "            print(CSV)\n",
    "            entities = extractor.get_csv_data(payload)\n",
    "        elif utils.get_file_ext(payload.filename) == MDB:\n",
    "            print(MDB)\n",
    "            entities = extractor.get_mdb_data(payload)\n",
    "        elif utils.get_file_ext(payload.filename) == DBF:\n",
    "            print(DBF)\n",
    "            entities = extractor.get_dbf_data(payload)\n",
    "        elif utils.get_file_ext(payload.filename) == SHP:\n",
    "            print(SHP)\n",
    "            entities = extractor.get_shp_data(response, payload)\n",
    "        # Add to master entity list\n",
    "        entity_dict.update(entities)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_dict['par.dbf']['SITEADDR']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_list[2].tablename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_list[3].tablename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_list[8].tablename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bleh.set_option('display.max_rows', 200, 'display.max_columns',148)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BldgCom = entity_list[2].dataframe.set_index('Handle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_list[3].dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bleh.merge(entity_list[8].dataframe,entity_list[3].dataframe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transformer\n",
    "for entity in entity_list:\n",
    "    print(entity.tablename)\n",
    "    #print(entity.dataframe.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transformer\n",
    "for entity in entity_list:\n",
    "    print(entity.tablename)\n",
    "    #print(entity.dataframe.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(entity[])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "entity.tablename == `par.dbf`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_list[1].dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": ".venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}