Magic Story Crawler.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "from ebooklib import epub\n",
    "import re\n",
    "import datetime\n",
    "\n",
    "import urllib.request\n",
    "with urllib.request.urlopen('http://magic.wizards.com/en/articles/columns/magic-story') as response:\n",
    "    html_doc = response.read()\n",
    "\n",
    "soup = BeautifulSoup(html_doc, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def getDate(string):\n",
    "    match = re.search(r'\\d{4}-\\d{2}-\\d{2}', string)\n",
    "    date = datetime.datetime.strptime(match.group(), '%Y-%m-%d').date()\n",
    "    return date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "links = list()\n",
    "\n",
    "for link in soup.find_all('a'):\n",
    "    links.append(str(link.get('href')))\n",
    "    \n",
    "links_filtered = [x for x in links if 'story' in x]\n",
    "links_filtered = [x for x in links if 'en/articles/archive/' in x]\n",
    "links_filtered = list(set(links_filtered))\n",
    "links_sorted = sorted(links_filtered, key=getDate)\n",
    "#links_filtered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['http://locator.wizards.com/',\n",
       " 'http://gatherer.wizards.com',\n",
       " '/en',\n",
       " '/en',\n",
       " '/en',\n",
       " '/en/new-to-magic',\n",
       " '/en/new-to-magic',\n",
       " '/en/content/strategy',\n",
       " '/en/new-to-magic/lore',\n",
       " 'http://magic.wizards.com/en/community',\n",
       " 'http://magic.wizards.com/en/products/amonkhet',\n",
       " 'http://magic.wizards.com/en/products/amonkhet',\n",
       " '/en/products/coming-soon',\n",
       " '/en/content/magic-duels',\n",
       " '/en/content/magic-online-products-game-info',\n",
       " '/en/game-info/products/card-set-archive',\n",
       " 'http://magic.wizards.com/en/game-information',\n",
       " '/en/gameplay',\n",
       " '/en/gameplay/about-the-game',\n",
       " '/en/gameplay/how-to-play',\n",
       " '/en/gameplay/how-to-build-a-deck',\n",
       " '/en/gameplay/improve-your-game',\n",
       " '/en/gameplay/rules-and-formats',\n",
       " 'http://magic.wizards.com/en/game-info/story/magic-story',\n",
       " '/en/story',\n",
       " '/en/story/planeswalkers',\n",
       " '/en/story/planes',\n",
       " '/en/events',\n",
       " '/en/events',\n",
       " '/en/events',\n",
       " '/en/events/what-is-organized-play',\n",
       " '/en/events/event-types',\n",
       " 'None',\n",
       " '/en/events/coverage',\n",
       " '/en/events/coverage/top-players',\n",
       " '/en/content/premier-events-schedule#/overview',\n",
       " 'None',\n",
       " '/en/events/resources/planeswalker-points',\n",
       " '/en/events/resources/judges',\n",
       " '/en/articles',\n",
       " '/en/articles',\n",
       " '/en/articles',\n",
       " 'None',\n",
       " '/en/articles/columns/columnists',\n",
       " '/en/articles/columns/making-magic',\n",
       " '/en/articles/columns/magic-story',\n",
       " '/en/articles/columns/top-decks',\n",
       " '/en/articles/columns/organized-play',\n",
       " '/en/articles/columns/latest-developments',\n",
       " '/en/articles/archive',\n",
       " '/en/articles/winning-decks',\n",
       " 'None',\n",
       " '/en/articles/media/videos',\n",
       " '/en/articles/media/podcasts',\n",
       " '/en/articles/media/wallpapers',\n",
       " '/en/articles/media/fan-kits',\n",
       " 'http://magic.wizards.com/en/products/amonkhet/story',\n",
       " '/products/aether-revolt-story',\n",
       " 'http://magic.wizards.com/en/content/kaladesh-story',\n",
       " 'http://magic.wizards.com/en/content/eldritch-moon-story',\n",
       " 'http://magic.wizards.com/en/content/shadows-over-innistrad-story',\n",
       " 'http://magic.wizards.com/en/content/oath-gatewatch-story',\n",
       " 'http://magic.wizards.com/en/content/battle-zendikar-story',\n",
       " '/en/articles/archive/uncharted-realms/gideons-origin-kytheon-iora-akros-2015-07-01',\n",
       " '/en/articles/archive/uncharted-realms/gideons-origin-kytheon-iora-akros-2015-07-01',\n",
       " '/en/articles/archive/uncharted-realms/gideons-origin-kytheon-iora-akros-2015-07-01',\n",
       " '/en/articles/archive/uncharted-realms/gideons-origin-kytheon-iora-akros-2015-07-01',\n",
       " '/en/articles/archive/uncharted-realms/jaces-origin-absent-minds-2015-06-24',\n",
       " '/en/articles/archive/uncharted-realms/jaces-origin-absent-minds-2015-06-24',\n",
       " '/en/articles/archive/uncharted-realms/jaces-origin-absent-minds-2015-06-24',\n",
       " '/en/articles/archive/uncharted-realms/jaces-origin-absent-minds-2015-06-24',\n",
       " '/en/articles/archive/uncharted-realms/lilianas-origin-fourth-pact-2015-06-17',\n",
       " '/en/articles/archive/uncharted-realms/lilianas-origin-fourth-pact-2015-06-17',\n",
       " '/en/articles/archive/uncharted-realms/lilianas-origin-fourth-pact-2015-06-17',\n",
       " '/en/articles/archive/uncharted-realms/lilianas-origin-fourth-pact-2015-06-17',\n",
       " '/en/articles/archive/uncharted-realms/chandras-origin-fire-logic-2015-06-10',\n",
       " '/en/articles/archive/uncharted-realms/chandras-origin-fire-logic-2015-06-10',\n",
       " '/en/articles/archive/uncharted-realms/chandras-origin-fire-logic-2015-06-10',\n",
       " '/en/articles/archive/uncharted-realms/chandras-origin-fire-logic-2015-06-10',\n",
       " '/en/articles/archive/uncharted-realms/nissas-origin-home-2015-07-08',\n",
       " '/en/articles/archive/uncharted-realms/nissas-origin-home-2015-07-08',\n",
       " '/en/articles/archive/uncharted-realms/nissas-origin-home-2015-07-08',\n",
       " '/en/articles/archive/uncharted-realms/nissas-origin-home-2015-07-08',\n",
       " '/en/articles/archive',\n",
       " '/en/articles/archive/magic-story/story-tarkir-block-2015-04-29',\n",
       " '/en/articles/archive/magic-story/story-tarkir-block-2015-04-29',\n",
       " '/en/articles/archive/magic-story/story-tarkir-block-2015-04-29',\n",
       " '/en/articles/archive/magic-story/story-tarkir-block-2015-04-29',\n",
       " '/en/articles/archive/magic-story/unbroken-and-unbowed-2015-04-22',\n",
       " '/en/articles/archive/magic-story/unbroken-and-unbowed-2015-04-22',\n",
       " '/en/articles/archive/magic-story/unbroken-and-unbowed-2015-04-22',\n",
       " '/en/articles/archive/magic-story/unbroken-and-unbowed-2015-04-22',\n",
       " '/en/articles/archive/magic-story/call-2015-04-15',\n",
       " '/en/articles/archive/magic-story/call-2015-04-15',\n",
       " '/en/articles/archive/magic-story/call-2015-04-15',\n",
       " '/en/articles/archive/magic-story/call-2015-04-15',\n",
       " '/en/articles/archive/magic-story/poisoned-heart-2015-04-08',\n",
       " '/en/articles/archive/magic-story/poisoned-heart-2015-04-08',\n",
       " '/en/articles/archive/magic-story/poisoned-heart-2015-04-08',\n",
       " '/en/articles/archive/magic-story/poisoned-heart-2015-04-08',\n",
       " '/en/articles/archive/magic-story/guardian-2015-04-02',\n",
       " '/en/articles/archive/magic-story/guardian-2015-04-02',\n",
       " '/en/articles/archive/magic-story/guardian-2015-04-02',\n",
       " '/en/articles/archive/magic-story/guardian-2015-04-02',\n",
       " '/en/articles/archive/magic-story/sorins-restoration-2015-03-27',\n",
       " '/en/articles/archive/magic-story/sorins-restoration-2015-03-27',\n",
       " '/en/articles/archive/magic-story/sorins-restoration-2015-03-27',\n",
       " '/en/articles/archive/magic-story/sorins-restoration-2015-03-27',\n",
       " '/en/articles/archive/magic-story/planeswalkers-guide-dragons-tarkir-part-2-2015-03-18',\n",
       " '/en/articles/archive/magic-story/planeswalkers-guide-dragons-tarkir-part-2-2015-03-18',\n",
       " '/en/articles/archive/magic-story/planeswalkers-guide-dragons-tarkir-part-2-2015-03-18',\n",
       " '/en/articles/archive/magic-story/planeswalkers-guide-dragons-tarkir-part-2-2015-03-18',\n",
       " '/en/articles/archive/uncharted-realms/planeswalkers-guide-dragons-tarkir-part-1-2015-03-11',\n",
       " '/en/articles/archive/uncharted-realms/planeswalkers-guide-dragons-tarkir-part-1-2015-03-11',\n",
       " '/en/articles/archive/uncharted-realms/planeswalkers-guide-dragons-tarkir-part-1-2015-03-11',\n",
       " '/en/articles/archive/uncharted-realms/planeswalkers-guide-dragons-tarkir-part-1-2015-03-11',\n",
       " '/en/articles/archive/magic-story/great-teachers-student-2015-03-04',\n",
       " '/en/articles/archive/magic-story/great-teachers-student-2015-03-04',\n",
       " '/en/articles/archive/magic-story/great-teachers-student-2015-03-04',\n",
       " '/en/articles/archive/magic-story/great-teachers-student-2015-03-04',\n",
       " '/en/articles/archive/magic-story/tarkir-dragons-2015-02-25',\n",
       " '/en/articles/archive/magic-story/tarkir-dragons-2015-02-25',\n",
       " '/en/articles/archive/magic-story/tarkir-dragons-2015-02-25',\n",
       " '/en/articles/archive/magic-story/tarkir-dragons-2015-02-25',\n",
       " '/en/articles/archive/magic-story/khanfall-2015-02-18',\n",
       " '/en/articles/archive/magic-story/khanfall-2015-02-18',\n",
       " '/en/articles/archive/magic-story/khanfall-2015-02-18',\n",
       " '/en/articles/archive/magic-story/khanfall-2015-02-18',\n",
       " '/en/articles/archive/magic-story/no-end-and-no-beginning-2015-02-11',\n",
       " '/en/articles/archive/magic-story/no-end-and-no-beginning-2015-02-11',\n",
       " '/en/articles/archive/magic-story/no-end-and-no-beginning-2015-02-11',\n",
       " '/en/articles/archive/magic-story/no-end-and-no-beginning-2015-02-11',\n",
       " '/en/articles/archive/magic-story/doom-golden-fang-2015-02-04',\n",
       " '/en/articles/archive/magic-story/doom-golden-fang-2015-02-04',\n",
       " '/en/articles/archive/magic-story/doom-golden-fang-2015-02-04',\n",
       " '/en/articles/archive/magic-story/doom-golden-fang-2015-02-04',\n",
       " '/en/articles/archive/magic-story/new-tarkir-old-2014-12-31',\n",
       " '/en/articles/archive/magic-story/new-tarkir-old-2014-12-31',\n",
       " '/en/articles/archive/magic-story/new-tarkir-old-2014-12-31',\n",
       " '/en/articles/archive/magic-story/new-tarkir-old-2014-12-31',\n",
       " '/en/articles/archive/ur/chensal-twins-2014-10-15',\n",
       " '/en/articles/archive/ur/chensal-twins-2014-10-15',\n",
       " '/en/articles/archive/ur/chensal-twins-2014-10-15',\n",
       " '/en/articles/archive/ur/chensal-twins-2014-10-15',\n",
       " '/en/articles/archive/planeswalkers-guide-khans-tarkir-part-2-2014-09-10',\n",
       " '/en/articles/archive/planeswalkers-guide-khans-tarkir-part-2-2014-09-10',\n",
       " '/en/articles/archive/planeswalkers-guide-khans-tarkir-part-2-2014-09-10',\n",
       " '/en/articles/archive/planeswalkers-guide-khans-tarkir-part-2-2014-09-10',\n",
       " '/en/articles/archive/planeswalkers-guide-khans-tarkir-part-1-2014-09-03',\n",
       " '/en/articles/archive/planeswalkers-guide-khans-tarkir-part-1-2014-09-03',\n",
       " '/en/articles/archive/planeswalkers-guide-khans-tarkir-part-1-2014-09-03',\n",
       " '/en/articles/archive/planeswalkers-guide-khans-tarkir-part-1-2014-09-03',\n",
       " '/en/articles/archive/ur/madness-sarkhan-2014-08-27',\n",
       " '/en/articles/archive/ur/madness-sarkhan-2014-08-27',\n",
       " '/en/articles/archive/ur/madness-sarkhan-2014-08-27',\n",
       " '/en/articles/archive/ur/madness-sarkhan-2014-08-27',\n",
       " '/en/articles/archive/ur/taigams-scheming-2014-10-01',\n",
       " '/en/articles/archive/ur/taigams-scheming-2014-10-01',\n",
       " '/en/articles/archive/ur/taigams-scheming-2014-10-01',\n",
       " '/en/articles/archive/ur/taigams-scheming-2014-10-01',\n",
       " '/en/articles/archive/feature/magic-fiction-ebook-collection-2015-09-23',\n",
       " '/en/articles/archive/feature/magic-fiction-ebook-collection-2015-09-23',\n",
       " '/en/articles/archive/feature/magic-fiction-ebook-collection-2015-09-23',\n",
       " '/en/articles/archive/feature/magic-fiction-ebook-collection-2015-09-23',\n",
       " '/en/articles/archive',\n",
       " '/en/articles/archive/magic-story/drop-drop-2015-05-20',\n",
       " '/en/articles/archive/magic-story/drop-drop-2015-05-20',\n",
       " '/en/articles/archive/magic-story/drop-drop-2015-05-20',\n",
       " '/en/articles/archive/magic-story/drop-drop-2015-05-20',\n",
       " '/en/articles/archive/ajanis-vengeance-2014-07-23',\n",
       " '/en/articles/archive/ajanis-vengeance-2014-07-23',\n",
       " '/en/articles/archive/ajanis-vengeance-2014-07-23',\n",
       " '/en/articles/archive/ajanis-vengeance-2014-07-23',\n",
       " '/en/articles/archive/magic-story/kruphixs-insight-2014-06-11',\n",
       " '/en/articles/archive/magic-story/kruphixs-insight-2014-06-11',\n",
       " '/en/articles/archive/magic-story/kruphixs-insight-2014-06-11',\n",
       " '/en/articles/archive/magic-story/kruphixs-insight-2014-06-11',\n",
       " '/en/articles/archive/magic-story/path-or-horizon-2014-05-07',\n",
       " '/en/articles/archive/magic-story/path-or-horizon-2014-05-07',\n",
       " '/en/articles/archive/magic-story/path-or-horizon-2014-05-07',\n",
       " '/en/articles/archive/magic-story/path-or-horizon-2014-05-07',\n",
       " '/en/articles/archive/magic-story/thank-gods-2014-04-30',\n",
       " '/en/articles/archive/magic-story/thank-gods-2014-04-30',\n",
       " '/en/articles/archive/magic-story/thank-gods-2014-04-30',\n",
       " '/en/articles/archive/magic-story/thank-gods-2014-04-30',\n",
       " '/en/articles/archive/magic-story/dreams-city-2014-04-23',\n",
       " '/en/articles/archive/magic-story/dreams-city-2014-04-23',\n",
       " '/en/articles/archive/magic-story/dreams-city-2014-04-23',\n",
       " '/en/articles/archive/magic-story/dreams-city-2014-04-23',\n",
       " '/en/articles/archive/magic-story/desperate-stand-2014-04-16',\n",
       " '/en/articles/archive/magic-story/desperate-stand-2014-04-16',\n",
       " '/en/articles/archive/magic-story/desperate-stand-2014-04-16',\n",
       " '/en/articles/archive/magic-story/desperate-stand-2014-04-16',\n",
       " '/en/articles/archive/magic-story/ajani-mentor-heroes-2014-04-09',\n",
       " '/en/articles/archive/magic-story/ajani-mentor-heroes-2014-04-09',\n",
       " '/en/articles/archive/magic-story/ajani-mentor-heroes-2014-04-09',\n",
       " '/en/articles/archive/magic-story/ajani-mentor-heroes-2014-04-09',\n",
       " '/en/articles/archive/seasons-setessa-2014-03-26',\n",
       " '/en/articles/archive/seasons-setessa-2014-03-26',\n",
       " '/en/articles/archive/seasons-setessa-2014-03-26',\n",
       " '/en/articles/archive/seasons-setessa-2014-03-26',\n",
       " '/en/articles/archive/magic-story/oracle-ephara-2014-03-19',\n",
       " '/en/articles/archive/magic-story/oracle-ephara-2014-03-19',\n",
       " '/en/articles/archive/magic-story/oracle-ephara-2014-03-19',\n",
       " '/en/articles/archive/magic-story/oracle-ephara-2014-03-19',\n",
       " '/en/articles/archive/magic-story/hero-iroas-2014-03-05',\n",
       " '/en/articles/archive/magic-story/hero-iroas-2014-03-05',\n",
       " '/en/articles/archive/magic-story/hero-iroas-2014-03-05',\n",
       " '/en/articles/archive/magic-story/hero-iroas-2014-03-05',\n",
       " '/en/articles/archive/magic-story/walls-akros-2014-02-26',\n",
       " '/en/articles/archive/magic-story/walls-akros-2014-02-26',\n",
       " '/en/articles/archive/magic-story/walls-akros-2014-02-26',\n",
       " '/en/articles/archive/magic-story/walls-akros-2014-02-26',\n",
       " '/en/articles/archive/magic-story/dance-flitterstep-2014-02-19',\n",
       " '/en/articles/archive/magic-story/dance-flitterstep-2014-02-19',\n",
       " '/en/articles/archive/magic-story/dance-flitterstep-2014-02-19',\n",
       " '/en/articles/archive/magic-story/dance-flitterstep-2014-02-19',\n",
       " '/en/articles/archive/magic-story/kioras-followers-2014-02-14',\n",
       " '/en/articles/archive/magic-story/kioras-followers-2014-02-14',\n",
       " '/en/articles/archive/magic-story/kioras-followers-2014-02-14',\n",
       " '/en/articles/archive/magic-story/kioras-followers-2014-02-14',\n",
       " '/en/articles/archive/magic-story/emonberry-red-2014-02-05',\n",
       " '/en/articles/archive/magic-story/emonberry-red-2014-02-05',\n",
       " '/en/articles/archive/magic-story/emonberry-red-2014-02-05',\n",
       " '/en/articles/archive/magic-story/emonberry-red-2014-02-05',\n",
       " '/en/articles/archive/magic-story/cowardice-hero-2014-01-22',\n",
       " '/en/articles/archive/magic-story/cowardice-hero-2014-01-22',\n",
       " '/en/articles/archive/magic-story/cowardice-hero-2014-01-22',\n",
       " '/en/articles/archive/magic-story/cowardice-hero-2014-01-22',\n",
       " '/en/articles/archive',\n",
       " '/en/articles/archive/magic-story/zendikar-2015-08-12',\n",
       " '/en/articles/archive/magic-story/zendikar-2015-08-12',\n",
       " '/en/articles/archive/magic-story/zendikar-2015-08-12',\n",
       " '/en/articles/archive/magic-story/zendikar-2015-08-12',\n",
       " '/en/articles/archive/magic-story/stirring-slumber-2015-05-13',\n",
       " '/en/articles/archive/magic-story/stirring-slumber-2015-05-13',\n",
       " '/en/articles/archive/magic-story/stirring-slumber-2015-05-13',\n",
       " '/en/articles/archive/magic-story/stirring-slumber-2015-05-13',\n",
       " '/en/articles/archive/ur/dreams-damned-2014-08-06',\n",
       " '/en/articles/archive/ur/dreams-damned-2014-08-06',\n",
       " '/en/articles/archive/ur/dreams-damned-2014-08-06',\n",
       " '/en/articles/archive/ur/dreams-damned-2014-08-06',\n",
       " '/en/articles/archive/magic-story/nissa-worldwaker-2014-06-25',\n",
       " '/en/articles/archive/magic-story/nissa-worldwaker-2014-06-25',\n",
       " '/en/articles/archive/magic-story/nissa-worldwaker-2014-06-25',\n",
       " '/en/articles/archive/magic-story/nissa-worldwaker-2014-06-25',\n",
       " '/en/articles/archive',\n",
       " '/en/articles/archive/magic-story/gorgon-and-guildpact-2014-03-12',\n",
       " '/en/articles/archive/magic-story/gorgon-and-guildpact-2014-03-12',\n",
       " '/en/articles/archive/magic-story/gorgon-and-guildpact-2014-03-12',\n",
       " '/en/articles/archive/magic-story/gorgon-and-guildpact-2014-03-12',\n",
       " '/en/articles/archive/magic-story/krenko-mob-boss-2012-07-18',\n",
       " '/en/articles/archive/magic-story/krenko-mob-boss-2012-07-18',\n",
       " '/en/articles/archive/magic-story/krenko-mob-boss-2012-07-18',\n",
       " '/en/articles/archive/magic-story/krenko-mob-boss-2012-07-18',\n",
       " '/en/articles/archive',\n",
       " '/en/articles/archive/magic-story/bard-and-biologist-2014-07-09',\n",
       " '/en/articles/archive/magic-story/bard-and-biologist-2014-07-09',\n",
       " '/en/articles/archive/magic-story/bard-and-biologist-2014-07-09',\n",
       " '/en/articles/archive/magic-story/bard-and-biologist-2014-07-09',\n",
       " '/en/articles/archive/magic-story/veil-deceit-2014-07-01',\n",
       " '/en/articles/archive/magic-story/veil-deceit-2014-07-01',\n",
       " '/en/articles/archive/magic-story/veil-deceit-2014-07-01',\n",
       " '/en/articles/archive/magic-story/veil-deceit-2014-07-01',\n",
       " '/en/articles/archive/ur/monster-2014-08-20',\n",
       " '/en/articles/archive/ur/monster-2014-08-20',\n",
       " '/en/articles/archive/ur/monster-2014-08-20',\n",
       " '/en/articles/archive/ur/monster-2014-08-20',\n",
       " '/en/articles/archive/magic-story/cogwork-2014-05-28',\n",
       " '/en/articles/archive/magic-story/cogwork-2014-05-28',\n",
       " '/en/articles/archive/magic-story/cogwork-2014-05-28',\n",
       " '/en/articles/archive/magic-story/cogwork-2014-05-28',\n",
       " '/en/articles/archive/magic-story/black-rose-2014-05-21',\n",
       " '/en/articles/archive/magic-story/black-rose-2014-05-21',\n",
       " '/en/articles/archive/magic-story/black-rose-2014-05-21',\n",
       " '/en/articles/archive/magic-story/black-rose-2014-05-21',\n",
       " '/en/articles/archive/magic-story/betrayal-2014-05-14',\n",
       " '/en/articles/archive/magic-story/betrayal-2014-05-14',\n",
       " '/en/articles/archive/magic-story/betrayal-2014-05-14',\n",
       " '/en/articles/archive/magic-story/betrayal-2014-05-14',\n",
       " '/en/articles/archive/blood-will-have-blood-2014-06-04',\n",
       " '/en/articles/archive/blood-will-have-blood-2014-06-04',\n",
       " '/en/articles/archive/blood-will-have-blood-2014-06-04',\n",
       " '/en/articles/archive/blood-will-have-blood-2014-06-04',\n",
       " '/en/articles/archive',\n",
       " '/en/articles/archive/comin-through-2014-07-30',\n",
       " '/en/articles/archive/comin-through-2014-07-30',\n",
       " '/en/articles/archive/comin-through-2014-07-30',\n",
       " '/en/articles/archive/comin-through-2014-07-30',\n",
       " '/en/articles/archive/magic-story/blessed-life-2012-07-24',\n",
       " '/en/articles/archive/magic-story/blessed-life-2012-07-24',\n",
       " '/en/articles/archive/magic-story/blessed-life-2012-07-24',\n",
       " '/en/articles/archive/magic-story/blessed-life-2012-07-24',\n",
       " '/en/articles/archive',\n",
       " '/en/articles/archive/magic-story/hour-promise-2017-06-28',\n",
       " '/en/articles/archive/magic-story/hour-glory-2017-06-21',\n",
       " '/en/articles/archive/magic-story/feast-2017-06-14',\n",
       " '/en/articles/archive/magic-story/hour-revelation-2017-06-07',\n",
       " '/en/articles/archive/magic-story/pride-kraul-2017-05-31',\n",
       " '/en/articles/archive/magic-story/judgment-2017-05-17',\n",
       " 'javascript:void(0);',\n",
       " '#viewport',\n",
       " 'http://magic.wizards.com',\n",
       " 'http://www.wizards.com/dnd',\n",
       " 'http://www.wizards.com/wpn',\n",
       " 'http://dm.takaratomy.co.jp',\n",
       " 'http://www.wizards.com/default.asp?x=ah/welcome',\n",
       " 'http://company.wizards.com/tou',\n",
       " 'http://company.wizards.com/conduct',\n",
       " 'http://company.wizards.com/policy',\n",
       " 'http://www.wizards.com/customerservice',\n",
       " '/en/articles/archive/our-cookie-policy-2013-10-28',\n",
       " 'http://company.wizards.com',\n",
       " 'http://www.hasbro.com',\n",
       " 'http://www.esrb.org/confirm/wizards-confirmation.aspx',\n",
       " 'http://company.wizards.com/policies/web/cookie',\n",
       " 'http://company.wizards.com/policies/web/cookie']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def makeTitlePretty(soupTitle):\n",
    "    stringTitle = str(soupTitle)\n",
    "    stringTitle = stringTitle.split('>')[1].split('|')[0]\n",
    "    return stringTitle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def createChapter(url):\n",
    "    response = urllib.request.urlopen(url)\n",
    "    temp_doc = response.read()\n",
    "    sub_soup = BeautifulSoup(temp_doc, 'html.parser')\n",
    "    c1_title = str(makeTitlePretty(sub_soup.title))\n",
    "\n",
    "    content = sub_soup.find_all(\"div\", id=\"content-detail-page-of-an-article\")\n",
    "    c1_content = str(content[0].find_all('body')).replace('[','').replace(']','')\n",
    "    return c1_title, c1_content"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gesamtcrawler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "chapters = dict()   \n",
    "for link in links_sorted:\n",
    "    c_title, c_content = createChapter('http://magic.wizards.com' + link)\n",
    "    print(c_title + ' done.')\n",
    "    chapters[c_title] = c_content\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Epub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def create_and_add_chapter(c_title, c_content, book):\n",
    "    \n",
    "    c_xhtml = c_title+'.xhtml'\n",
    "\n",
    "    chapter = epub.EpubHtml(title=c_title, file_name=c_xhtml, lang='hr')\n",
    "    chapter.content=u'<h1>'+ c_title + '</h1>' + str(c_content)\n",
    "\n",
    "    ### add chapter\n",
    "    magic_stories_epub.add_item(chapter)\n",
    "    return chapter, book"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "magic_stories_epub = epub.EpubBook()\n",
    "magic_stories_epub.set_title('Magic the Gathering Stories') \n",
    "magic_stories_epub.set_language('en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ebook_chapters = list()\n",
    "\n",
    "for c_title in chapters:\n",
    "    chap, magic_stories_epub = create_and_add_chapter(c_title, chapters[c_title], magic_stories_epub)\n",
    "    ebook_chapters.append(chap)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "### define Table Of Contents\n",
    "magic_stories_epub.toc = (epub.Link('Test1', 'Test2', 'Test3'),(epub.Section('Stories'),ebook_chapters))\n",
    "\n",
    "### add default NCX and Nav file\n",
    "magic_stories_epub.add_item(epub.EpubNcx())\n",
    "magic_stories_epub.add_item(epub.EpubNav())\n",
    "\n",
    "### define CSS style\n",
    "style = 'BODY {color: white;}'\n",
    "nav_css = epub.EpubItem(uid=\"style_nav\", file_name=\"style/nav.css\", media_type=\"text/css\", content=style)\n",
    "\n",
    "### add CSS file\n",
    "magic_stories_epub.add_item(nav_css)\n",
    "\n",
    "### basic spine\n",
    "spine = ['nav']\n",
    "spine.extend(ebook_chapters)\n",
    "magic_stories_epub.spine = spine\n",
    "\n",
    "### write to the file\n",
    "epub.write_epub('test.epub', magic_stories_epub, {})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Notes\n",
    "\n",
    "http://magic.wizards.com/en/articles/archive -  Alle Artikel\n",
    "\n",
    "http://magic.wizards.com/en/articles/columns/magic-story - Story Auflistungsseite\n",
    "\n",
    "\n",
    "from ebooklib import epub\n",
    "\n",
    "magic_stories_epub = epub.EpubBook()\n",
    "\n",
    "### set metadata\n",
    "book.set_identifier('id123456')\n",
    "book.set_title('Sample book')\n",
    "book.set_language('en')\n",
    "\n",
    "book.add_author('Author Authorowski')\n",
    "book.add_author('Danko Bananko', file_as='Gospodin Danko Bananko', role='ill', uid='coauthor')\n",
    "\n",
    "### create chapter\n",
    "c1 = epub.EpubHtml(title='Intro', file_name='chap_01.xhtml', lang='hr')\n",
    "c1.content=u'<h1>Intro heading</h1><p>Žaba je skočila u baru.</p>'\n",
    "\n",
    "### add chapter\n",
    "book.add_item(c1)\n",
    "\n",
    "### define Table Of Contents\n",
    "book.toc = (epub.Link('chap_01.xhtml', 'Introduction', 'intro'),\n",
    "(epub.Section('Simple book'),\n",
    "(c1, ))\n",
    ")\n",
    "\n",
    "### add default NCX and Nav file\n",
    "book.add_item(epub.EpubNcx())\n",
    "book.add_item(epub.EpubNav())\n",
    "\n",
    "### define CSS style\n",
    "style = 'BODY {color: white;}'\n",
    "nav_css = epub.EpubItem(uid=\"style_nav\", file_name=\"style/nav.css\", media_type=\"text/css\", content=style)\n",
    "\n",
    "### add CSS file\n",
    "book.add_item(nav_css)\n",
    "\n",
    "### basic spine\n",
    "book.spine = ['nav', c1]\n",
    "\n",
    "### write to the file\n",
    "epub.write_epub('test.epub', book, {})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}