Scrapping_Booking_v5.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "f79a9553",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%\n",
    "#!pip3 install -r requirements.txt\n",
    "#import dropbox\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "# auxiliary functions modified by Luis.\n",
    "import scrape_functions as kzd\n",
    "import sys\n",
    "import calendar\n",
    "from selenium.common.exceptions import NoSuchElementException, ElementClickInterceptedException, StaleElementReferenceException\n",
    "import re\n",
    "from selenium.webdriver.support.ui import Select\n",
    "import random\n",
    "import time\n",
    "from selenium import webdriver\n",
    "import os\n",
    "import numpy as np\n",
    "import importlib\n",
    "importlib.reload(sys.modules['scrape_functions'])\n",
    "from datetime import date\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.remote.webelement import WebElement\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "import pandas as pd\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "a145dc3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def BookingReport(deal_boxes):\n",
    "    page_report = []\n",
    "    for deal_box in deal_boxes:\n",
    "        hotel_name = deal_box.find_element(By.CSS_SELECTOR,\n",
    "            'div[data-testid=\"title\"]'\n",
    "        ).get_attribute('innerHTML').strip()\n",
    "\n",
    "        if not deal_box.find_elements(By.CSS_SELECTOR, 'span[data-testid=\"price-and-discounted-price\"]'):\n",
    "            hotel_price = deal_box.find_element(By.CSS_SELECTOR,\n",
    "                'div[data-testid=\"price-and-discounted-price\"]'\n",
    "            ).find_element(By.TAG_NAME, 'div').get_attribute('innerHTML').strip()\n",
    "        else:\n",
    "            hotel_price = deal_box.find_element(By.CSS_SELECTOR,\n",
    "                'span[data-testid=\"price-and-discounted-price\"]'\n",
    "            ).get_attribute('innerHTML').strip()\n",
    "            hotel_price = hotel_price.replace(\"€&nbsp;\", \"\")\n",
    "            hotel_price = hotel_price.replace(\",\", \"\")\n",
    "            hotel_price = int(hotel_price)\n",
    "\n",
    "        try:\n",
    "            hotel_score = deal_box.find_element(By.CSS_SELECTOR,\n",
    "                'div[aria-label*=\"Scored\"]'\n",
    "            ).get_attribute('innerHTML').strip()\n",
    "        except:\n",
    "            hotel_score = 'nan'\n",
    "        page_report.append(\n",
    "            [hotel_name, hotel_price, hotel_score]\n",
    "        )\n",
    "        # print([hotel_name, hotel_price, hotel_score])\n",
    "    return page_report\n",
    "\n",
    "def get_number_pages(browser):\n",
    "    '''\n",
    "    Get the number of pages. \n",
    "    '''\n",
    "    a = browser.find_elements('xpath',\n",
    "        '//button[text() and @class=\"fc63351294 f9c5690c58\"]')\n",
    "    total_pages = int(a[-1].text)\n",
    "    time.sleep(1)\n",
    "    return(total_pages)\n",
    "\n",
    "def click(browser,element_type, element):\n",
    "    thing = browser.find_element(element_type, element)\n",
    "    browser.execute_script(\"arguments[0].click()\", thing)\n",
    "\n",
    "def zoom_out(browser):\n",
    "    browser.execute_script(\"document.body.style.zoom='30%'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "3c0a6f4b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dates:  59\n"
     ]
    },
    {
     "ename": "NoSuchElementException",
     "evalue": "Message: no such element: Unable to locate element: {\"method\":\"xpath\",\"selector\":\"//button[@class=\"fc63351294 a822bdf511 e3c025e003 fa565176a8 cfb238afa1 ae1678b153 c9fa5fc96d be298b15fa\"]\"}\n  (Session info: chrome=109.0.5414.120)\nStacktrace:\nBacktrace:\n\t(No symbol) [0x00DA6643]\n\t(No symbol) [0x00D3BE21]\n\t(No symbol) [0x00C3DA9D]\n\t(No symbol) [0x00C71342]\n\t(No symbol) [0x00C7147B]\n\t(No symbol) [0x00CA8DC2]\n\t(No symbol) [0x00C8FDC4]\n\t(No symbol) [0x00CA6B09]\n\t(No symbol) [0x00C8FB76]\n\t(No symbol) [0x00C649C1]\n\t(No symbol) [0x00C65E5D]\n\tGetHandleVerifier [0x0101A142+2497106]\n\tGetHandleVerifier [0x010485D3+2686691]\n\tGetHandleVerifier [0x0104BB9C+2700460]\n\tGetHandleVerifier [0x00E53B10+635936]\n\t(No symbol) [0x00D44A1F]\n\t(No symbol) [0x00D4A418]\n\t(No symbol) [0x00D4A505]\n\t(No symbol) [0x00D5508B]\n\tBaseThreadInitThunk [0x759D00F9+25]\n\tRtlGetAppContainerNamedObjectPath [0x77447BBE+286]\n\tRtlGetAppContainerNamedObjectPath [0x77447B8E+238]\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNoSuchElementException\u001b[0m                    Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-32-179ece18cd19>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     37\u001b[0m             \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'dates: '\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdates\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m             \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 39\u001b[1;33m             \u001b[0mclick\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbrowser\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'xpath'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'//button[@class=\"fc63351294 a822bdf511 e3c025e003 fa565176a8 cfb238afa1 ae1678b153 c9fa5fc96d be298b15fa\"]'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     40\u001b[0m             \u001b[1;31m#click(browser, 'xpath', '//button[@class=\"b6dc9a9e69 e6c50852bd\"]')\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     41\u001b[0m             \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'month clicked'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-31-014557295c2b>\u001b[0m in \u001b[0;36mclick\u001b[1;34m(browser, element_type, element)\u001b[0m\n\u001b[0;32m     41\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     42\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mclick\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbrowser\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0melement_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0melement\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 43\u001b[1;33m     \u001b[0mthing\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbrowser\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0melement_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0melement\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     44\u001b[0m     \u001b[0mbrowser\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute_script\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"arguments[0].click()\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mthing\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     45\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mfind_element\u001b[1;34m(self, by, value)\u001b[0m\n\u001b[0;32m    828\u001b[0m             \u001b[0mvalue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34mf'[name=\"{value}\"]'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    829\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 830\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCommand\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mFIND_ELEMENT\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;34m\"using\"\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mby\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"value\"\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"value\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    831\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    832\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfind_elements\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mby\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mBy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mID\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m->\u001b[0m \u001b[0mList\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mWebElement\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    438\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommand_executor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver_command\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    439\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 440\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    441\u001b[0m             \u001b[0mresponse\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"value\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_unwrap_value\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"value\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    442\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    243\u001b[0m                 \u001b[0malert_text\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"alert\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"text\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    244\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malert_text\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# type: ignore[call-arg]  # mypy is not smart enough here\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 245\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNoSuchElementException\u001b[0m: Message: no such element: Unable to locate element: {\"method\":\"xpath\",\"selector\":\"//button[@class=\"fc63351294 a822bdf511 e3c025e003 fa565176a8 cfb238afa1 ae1678b153 c9fa5fc96d be298b15fa\"]\"}\n  (Session info: chrome=109.0.5414.120)\nStacktrace:\nBacktrace:\n\t(No symbol) [0x00DA6643]\n\t(No symbol) [0x00D3BE21]\n\t(No symbol) [0x00C3DA9D]\n\t(No symbol) [0x00C71342]\n\t(No symbol) [0x00C7147B]\n\t(No symbol) [0x00CA8DC2]\n\t(No symbol) [0x00C8FDC4]\n\t(No symbol) [0x00CA6B09]\n\t(No symbol) [0x00C8FB76]\n\t(No symbol) [0x00C649C1]\n\t(No symbol) [0x00C65E5D]\n\tGetHandleVerifier [0x0101A142+2497106]\n\tGetHandleVerifier [0x010485D3+2686691]\n\tGetHandleVerifier [0x0104BB9C+2700460]\n\tGetHandleVerifier [0x00E53B10+635936]\n\t(No symbol) [0x00D44A1F]\n\t(No symbol) [0x00D4A418]\n\t(No symbol) [0x00D4A505]\n\t(No symbol) [0x00D5508B]\n\tBaseThreadInitThunk [0x759D00F9+25]\n\tRtlGetAppContainerNamedObjectPath [0x77447BBE+286]\n\tRtlGetAppContainerNamedObjectPath [0x77447B8E+238]\n"
     ]
    }
   ],
   "source": [
    "#%%\n",
    "# %% Starting code\n",
    "cities = ['Paris', 'Barcelona', 'Lisboa', 'Palma de Mallorca']\n",
    "\n",
    "i=0\n",
    "for place in cities:\n",
    "    if place != 'Paris':\n",
    "        i=0\n",
    "    check_in_date = [\"2023-05-15\",\"2023-05-22\",\"2023-05-29\"]\n",
    "    check_out_date = [\"2023-05-21\",\"2023-05-28\",\"2023-06-04\"]\n",
    "    \n",
    "    for d in check_in_date:\n",
    "        link = 'https://www.booking.com/'\n",
    "        browser = webdriver.Chrome()\n",
    "        browser.get(link)\n",
    "\n",
    "        zoom_out(browser)\n",
    "        time.sleep(2)\n",
    "        destination_input = browser.find_element('xpath','//input[@placeholder=\"Where are you going?\"]')\n",
    "        destination_input.send_keys(place)\n",
    "\n",
    "\n",
    "        click(browser, 'xpath', '//button[@type=\"submit\"]')\n",
    "\n",
    "        time.sleep(2)\n",
    "\n",
    "        zoom_out(browser)\n",
    "\n",
    "        week=[check_in_date[i], check_out_date[i]]\n",
    "        today_month = datetime.now().month\n",
    "        while today_month < int(week[0].split('-')[1]):\n",
    "            time.sleep(2)\n",
    "            dates = browser.find_elements('xpath','//table[@class=\"aadb8ed6d3\"]/tbody/tr/td/span')\n",
    "            print('dates: ', len(dates))\n",
    "            time.sleep(2)\n",
    "            click(browser, 'xpath', '//button[@class=\"fc63351294 a822bdf511 e3c025e003 fa565176a8 cfb238afa1 ae1678b153 c9fa5fc96d be298b15fa\"]')\n",
    "            #click(browser, 'xpath', '//button[@class=\"b6dc9a9e69 e6c50852bd\"]')\n",
    "            print('month clicked')\n",
    "            today_month+=1\n",
    "\n",
    "           \n",
    "        #%%\n",
    "        #enter start-date\n",
    "        click(browser, 'xpath', '//span[@data-date=\"{}\"]'.format(week[0]))\n",
    "        #click empty space\n",
    "        click(browser, 'xpath', '//input[@class=\"ce45093752\"]')\n",
    "        #click end date button\n",
    "        click(browser, 'xpath', '//button[@data-testid=\"date-display-field-end\"]')\n",
    "        #select specific end date\n",
    "        click(browser, 'xpath', '//span[@data-date=\"{}\"]'.format(week[1]))\n",
    "\n",
    "        # click on empty space\n",
    "        click(browser, 'xpath', '//input[@class=\"ce45093752\"]')\n",
    "\n",
    "        time.sleep(1)\n",
    "\n",
    "        search_button = browser.find_element('xpath', '//button[@type=\"submit\"]')\n",
    "        browser.execute_script(\"arguments[0].click()\", search_button)\n",
    "        print('search button clicked')\n",
    "\n",
    "        time.sleep(2)\n",
    "\n",
    "        pages = get_number_pages(browser)\n",
    "        print('total pages: ', pages)\n",
    "        #%%\n",
    "\n",
    "        #%%        \n",
    "        # Loop pages\n",
    "        full_report = []\n",
    "\n",
    "        for page in range(pages):\n",
    "            # Get the list of hotel elements on the page\n",
    "\n",
    "            print('Scraping page {} out of {}...'.format(page+1, pages))\n",
    "            print('before: ', len(full_report))\n",
    "\n",
    "            browser.implicitly_wait(2)\n",
    "\n",
    "            zoom_out(browser)\n",
    "\n",
    "            deal_boxes = browser.find_elements(\n",
    "                    By.XPATH,\n",
    "                    \"//div[@data-testid='property-card']\"\n",
    "                    )\n",
    "            report = BookingReport(deal_boxes)\n",
    "            time.sleep(4)\n",
    "\n",
    "            full_report.extend(report)\n",
    "\n",
    "            print('after: ', len(full_report))\n",
    "\n",
    "            click(browser, 'xpath', '//button[@aria-label=\"Next page\"]')\n",
    "\n",
    "            time.sleep(4)\n",
    "\n",
    "        report_df = pd.DataFrame(full_report)\n",
    "\n",
    "        # %%\n",
    "        deal_boxes = browser.find_elements(\n",
    "                    By.XPATH,\n",
    "                    \"//div[@data-testid='property-card']\")\n",
    "\n",
    "        report = BookingReport(deal_boxes)\n",
    "        report_df = pd.DataFrame(full_report, columns=['hotel','price','rating'])\n",
    "        report_df['start_date'] = week[0]\n",
    "        report_df['city']=place\n",
    "        df = report_df.copy()\n",
    "        j=0\n",
    "        for r in df['price']:\n",
    "            if type(r) is int:\n",
    "                pass\n",
    "            else:\n",
    "                df['price'][j] = df['price'][j].replace(\"Price €&nbsp;\", \"\")\n",
    "                df['price'][j] = df['price'][j].replace(\",\", \"\")\n",
    "                df['price'][j] = int(df['price'][j])\n",
    "            j+=1\n",
    "        \n",
    "        if place == 'Palma de Mallorca':\n",
    "            place = 'Palma_de_Mallorca'\n",
    "        df.to_csv('scrape{}v3.csv'.format(place+week[0]))\n",
    "        i+=1\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7289d8a0",
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'scrapePalma_de_Mallorca2023-05-08v3.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-8-cae0316d3966>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[0mdf11\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'scrapeLisboa2023-05-22v3.csv'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[0mdf12\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'scrapeLisboa2023-05-29v3.csv'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 13\u001b[1;33m \u001b[0mdf13\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'scrapePalma_de_Mallorca2023-05-08v3.csv'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     14\u001b[0m \u001b[0mdf14\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'scrapePalma_de_Mallorca2023-05-15v3.csv'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     15\u001b[0m \u001b[0mdf15\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'scrapePalma_de_Mallorca2023-05-22v3.csv'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\util\\_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    309\u001b[0m                     \u001b[0mstacklevel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mstacklevel\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    310\u001b[0m                 )\n\u001b[1;32m--> 311\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    312\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    313\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001b[0m in \u001b[0;36mread_csv\u001b[1;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, error_bad_lines, warn_bad_lines, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options)\u001b[0m\n\u001b[0;32m    584\u001b[0m     \u001b[0mkwds\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkwds_defaults\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    585\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 586\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    587\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    588\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001b[0m in \u001b[0;36m_read\u001b[1;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[0;32m    480\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    481\u001b[0m     \u001b[1;31m# Create the parser.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 482\u001b[1;33m     \u001b[0mparser\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    483\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    484\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[0;32m    809\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"has_index_names\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"has_index_names\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    810\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 811\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    812\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    813\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[1;34m(self, engine)\u001b[0m\n\u001b[0;32m   1038\u001b[0m             )\n\u001b[0;32m   1039\u001b[0m         \u001b[1;31m# error: Too many arguments for \"ParserBase\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1040\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mmapping\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# type: ignore[call-arg]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1041\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1042\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_failover_to_python\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\parsers\\c_parser_wrapper.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, src, **kwds)\u001b[0m\n\u001b[0;32m     49\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     50\u001b[0m         \u001b[1;31m# open handles\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 51\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_open_handles\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     52\u001b[0m         \u001b[1;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhandles\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     53\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\parsers\\base_parser.py\u001b[0m in \u001b[0;36m_open_handles\u001b[1;34m(self, src, kwds)\u001b[0m\n\u001b[0;32m    220\u001b[0m         \u001b[0mLet\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mreaders\u001b[0m \u001b[0mopen\u001b[0m \u001b[0mIOHandles\u001b[0m \u001b[0mafter\u001b[0m \u001b[0mthey\u001b[0m \u001b[0mare\u001b[0m \u001b[0mdone\u001b[0m \u001b[1;32mwith\u001b[0m \u001b[0mtheir\u001b[0m \u001b[0mpotential\u001b[0m \u001b[0mraises\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    221\u001b[0m         \"\"\"\n\u001b[1;32m--> 222\u001b[1;33m         self.handles = get_handle(\n\u001b[0m\u001b[0;32m    223\u001b[0m             \u001b[0msrc\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    224\u001b[0m             \u001b[1;34m\"r\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\common.py\u001b[0m in \u001b[0;36mget_handle\u001b[1;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[0;32m    699\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mioargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencoding\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;34m\"b\"\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mioargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    700\u001b[0m             \u001b[1;31m# Encoding\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 701\u001b[1;33m             handle = open(\n\u001b[0m\u001b[0;32m    702\u001b[0m                 \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    703\u001b[0m                 \u001b[0mioargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'scrapePalma_de_Mallorca2023-05-08v3.csv'"
     ]
    }
   ],
   "source": [
    "df1 = pd.read_csv('scrapeParis2023-05-08v3.csv')\n",
    "df2 = pd.read_csv('scrapeParis2023-05-15v3.csv')\n",
    "df3 = pd.read_csv('scrapeParis2023-05-22v3.csv')\n",
    "df4 = pd.read_csv('scrapeParis2023-05-29v3.csv')\n",
    "df5 = pd.read_csv('scrapeBarcelona2023-05-08v3.csv')\n",
    "df6 = pd.read_csv('scrapeBarcelona2023-05-15v3.csv')\n",
    "df7 = pd.read_csv('scrapeBarcelona2023-05-22v3.csv')\n",
    "df8 = pd.read_csv('scrapeBarcelona2023-05-29v3.csv')\n",
    "df9 = pd.read_csv('scrapeLisboa2023-05-08v3.csv')\n",
    "df10 = pd.read_csv('scrapeLisboa2023-05-15v3.csv')\n",
    "df11 = pd.read_csv('scrapeLisboa2023-05-22v3.csv')\n",
    "df12 = pd.read_csv('scrapeLisboa2023-05-29v3.csv')\n",
    "df13 = pd.read_csv('scrapePalma_de_Mallorca2023-05-08v3.csv')\n",
    "df14 = pd.read_csv('scrapePalma_de_Mallorca2023-05-15v3.csv')\n",
    "df15 = pd.read_csv('scrapePalma_de_Mallorca2023-05-22v3.csv')\n",
    "df16 = pd.read_csv('scrapePalma_de_Mallorca2023-05-29v3.csv')\n",
    "\n",
    "df1 = df1.append(df2)\n",
    "df1 = df1.append(df3)\n",
    "df1 = df1.append(df4)\n",
    "df1 = df1.append(df5)\n",
    "df1 = df1.append(df6)\n",
    "df1 = df1.append(df7)\n",
    "df1 = df1.append(df8)\n",
    "df1 = df1.append(df9)\n",
    "df1 = df1.append(df10)\n",
    "df1 = df1.append(df11)\n",
    "df1 = df1.append(df12)\n",
    "df1 = df1.append(df13)\n",
    "df1 = df1.append(df14)\n",
    "df1 = df1.append(df15)\n",
    "df1 = df1.append(df16)\n",
    "\n",
    "df1.to_csv('scrape_all_citiesv2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cef22d85",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "vscode": {
   "interpreter": {
    "hash": "866c4eef4daae6ad66333bb21c11470a23f6efb7eeb24158657848439181d3b4"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}