8_web scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: bs4 in c:\\users\\raj krishna mondal\\anaconda3\\lib\\site-packages (0.0.1)\n",
      "Requirement already satisfied: beautifulsoup4 in c:\\users\\raj krishna mondal\\anaconda3\\lib\\site-packages (from bs4) (4.8.2)\n",
      "Requirement already satisfied: soupsieve>=1.2 in c:\\users\\raj krishna mondal\\anaconda3\\lib\\site-packages (from beautifulsoup4->bs4) (1.9.5)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR: Could not find a version that satisfies the requirement urllib (from versions: none)\n",
      "ERROR: No matching distribution found for urllib\n"
     ]
    }
   ],
   "source": [
    "!pip install bs4\n",
    "!pip install urllib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup as soup\n",
    "from urllib.request import urlopen as uReq\n",
    "\n",
    "my_url=\"https://www.flipkart.com/search?q=laptop&sid=6bo%2Cb5g&as=on&as-show=on&otracker=AS_QueryStore_OrganicAutoSuggest_1_6_na_na_na&otracker1=AS_QueryStore_OrganicAutoSuggest_1_6_na_na_na&as-pos=1&as-type=RECENT&suggestionId=laptop%7CLaptops&requestId=03fb4dd6-281e-4278-97f4-c74c25dc8d22&as-backfill=on\"\n",
    "\n",
    "uclient=uReq(my_url)\n",
    "\n",
    "page_html=uclient.read()\n",
    "uclient.close()\n",
    "\n",
    "page_soup=soup(page_html, \"html.parser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24\n"
     ]
    }
   ],
   "source": [
    "containers=page_soup.find_all(\"div\", {\"class\": \"_2kHMtA\"})\n",
    "print(len(containers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<div class=\"_2kHMtA\">\n",
      " <a class=\"_1fQZEK\" href=\"/dell-inspiron-pentium-quad-core-4-gb-256-gb-ssd-windows-10-home-3502-laptop/p/itm69203f870345e?pid=COMGYCG8G3VFVBPA&amp;lid=LSTCOMGYCG8G3VFVBPA8J3QCX&amp;marketplace=FLIPKART&amp;q=laptop&amp;store=6bo%2Fb5g&amp;srno=s_1_1&amp;otracker=AS_QueryStore_OrganicAutoSuggest_1_6_na_na_na&amp;otracker1=AS_QueryStore_OrganicAutoSuggest_1_6_na_na_na&amp;fm=organic&amp;iid=24890071-eeb5-444a-b857-7ceb5044da0b.COMGYCG8G3VFVBPA.SEARCH&amp;ppt=None&amp;ppn=None&amp;ssid=q2vt7rpk7k0000001627830960435&amp;qH=312f91285e048e09\" rel=\"noopener noreferrer\" target=\"_blank\">\n",
      "  <div class=\"MIXNux\">\n",
      "   <div class=\"_2QcLo-\">\n",
      "    <div>\n",
      "     <div class=\"CXW8mj\" style=\"height:200px;width:200px\">\n",
      "      <img alt=\"DELL Inspiron Pentium Quad Core - (4 GB/256 GB SSD/Windows 10 Home) Inspiron 3502 Laptop\" class=\"_396cs4 _3exPp9\" src=\"https://rukminim1.flixcart.com/image/312/312/kl5hh8w0/computer/n/q/i/dell-original-imagychchm2h6tsz.jpeg?q=70\"/>\n",
      "     </div>\n",
      "    </div>\n",
      "   </div>\n",
      "   <div class=\"_3wLduG\">\n",
      "    <div class=\"_3PzNI-\">\n",
      "     <span class=\"f3A4_V\">\n",
      "      <label class=\"_2iDkf8\">\n",
      "       <input class=\"_30VH1S\" readonly=\"\" type=\"checkbox\"/>\n",
      "       <div class=\"_24_Dny\">\n",
      "       </div>\n",
      "      </label>\n",
      "     </span>\n",
      "     <label class=\"_6Up2sF\">\n",
      "      <span>\n",
      "       Add to Compare\n",
      "      </span>\n",
      "     </label>\n",
      "    </div>\n",
      "   </div>\n",
      "   <div class=\"_2hVSre _3nq8ih\">\n",
      "    <div class=\"_36FSn5\">\n",
      "     <svg class=\"_1l0elc\" height=\"16\" viewbox=\"0 0 20 16\" width=\"16\" xmlns=\"http://www.w3.org/2000/svg\">\n",
      "      <path class=\"eX72wL\" d=\"M8.695 16.682C4.06 12.382 1 9.536 1 6.065 1 3.219 3.178 1 5.95 1c1.566 0 3.069.746 4.05 1.915C10.981 1.745 12.484 1 14.05 1 16.822 1 19 3.22 19 6.065c0 3.471-3.06 6.316-7.695 10.617L10 17.897l-1.305-1.215z\" fill=\"#2874F0\" fill-rule=\"evenodd\" opacity=\".9\" stroke=\"#FFF\">\n",
      "      </path>\n",
      "     </svg>\n",
      "    </div>\n",
      "   </div>\n",
      "  </div>\n",
      "  <div class=\"_3pLy-c row\">\n",
      "   <div class=\"col col-7-12\">\n",
      "    <div class=\"_4rR01T\">\n",
      "     DELL Inspiron Pentium Quad Core - (4 GB/256 GB SSD/Windows 10 Home) Inspiron 3502 Laptop\n",
      "    </div>\n",
      "    <div class=\"gUuXy-\">\n",
      "     <span class=\"_1lRcqv\" id=\"productRating_LSTCOMGYCG8G3VFVBPA8J3QCX_COMGYCG8G3VFVBPA_\">\n",
      "      <div class=\"_3LWZlK\">\n",
      "       4.4\n",
      "       <img class=\"_1wB99o\" src=\"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIxMyIgaGVpZ2h0PSIxMiI+PHBhdGggZmlsbD0iI0ZGRiIgZD0iTTYuNSA5LjQzOWwtMy42NzQgMi4yMy45NC00LjI2LTMuMjEtMi44ODMgNC4yNTQtLjQwNEw2LjUuMTEybDEuNjkgNC4wMSA0LjI1NC40MDQtMy4yMSAyLjg4Mi45NCA0LjI2eiIvPjwvc3ZnPg==\"/>\n",
      "      </div>\n",
      "     </span>\n",
      "     <span class=\"_2_R_DZ\">\n",
      "      <span>\n",
      "       <span>\n",
      "        282 Ratings\n",
      "       </span>\n",
      "       <span class=\"_13vcmD\">\n",
      "        &amp;\n",
      "       </span>\n",
      "       <span>\n",
      "        44 Reviews\n",
      "       </span>\n",
      "      </span>\n",
      "     </span>\n",
      "    </div>\n",
      "    <div class=\"fMghEO\">\n",
      "     <ul class=\"_1xgFaf\">\n",
      "      <li class=\"rgWa7D\">\n",
      "       Intel Pentium Quad Core Processor\n",
      "      </li>\n",
      "      <li class=\"rgWa7D\">\n",
      "       4 GB DDR4 RAM\n",
      "      </li>\n",
      "      <li class=\"rgWa7D\">\n",
      "       64 bit Windows 10 Operating System\n",
      "      </li>\n",
      "      <li class=\"rgWa7D\">\n",
      "       256 GB SSD\n",
      "      </li>\n",
      "      <li class=\"rgWa7D\">\n",
      "       39.62 cm (15.6 inch) Display\n",
      "      </li>\n",
      "      <li class=\"rgWa7D\">\n",
      "       Microsoft Office Home and Student 2019\n",
      "      </li>\n",
      "      <li class=\"rgWa7D\">\n",
      "       1 Year Limited Hardware Warranty, In Home Service After Remote Diagnosis - Retail\n",
      "      </li>\n",
      "     </ul>\n",
      "    </div>\n",
      "   </div>\n",
      "   <div class=\"col col-5-12 nlI3QM\">\n",
      "    <div class=\"_3tbKJL\">\n",
      "     <div class=\"_25b18c\">\n",
      "      <div class=\"_30jeq3 _1_WHN1\">\n",
      "       ₹28,590\n",
      "      </div>\n",
      "      <div class=\"_3I9_wc _27UcVY\">\n",
      "       ₹\n",
      "       <!-- -->\n",
      "       31,846\n",
      "      </div>\n",
      "      <div class=\"_3Ay6Sb\">\n",
      "       <span>\n",
      "        10% off\n",
      "       </span>\n",
      "      </div>\n",
      "     </div>\n",
      "    </div>\n",
      "    <div class=\"_13J9qT\">\n",
      "     <img height=\"21\" src=\"//static-assets-web.flixcart.com/www/linchpin/fk-cp-zion/img/fa_62673a.png\"/>\n",
      "    </div>\n",
      "   </div>\n",
      "  </div>\n",
      " </a>\n",
      "</div>\n"
     ]
    }
   ],
   "source": [
    "print(soup.prettify(containers[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DELL Inspiron Pentium Quad Core - (4 GB/256 GB SSD/Windows 10 Home) Inspiron 3502 Laptop\n"
     ]
    }
   ],
   "source": [
    "cont=containers[0]\n",
    "print(cont.div.img[\"alt\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "₹28,590\n"
     ]
    }
   ],
   "source": [
    "price=cont.find_all(\"div\", {\"class\": \"_30jeq3 _1_WHN1\"})\n",
    "print(price[0].text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.4282 Ratings & 44 Reviews\n"
     ]
    }
   ],
   "source": [
    "rating=cont.find_all(\"div\", {\"class\": \"gUuXy-\"})\n",
    "print(rating[0].text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Intel Pentium Quad Core Processor4 GB DDR4 RAM64 bit Windows 10 Operating System256 GB SSD39.62 cm (15.6 inch) DisplayMicrosoft Office Home and Student 20191 Year Limited Hardware Warranty, In Home Service After Remote Diagnosis - Retail\n"
     ]
    }
   ],
   "source": [
    "info=cont.find_all(\"div\", {\"class\": \"fMghEO\"})\n",
    "print(info[0].text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file=\"product.csv\"\n",
    "f=open(file,\"w\")\n",
    "header=\"name, price, rating, information \\n\"\n",
    "f.write(header)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name:DELL Inspiron Pentium Quad Core - (4 GB/256 GB SSD/Windows 10 Home) Inspiron 3502 Laptop\n",
      "price:₹28,590\n",
      "rating:4.4282 Ratings & 44 Reviews\n",
      "Information:Intel Pentium Quad Core Processor4 GB DDR4 RAM64 bit Windows 10 Operating System256 GB SSD39.62 cm (15.6 inch) DisplayMicrosoft Office Home and Student 20191 Year Limited Hardware Warranty, In Home Service After Remote Diagnosis - Retail\n",
      "Name:HP 15s Ryzen 3 Dual Core 3250U - (8 GB/1 TB HDD/256 GB SSD/Windows 10 Home) 15s-GR0012AU Laptop\n",
      "price:₹42,990\n",
      "rating:4.41,733 Ratings & 235 Reviews\n",
      "Information:AMD Ryzen 3 Dual Core Processor8 GB DDR4 RAM64 bit Windows 10 Operating System1 TB HDD|256 GB SSD39.62 cm (15.6 inch) DisplayMicrosoft Office Home and Student 2019, HP Documentation, HP SSRM, HP Smart, HP Support Assistant1 Year Onsite Warranty\n",
      "Name:HP 15s Ryzen 3 Dual Core 3250U - (8 GB/1 TB HDD/Windows 10 Home) 15s-GR0011AU Thin and Light Laptop\n",
      "price:₹37,490\n",
      "rating:4.23,831 Ratings & 480 Reviews\n",
      "Information:AMD Ryzen 3 Dual Core Processor8 GB DDR4 RAM64 bit Windows 10 Operating System1 TB HDD39.62 cm (15.6 inch) DisplayMicrosoft Office Home and Student 2019, HP Documentation, HP SSRM, HP Smart, HP Support Assistant1 Year Onsite Warranty\n",
      "Name:HP 15s Core i3 10th Gen - (8 GB/1 TB HDD/256 GB SSD/Windows 10 Home) 15s-du1064TU Laptop\n",
      "price:₹44,490\n",
      "rating:4.31,031 Ratings & 125 Reviews\n",
      "Information:Intel Core i3 Processor (10th Gen)8 GB DDR4 RAM64 bit Windows 10 Operating System1 TB HDD|256 GB SSD39.62 cm (15.6 inch) DisplayMicrosoft Office Home & Student 2019, HP Documentation, HP BIOS recovery, HP Smart1 Year Onsite Warranty\n",
      "Name:ASUS VivoBook Ultra 14 Core i3 11th Gen - (8 GB/512 GB SSD/Windows 10 Home) X413EA-EB322TS Thin and Li...\n",
      "price:₹45,990\n",
      "rating:4.41,032 Ratings & 187 Reviews\n",
      "Information:Intel Core i3 Processor (11th Gen)8 GB DDR4 RAM64 bit Windows 10 Operating System512 GB SSD35.56 cm (14 inch) DisplayMicrosoft Office Home and Student 2019, Tru2Life, Splendid, My Asus1 Year Onsite Warranty\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "'NoneType' object is not subscriptable",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-36-dbac37fe838a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mcont\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mcontainers\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m     \u001b[0mname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcont\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdiv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mimg\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"alt\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[0mprice_cont\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcont\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_all\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"div\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;34m\"class\"\u001b[0m\u001b[1;33m:\u001b[0m \u001b[1;34m\"_30jeq3 _1_WHN1\"\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[0mprice\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mprice_cont\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: 'NoneType' object is not subscriptable"
     ]
    }
   ],
   "source": [
    "for cont in containers:\n",
    "    name=cont.div.img[\"alt\"]\n",
    "    \n",
    "    price_cont=cont.find_all(\"div\", {\"class\": \"_30jeq3 _1_WHN1\"})\n",
    "    price=price_cont[0].text\n",
    "    \n",
    "    rating_cont=cont.find_all(\"div\", {\"class\": \"gUuXy-\"})\n",
    "    rating=rating_cont[0].text\n",
    "    \n",
    "    info_cont=cont.find_all(\"div\", {\"class\": \"fMghEO\"})\n",
    "    info=info_cont[0].text\n",
    "    \n",
    "    print(\"Name:\" + name)\n",
    "    print(\"price:\" + price)\n",
    "    print(\"rating:\" + rating)\n",
    "    print(\"Information:\" + info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}