data_collection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib.request import urlopen\n",
    "from urllib import error\n",
    "from requests.exceptions import ConnectionError\n",
    "from urllib3.exceptions import NewConnectionError, MaxRetryError\n",
    "from bs4 import BeautifulSoup\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _66girls_info_crawler(product_url):\n",
    "    if product_url.startswith('https://m.') or product_url.startswith('m.'):\n",
    "        product_url = product_url.split('.')[1:]\n",
    "        new_product_url = ''\n",
    "        for i in range(len(product_url)):\n",
    "            new_product_url = new_product_url + product_url[i] + '.'\n",
    "        new_product_url = 'https://' + new_product_url\n",
    "    else:\n",
    "        new_product_url = product_url\n",
    "    try:\n",
    "        html = urlopen(new_product_url)\n",
    "        source = BeautifulSoup(html, 'html.parser')\n",
    "\n",
    "        # 이미지 detail source 정보 추출하기\n",
    "        url_list = []\n",
    "        detail_list = source.select('div.cont_detail')[0].select('img')\n",
    "        for i in range(len(detail_list)):\n",
    "            url_list.append('http://66girls.co.kr' + detail_list[i]['src'])\n",
    "\n",
    "        # 서버 과부하를 위해 2s 간 멈춤\n",
    "        time.sleep(2)\n",
    "    except (ConnectionResetError, error.URLError, error.HTTPError, ConnectionRefusedError,\n",
    "            ConnectionError, NewConnectionError, MaxRetryError):\n",
    "        print(\"Connection Error\")\n",
    "    return url_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(1).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(2).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(3).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(4).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(5).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(6).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(7).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(8).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(9).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(10).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(11).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(12).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(13).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(14).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(15).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(16).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(17).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(18).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(19).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(20).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(21).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(22).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(23).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(24).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(25).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(26).jpg\n",
      "http://66girls.co.kr/2021/1JAN/4/210119_sr_a_01%20(27).jpg\n"
     ]
    }
   ],
   "source": [
    "from PIL import Image\n",
    "url_list = _66girls_info_crawler('https://66girls.co.kr/product/detail.html?product_no=110948&cate_no=70&display_group=1')\n",
    "for i in range(len(url_list)):\n",
    "    url = url_list[i].replace(\" \", \"%20\")\n",
    "    print(url)\n",
    "    try:\n",
    "        im = Image.open(urlopen(url))\n",
    "        im.save('data/data80/test_{}.jpg'.format(i))\n",
    "    except:\n",
    "        print(url_list[i])\n",
    "        print(\"error\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}