diff --git a/[lab-web-scraping-multipages (01_Billboard)] ignacio.ipynb b/[lab-web-scraping-multipages (01_Billboard)] ignacio.ipynb new file mode 100644 index 0000000..c4cd73d --- /dev/null +++ b/[lab-web-scraping-multipages (01_Billboard)] ignacio.ipynb @@ -0,0 +1,2816 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "adequate-aurora", + "metadata": { + "toc": true + }, + "source": [ + "

Table of Contents

\n", + "
" + ] + }, + { + "cell_type": "markdown", + "id": "together-terminology", + "metadata": {}, + "source": [ + "## Import libraries" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "little-prophet", + "metadata": {}, + "outputs": [], + "source": [ + "from bs4 import BeautifulSoup\n", + "import requests\n", + "import pandas as pd" + ] + }, + { + "cell_type": "markdown", + "id": "political-wright", + "metadata": {}, + "source": [ + "## Store the hot-100 songs list from billboard url in a variable" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "promotional-algorithm", + "metadata": {}, + "outputs": [], + "source": [ + "url = \"https://www.billboard.com/charts/hot-100\"" + ] + }, + { + "cell_type": "markdown", + "id": "indian-fossil", + "metadata": {}, + "source": [ + "## Download html with a get request" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "civic-broad", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "r = requests.get(url)\n", + "r.status_code" + ] + }, + { + "cell_type": "markdown", + "id": "arbitrary-patrick", + "metadata": {}, + "source": [ + "## Parse html (create the 'soup')" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "revised-digest", + "metadata": {}, + "outputs": [], + "source": [ + "soup = BeautifulSoup(r.content, 'html.parser')" + ] + }, + { + "cell_type": "markdown", + "id": "damaged-watts", + "metadata": {}, + "source": [ + "## Retrieve/extract the desired info" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "c6c4f70d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ,\n", + "
  • \n", + " \n", + "
  • ]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "songs = soup.find_all('li', attrs={'class': 'chart-list__element display--flex'})\n", + "songs" + ] + }, + { + "cell_type": "markdown", + "id": "employed-chemistry", + "metadata": {}, + "source": [ + "## Get the text" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "amateur-protocol", + "metadata": {}, + "outputs": [], + "source": [ + "ranking = []\n", + "title = []\n", + "artists = []\n", + "last_week = []\n", + "peak_position = []\n", + "weeks_on_chart = []\n", + "\n", + "for i,songs in enumerate(songs):\n", + " ranking.append(songs.find('span', attrs={'class': 'chart-element__rank__number'}).get_text(strip=True))\n", + " title.append(songs.find('span', attrs={'class': 'chart-element__information__song text--truncate color--primary'}).get_text(strip=True))\n", + " artists.append(songs.find('span', attrs={'class': 'chart-element__information__artist text--truncate color--secondary'}).get_text(strip=True))\n", + " last_week.append(songs.find('span', attrs={'class': 'chart-element__information__delta__text text--last'}).get_text(strip=True).split(\" \", 1)[0])\n", + " peak_position.append(songs.find('span', attrs={'class': 'chart-element__information__delta__text text--peak'}).get_text(strip=True).split(\" \", 1)[0])\n", + " weeks_on_chart.append(songs.find('span', attrs={'class': 'chart-element__information__delta__text text--week'}).get_text(strip=True).split(\" \", 1)[0])" + ] + }, + { + "cell_type": "markdown", + "id": "every-degree", + "metadata": {}, + "source": [ + "## Build a dataframe" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "external-instrumentation", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    rankingtitleartistslast_weekpeak_positionweeks_on_chart
    01My UniverseColdplay x BTS-11
    12StayThe Kid LAROI & Justin Bieber1112
    23Industry BabyLil Nas X & Jack Harlow2210
    34Way 2 SexyDrake Featuring Future & Young Thug314
    45Fancy LikeWalker Hayes5515
    .....................
    9596Pipe DownDrake68144
    9697Papi's HomeDrake6684
    9798ChosenBlxst & Tyga Featuring Ty Dolla $ign-981
    9899Toxic PunkYoungBoy Never Broke Again-991
    99100MoonKanye West76175
    \n", + "

    100 rows × 6 columns

    \n", + "
    " + ], + "text/plain": [ + " ranking title artists last_week \\\n", + "0 1 My Universe Coldplay x BTS - \n", + "1 2 Stay The Kid LAROI & Justin Bieber 1 \n", + "2 3 Industry Baby Lil Nas X & Jack Harlow 2 \n", + "3 4 Way 2 Sexy Drake Featuring Future & Young Thug 3 \n", + "4 5 Fancy Like Walker Hayes 5 \n", + ".. ... ... ... ... \n", + "95 96 Pipe Down Drake 68 \n", + "96 97 Papi's Home Drake 66 \n", + "97 98 Chosen Blxst & Tyga Featuring Ty Dolla $ign - \n", + "98 99 Toxic Punk YoungBoy Never Broke Again - \n", + "99 100 Moon Kanye West 76 \n", + "\n", + " peak_position weeks_on_chart \n", + "0 1 1 \n", + "1 1 12 \n", + "2 2 10 \n", + "3 1 4 \n", + "4 5 15 \n", + ".. ... ... \n", + "95 14 4 \n", + "96 8 4 \n", + "97 98 1 \n", + "98 99 1 \n", + "99 17 5 \n", + "\n", + "[100 rows x 6 columns]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x = min(len(ranking), len(title), len(artists), len(last_week), len(peak_position), len(weeks_on_chart))\n", + "\n", + "dct = {'ranking': ranking[:x], 'title': title[:x], 'artists': artists[:x], 'last_week': last_week[:x], 'peak_position': peak_position[:x], 'weeks_on_chart': weeks_on_chart[:x]}\n", + "\n", + "df = pd.DataFrame.from_dict(dct)\n", + "df" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ironhack_1", + "language": "python", + "name": "ironhack_1" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": true, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + }, + "varInspector": { + "cols": { + "lenName": 16, + "lenType": 16, + "lenVar": 40 + }, + "kernels_config": { + "python": { + "delete_cmd_postfix": "", + "delete_cmd_prefix": "del ", + "library": "var_list.py", + "varRefreshCmd": "print(var_dic_list())" + }, + "r": { + "delete_cmd_postfix": ") ", + "delete_cmd_prefix": "rm(", + "library": "var_list.r", + "varRefreshCmd": "cat(var_dic_list()) " + } + }, + "types_to_exclude": [ + "module", + "function", + "builtin_function_or_method", + "instance", + "_Feature" + ], + "window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}