Skip to content

[lab-web-scraping-multipages] Susumu #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
susumusakamoto wants to merge 2 commits into
base: main
Choose a base branch
from
Open

[lab-web-scraping-multipages] Susumu #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
44900a6
initial commit
susumusakamoto Oct 9, 2021
68865c0
Finish all questions
susumusakamoto Oct 9, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
266 changes: 255 additions & 11 deletions 01_Billboard.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 2,
"id": "promotional-algorithm",
"metadata": {},
"outputs": [],
Expand All @@ -59,11 +59,24 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 4,
"id": "civic-broad",
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"200"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"response = requests.get(url)\n",
"response.status_code\n",
"# 200 status code means OK!"
]
},
Expand All @@ -77,11 +90,13 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 5,
"id": "revised-digest",
"metadata": {},
"outputs": [],
"source": []
"source": [
"soup = BeautifulSoup(response.content, \"html.parser\")"
]
},
{
"cell_type": "markdown",
Expand All @@ -93,20 +108,28 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 22,
"id": "falling-chambers",
"metadata": {},
"outputs": [],
"source": [
"# song titles\n",
"songs = soup.find_all(\"span\", class_=\"chart-element__information__song text--truncate color--primary\")\n",
"\n",
"# artists\n",
"artists = soup.find_all(\"span\", class_=\"chart-element__information__artist text--truncate color--secondary\")\n",
"\n",
"# this week\n",
"this_week_ranks = soup.find_all(\"span\", class_=\"chart-element__rank__number\")\n",
"\n",
"# last week\n",
"last_week_ranks = soup.find_all(\"span\", class_= \"chart-element__meta text--center color--secondary text--last\")\n",
"\n",
"# peak rank\n",
"peak_ranks = soup.find_all(\"span\", class_= \"chart-element__meta text--center color--secondary text--peak\")\n",
"\n",
"# weeks on chart\n"
"# weeks on chart\n",
"weeks_on_chart = soup.find_all(\"span\", class_= \"chart-element__meta text--center color--secondary text--week\")"
]
},
{
Expand All @@ -119,11 +142,36 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 24,
"id": "020cc006",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'Industry Baby'"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"songs[2].getText()"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "amateur-protocol",
"metadata": {},
"outputs": [],
"source": []
"source": [
"for i in [songs, artists, this_week_ranks, last_week_ranks, peak_ranks, weeks_on_chart]:\n",
" for j in range(len(i)):\n",
" i[j] = i[j].getText()"
]
},
{
"cell_type": "markdown",
Expand All @@ -135,10 +183,206 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 18,
"id": "external-instrumentation",
"metadata": {},
"outputs": [],
"source": [
"billboard = pd.DataFrame(\n",
" {\"song\": songs,\n",
" \"artist\": artists,\n",
" \"current_rank\": this_week_ranks,\n",
" \"last_week_rank\": last_week_ranks,\n",
" \"peak_rank\": peak_ranks,\n",
" \"weeks_on_chart\": weeks_on_chart}\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "7755df6d",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>song</th>\n",
" <th>artist</th>\n",
" <th>current_rank</th>\n",
" <th>last_week_rank</th>\n",
" <th>peak_rank</th>\n",
" <th>weeks_on_chart</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>My Universe</td>\n",
" <td>Coldplay x BTS</td>\n",
" <td>1</td>\n",
" <td>-</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Stay</td>\n",
" <td>The Kid LAROI &amp; Justin Bieber</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>12</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Industry Baby</td>\n",
" <td>Lil Nas X &amp; Jack Harlow</td>\n",
" <td>3</td>\n",
" <td>2</td>\n",
" <td>2</td>\n",
" <td>10</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Way 2 Sexy</td>\n",
" <td>Drake Featuring Future &amp; Young Thug</td>\n",
" <td>4</td>\n",
" <td>3</td>\n",
" <td>1</td>\n",
" <td>4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Fancy Like</td>\n",
" <td>Walker Hayes</td>\n",
" <td>5</td>\n",
" <td>5</td>\n",
" <td>5</td>\n",
" <td>15</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>95</th>\n",
" <td>Pipe Down</td>\n",
" <td>Drake</td>\n",
" <td>96</td>\n",
" <td>68</td>\n",
" <td>14</td>\n",
" <td>4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>96</th>\n",
" <td>Papi's Home</td>\n",
" <td>Drake</td>\n",
" <td>97</td>\n",
" <td>66</td>\n",
" <td>8</td>\n",
" <td>4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>97</th>\n",
" <td>Chosen</td>\n",
" <td>Blxst &amp; Tyga Featuring Ty Dolla $ign</td>\n",
" <td>98</td>\n",
" <td>-</td>\n",
" <td>98</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>98</th>\n",
" <td>Toxic Punk</td>\n",
" <td>YoungBoy Never Broke Again</td>\n",
" <td>99</td>\n",
" <td>-</td>\n",
" <td>99</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>99</th>\n",
" <td>Moon</td>\n",
" <td>Kanye West</td>\n",
" <td>100</td>\n",
" <td>76</td>\n",
" <td>17</td>\n",
" <td>5</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>100 rows × 6 columns</p>\n",
"</div>"
],
"text/plain": [
" song artist current_rank \\\n",
"0 My Universe Coldplay x BTS 1 \n",
"1 Stay The Kid LAROI & Justin Bieber 2 \n",
"2 Industry Baby Lil Nas X & Jack Harlow 3 \n",
"3 Way 2 Sexy Drake Featuring Future & Young Thug 4 \n",
"4 Fancy Like Walker Hayes 5 \n",
".. ... ... ... \n",
"95 Pipe Down Drake 96 \n",
"96 Papi's Home Drake 97 \n",
"97 Chosen Blxst & Tyga Featuring Ty Dolla $ign 98 \n",
"98 Toxic Punk YoungBoy Never Broke Again 99 \n",
"99 Moon Kanye West 100 \n",
"\n",
" last_week_rank peak_rank weeks_on_chart \n",
"0 - 1 1 \n",
"1 1 1 12 \n",
"2 2 2 10 \n",
"3 3 1 4 \n",
"4 5 5 15 \n",
".. ... ... ... \n",
"95 68 14 4 \n",
"96 66 8 4 \n",
"97 - 98 1 \n",
"98 - 99 1 \n",
"99 76 17 5 \n",
"\n",
"[100 rows x 6 columns]"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"billboard"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1bdcea94",
"metadata": {},
"outputs": [],
"source": []
}
],
Expand All @@ -158,7 +402,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
"version": "3.8.8"
},
"toc": {
"base_numbering": 1,
Expand Down
Loading