From ceca81f69a0afbe96ef51798b382ffa667021e63 Mon Sep 17 00:00:00 2001 From: KaroD123 Date: Sat, 9 Oct 2021 14:20:11 +0200 Subject: [PATCH] save --- 01_Billboard.ipynb | 238 +++++++++++++++--- 02_Further_questions.ipynb | 477 +++++++++++++++++++++++++++++++++++-- 2 files changed, 661 insertions(+), 54 deletions(-) diff --git a/01_Billboard.ipynb b/01_Billboard.ipynb index 779a38c..e646d4a 100644 --- a/01_Billboard.ipynb +++ b/01_Billboard.ipynb @@ -2,7 +2,6 @@ "cells": [ { "cell_type": "markdown", - "id": "adequate-aurora", "metadata": { "toc": true }, @@ -13,7 +12,6 @@ }, { "cell_type": "markdown", - "id": "together-terminology", "metadata": {}, "source": [ "## Import libraries" @@ -22,7 +20,6 @@ { "cell_type": "code", "execution_count": 1, - "id": "little-prophet", "metadata": {}, "outputs": [], "source": [ @@ -33,7 +30,6 @@ }, { "cell_type": "markdown", - "id": "political-wright", "metadata": {}, "source": [ "## Store the hot-100 songs list from billboard url in a variable" @@ -41,8 +37,7 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "promotional-algorithm", + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -51,7 +46,6 @@ }, { "cell_type": "markdown", - "id": "indian-fossil", "metadata": {}, "source": [ "## Download html with a get request" @@ -59,17 +53,29 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "civic-broad", + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "# 200 status code means OK!" + "# 200 status code means OK!\n", + "\n", + "response = requests.get(url)\n", + "response.status_code" ] }, { "cell_type": "markdown", - "id": "arbitrary-patrick", "metadata": {}, "source": [ "## Parse html (create the 'soup')" @@ -77,15 +83,15 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "revised-digest", + "execution_count": 4, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "soup = BeautifulSoup(response.content, \"html.parser\")" + ] }, { "cell_type": "markdown", - "id": "damaged-watts", "metadata": {}, "source": [ "## Retrieve/extract the desired info" @@ -93,25 +99,24 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "falling-chambers", + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "# song titles\n", - "\n", + "songs = soup.find_all(\"span\", class_=\"chart-element__information__song text--truncate color--primary\")\n", "# artists\n", - "\n", + "artists = soup.find_all(\"span\", class_=\"chart-element__information__artist text--truncate color--secondary\")\n", "# last week\n", - "\n", + "last_week_ranks = soup.find_all(\"span\", class_= \"chart-element__meta text--center color--secondary text--last\")\n", "# peak rank\n", - "\n", - "# weeks on chart\n" + "peak_ranks = soup.find_all(\"span\", class_= \"chart-element__meta text--center color--secondary text--peak\")\n", + "# weeks on chart\n", + "weeks_on_chart = soup.find_all(\"span\", class_= \"chart-element__meta text--center color--secondary text--week\")\n" ] }, { "cell_type": "markdown", - "id": "employed-chemistry", "metadata": {}, "source": [ "## Get the text" @@ -119,24 +124,199 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "amateur-protocol", + "execution_count": 6, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "for i in [songs, artists, last_week_ranks, peak_ranks, weeks_on_chart]:\n", + " for j in range(len(i)):\n", + " i[j] = i[j].getText()" + ] }, { "cell_type": "markdown", - "id": "every-degree", "metadata": {}, "source": [ "## Build a dataframe" ] }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
songartistlast_week_rankpeak_rankweeks_on_chart
0My UniverseColdplay x BTS-11
1StayThe Kid LAROI & Justin Bieber1112
2Industry BabyLil Nas X & Jack Harlow2210
3Way 2 SexyDrake Featuring Future & Young Thug314
4Fancy LikeWalker Hayes5515
..................
95Pipe DownDrake68144
96Papi's HomeDrake6684
97ChosenBlxst & Tyga Featuring Ty Dolla $ign-981
98Toxic PunkYoungBoy Never Broke Again-991
99MoonKanye West76175
\n", + "

100 rows × 5 columns

\n", + "
" + ], + "text/plain": [ + " song artist last_week_rank \\\n", + "0 My Universe Coldplay x BTS - \n", + "1 Stay The Kid LAROI & Justin Bieber 1 \n", + "2 Industry Baby Lil Nas X & Jack Harlow 2 \n", + "3 Way 2 Sexy Drake Featuring Future & Young Thug 3 \n", + "4 Fancy Like Walker Hayes 5 \n", + ".. ... ... ... \n", + "95 Pipe Down Drake 68 \n", + "96 Papi's Home Drake 66 \n", + "97 Chosen Blxst & Tyga Featuring Ty Dolla $ign - \n", + "98 Toxic Punk YoungBoy Never Broke Again - \n", + "99 Moon Kanye West 76 \n", + "\n", + " peak_rank weeks_on_chart \n", + "0 1 1 \n", + "1 1 12 \n", + "2 2 10 \n", + "3 1 4 \n", + "4 5 15 \n", + ".. ... ... \n", + "95 14 4 \n", + "96 8 4 \n", + "97 98 1 \n", + "98 99 1 \n", + "99 17 5 \n", + "\n", + "[100 rows x 5 columns]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "billboard = pd.DataFrame(\n", + " {\"song\": songs,\n", + " \"artist\": artists,\n", + " \"last_week_rank\": last_week_ranks,\n", + " \"peak_rank\": peak_ranks,\n", + " \"weeks_on_chart\": weeks_on_chart}\n", + ")\n", + "\n", + "billboard" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "external-instrumentation", "metadata": {}, "outputs": [], "source": [] diff --git a/02_Further_questions.ipynb b/02_Further_questions.ipynb index 66fb224..792190f 100644 --- a/02_Further_questions.ipynb +++ b/02_Further_questions.ipynb @@ -2,7 +2,6 @@ "cells": [ { "cell_type": "markdown", - "id": "classified-start", "metadata": { "toc": true }, @@ -13,15 +12,24 @@ }, { "cell_type": "markdown", - "id": "cellular-poster", "metadata": {}, "source": [ "As you've seen, scraping the internet is a skill that can get you all sorts of information. Here are some little challenges to gain more experience in the field" ] }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from bs4 import BeautifulSoup\n", + "import requests\n", + "import pandas as pd" + ] + }, { "cell_type": "markdown", - "id": "express-introduction", "metadata": {}, "source": [ "### Retrieve an arbitrary Wikipedia page of \"Python\" and create a list of links on that page" @@ -29,17 +37,78 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "appreciated-bubble", + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "url ='https://en.wikipedia.org/wiki/Python'" ] }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response = requests.get(url)\n", + "response.status_code" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "soup = BeautifulSoup(response.content, \"html.parser\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "links = []\n", + "\n", + "def scraper_links(soup,links_list):\n", + " for link in soup.find_all(\"a\"):\n", + " url = link.get(\"href\", \"\")\n", + " if \"https\" in url:\n", + " links_list.append(url)\n", + " print(links_list)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['https://en.wiktionary.org/wiki/Python', 'https://en.wiktionary.org/wiki/python', 'https://en.wikipedia.org/w/index.php?title=Special:WhatLinksHere/Python&namespace=0', 'https://en.wikipedia.org/w/index.php?title=Python&oldid=1048703433', 'https://donate.wikimedia.org/wiki/Special:FundraiserRedirector?utm_source=donate&utm_medium=sidebar&utm_campaign=C13_en.wikipedia.org&uselang=en', 'https://www.wikidata.org/wiki/Special:EntityPage/Q747452', 'https://commons.wikimedia.org/wiki/Category:Python', 'https://af.wikipedia.org/wiki/Python', 'https://als.wikipedia.org/wiki/Python', 'https://ar.wikipedia.org/wiki/%D8%A8%D8%A7%D9%8A%D8%AB%D9%88%D9%86_(%D8%AA%D9%88%D8%B6%D9%8A%D8%AD)', 'https://az.wikipedia.org/wiki/Python', 'https://bn.wikipedia.org/wiki/%E0%A6%AA%E0%A6%BE%E0%A6%87%E0%A6%A5%E0%A6%A8_(%E0%A6%A6%E0%A7%8D%E0%A6%AC%E0%A7%8D%E0%A6%AF%E0%A6%B0%E0%A7%8D%E0%A6%A5%E0%A6%A4%E0%A6%BE_%E0%A6%A8%E0%A6%BF%E0%A6%B0%E0%A6%B8%E0%A6%A8)', 'https://be.wikipedia.org/wiki/Python', 'https://bg.wikipedia.org/wiki/%D0%9F%D0%B8%D1%82%D0%BE%D0%BD_(%D0%BF%D0%BE%D1%8F%D1%81%D0%BD%D0%B5%D0%BD%D0%B8%D0%B5)', 'https://cs.wikipedia.org/wiki/Python_(rozcestn%C3%ADk)', 'https://da.wikipedia.org/wiki/Python', 'https://de.wikipedia.org/wiki/Python', 'https://eo.wikipedia.org/wiki/Pitono_(apartigilo)', 'https://eu.wikipedia.org/wiki/Python_(argipena)', 'https://fa.wikipedia.org/wiki/%D9%BE%D8%A7%DB%8C%D8%AA%D9%88%D9%86', 'https://fr.wikipedia.org/wiki/Python', 'https://ko.wikipedia.org/wiki/%ED%8C%8C%EC%9D%B4%EC%84%A0', 'https://hr.wikipedia.org/wiki/Python_(razdvojba)', 'https://io.wikipedia.org/wiki/Pitono', 'https://id.wikipedia.org/wiki/Python', 'https://ia.wikipedia.org/wiki/Python_(disambiguation)', 'https://is.wikipedia.org/wiki/Python_(a%C3%B0greining)', 'https://it.wikipedia.org/wiki/Python_(disambigua)', 'https://he.wikipedia.org/wiki/%D7%A4%D7%99%D7%AA%D7%95%D7%9F', 'https://ka.wikipedia.org/wiki/%E1%83%9E%E1%83%98%E1%83%97%E1%83%9D%E1%83%9C%E1%83%98_(%E1%83%9B%E1%83%A0%E1%83%90%E1%83%95%E1%83%90%E1%83%9A%E1%83%9B%E1%83%9C%E1%83%98%E1%83%A8%E1%83%95%E1%83%9C%E1%83%94%E1%83%9A%E1%83%9D%E1%83%95%E1%83%90%E1%83%9C%E1%83%98)', 'https://kg.wikipedia.org/wiki/Mboma_(nyoka)', 'https://la.wikipedia.org/wiki/Python_(discretiva)', 'https://lb.wikipedia.org/wiki/Python', 'https://hu.wikipedia.org/wiki/Python_(egy%C3%A9rtelm%C5%B1s%C3%ADt%C5%91_lap)', 'https://mr.wikipedia.org/wiki/%E0%A4%AA%E0%A4%BE%E0%A4%AF%E0%A4%A5%E0%A5%89%E0%A4%A8_(%E0%A4%86%E0%A4%9C%E0%A5%8D%E0%A4%9E%E0%A4%BE%E0%A4%B5%E0%A4%B2%E0%A5%80_%E0%A4%AD%E0%A4%BE%E0%A4%B7%E0%A4%BE)', 'https://nl.wikipedia.org/wiki/Python', 'https://ja.wikipedia.org/wiki/%E3%83%91%E3%82%A4%E3%82%BD%E3%83%B3', 'https://no.wikipedia.org/wiki/Pyton', 'https://pl.wikipedia.org/wiki/Pyton', 'https://pt.wikipedia.org/wiki/Python_(desambigua%C3%A7%C3%A3o)', 'https://ru.wikipedia.org/wiki/Python_(%D0%B7%D0%BD%D0%B0%D1%87%D0%B5%D0%BD%D0%B8%D1%8F)', 'https://sk.wikipedia.org/wiki/Python', 'https://sr.wikipedia.org/wiki/%D0%9F%D0%B8%D1%82%D0%BE%D0%BD_(%D0%B2%D0%B8%D1%88%D0%B5%D0%B7%D0%BD%D0%B0%D1%87%D0%BD%D0%B0_%D0%BE%D0%B4%D1%80%D0%B5%D0%B4%D0%BD%D0%B8%D1%86%D0%B0)', 'https://sh.wikipedia.org/wiki/Python', 'https://fi.wikipedia.org/wiki/Python', 'https://sv.wikipedia.org/wiki/Pyton', 'https://th.wikipedia.org/wiki/%E0%B9%84%E0%B8%9E%E0%B8%97%E0%B8%AD%E0%B8%99', 'https://tr.wikipedia.org/wiki/Python', 'https://uk.wikipedia.org/wiki/%D0%9F%D1%96%D1%84%D0%BE%D0%BD', 'https://ur.wikipedia.org/wiki/%D9%BE%D8%A7%D8%A6%DB%8C%D8%AA%DA%BE%D9%88%D9%86', 'https://vi.wikipedia.org/wiki/Python', 'https://zh.wikipedia.org/wiki/Python_(%E6%B6%88%E6%AD%A7%E4%B9%89)', 'https://www.wikidata.org/wiki/Special:EntityPage/Q747452#sitelinks-wikipedia', 'https://foundation.wikimedia.org/wiki/Privacy_policy', 'https://www.mediawiki.org/wiki/Special:MyLanguage/How_to_contribute', 'https://stats.wikimedia.org/#/en.wikipedia.org', 'https://foundation.wikimedia.org/wiki/Cookie_statement', 'https://wikimediafoundation.org/', 'https://www.mediawiki.org/']\n" + ] + } + ], + "source": [ + "scraper_links(soup,links)" + ] + }, { "cell_type": "markdown", - "id": "relevant-performer", "metadata": {}, "source": [ "### Find the number of titles that have changed in the United States Code since its last release point" @@ -47,17 +116,138 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "scenic-surgeon", + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "url = 'http://uscode.house.gov/download/download.shtml'" ] }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response = requests.get(url)\n", + "response.status_code" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "soup = BeautifulSoup(response.content, \"html.parser\")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "changed_titles = soup.find_all(\"div\", class_=\"usctitlechanged\")" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[
\n", + " \n", + " Title 1 - General Provisions ٭\n", + "
,\n", + "
\n", + " \n", + " Title 5 - Government Organization and Employees ٭\n", + "
,\n", + "
\n", + " \n", + " Title 8 - Aliens and Nationality\n", + " \n", + "
,\n", + "
\n", + " \n", + " Title 11 - Bankruptcy ٭\n", + "
,\n", + "
\n", + " \n", + " Title 18 - Crimes and Criminal Procedure ٭\n", + "
,\n", + "
\n", + " \n", + " Title 23 - Highways ٭\n", + "
,\n", + "
\n", + " \n", + " Title 26 - Internal Revenue Code\n", + " \n", + "
,\n", + "
\n", + " \n", + " Title 42 - The Public Health and Welfare\n", + " \n", + "
]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "changed_titles" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def scraper_text(html, text_list): \n", + " for i in html:\n", + " text_list.append(i.get_text())\n", + " print(text_list)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['\\n\\n Title 1 - General Provisions ٭\\n', '\\n\\n Title 5 - Government Organization and Employees ٭\\n', '\\n\\n Title 8 - Aliens and Nationality\\n\\n ', '\\n\\n Title 11 - Bankruptcy ٭\\n', '\\n\\n Title 18 - Crimes and Criminal Procedure ٭\\n', '\\n\\n Title 23 - Highways ٭\\n', '\\n\\n Title 26 - Internal Revenue Code\\n\\n ', '\\n\\n Title 42 - The Public Health and Welfare\\n\\n ']\n" + ] + } + ], + "source": [ + "changed_titles_list = []\n", + "scraper_text(changed_titles, changed_titles_list)" + ] + }, { "cell_type": "markdown", - "id": "acute-necessity", "metadata": {}, "source": [ "### Create a Python list with the top ten FBI's Most Wanted names" @@ -65,17 +255,72 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "starting-blackberry", + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ - "url = 'https://www.fbi.gov/wanted/topten" + "url = 'https://www.fbi.gov/wanted/topten'" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response = requests.get(url)\n", + "response.status_code" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "soup = BeautifulSoup(response.content, \"html.parser\")" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "most_wanted_names = soup.find_all(\"h3\")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['\\nJASON DEREK BROWN\\n', '\\nALEXIS FLORES\\n', '\\nJOSE RODOLFO VILLARREAL-HERNANDEZ\\n', '\\nOCTAVIANO JUAREZ-CORRO\\n', '\\nEUGENE PALMER\\n', '\\nRAFAEL CARO-QUINTERO\\n', '\\nBHADRESHKUMAR CHETANBHAI PATEL\\n', '\\nALEJANDRO ROSALES CASTILLO\\n', '\\nROBERT WILLIAM FISHER\\n', '\\nARNOLDO JIMENEZ\\n', 'federal bureau of investigation', '\\nFBI.gov Contact Center\\n']\n" + ] + } + ], + "source": [ + "most_wanted_names_list = []\n", + "scraper_text(most_wanted_names, most_wanted_names_list)" ] }, { "cell_type": "markdown", - "id": "joined-induction", "metadata": {}, "source": [ "### Display the 20 latest earthquakes info (date, time, latitude, longitude and region name) by the EMSC as a pandas dataframe" @@ -83,17 +328,54 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "copyrighted-taiwan", + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ "url = 'https://www.emsc-csem.org/Earthquake/'" ] }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.get(url)\n", + "response.status_code\n", + "soup = BeautifulSoup(response.content, \"html.parser\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", - "id": "dominican-defeat", "metadata": {}, "source": [ "### List all language names and number of related articles in the order they appear in [wikipedia.org](wikipedia.org)" @@ -101,17 +383,103 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "metric-vertex", + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ "url = 'https://www.wikipedia.org/'" ] }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.get(url)\n", + "response.status_code\n", + "soup = BeautifulSoup(response.content, \"html.parser\")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['The Free Encyclopedia', 'English', '日本語', 'Español', 'Deutsch', 'Русский', 'Français', '中文', 'Italiano', 'Português', 'Polski', '\\n\\nDownload Wikipedia for Android or iOS\\n\\n']\n" + ] + } + ], + "source": [ + "lan_text = []\n", + "lan = soup.find_all(\"strong\")\n", + "scraper_text(lan, lan_text)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "lan_text.remove('The Free Encyclopedia')" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['English',\n", + " '日本語',\n", + " 'Español',\n", + " 'Deutsch',\n", + " 'Русский',\n", + " 'Français',\n", + " '中文',\n", + " 'Italiano',\n", + " 'Português',\n", + " 'Polski']" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "lan_text.remove('\\n\\nDownload Wikipedia for Android or iOS\\n\\n')\n", + "lan_text" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['6\\xa0383\\xa0000+', '1\\xa0292\\xa0000+', '1\\xa0717\\xa0000+', '2\\xa0617\\xa0000+', '1\\xa0756\\xa0000+', '2\\xa0362\\xa0000+', '1\\xa0231\\xa0000+', '1\\xa0718\\xa0000+', '1\\xa0074\\xa0000+', '1\\xa0490\\xa0000+', '\\n1\\xa0000\\xa0000+\\n', 'العربية', 'مصرى', '\\n100\\xa0000+\\n', 'فارسی', 'עברית', 'قازاقشا', 'تۆرکجه', 'اردو', '\\n10\\xa0000+\\n', 'هَوُسَا', 'كوردی', 'کوردیی ناوەندی', 'مازِرونی', 'پنجابی (شاہ مکھی)', 'پښتو', 'سنڌي', 'ייִדיש', '\\n1\\xa0000+\\n', 'ܐܬܘܪܝܐ', 'گیلکی', 'לאדינו', 'ئۇيغۇرچه', 'ދިވެހިބަސް', '\\n100+\\n', 'كشميري']\n" + ] + } + ], + "source": [ + "article_count_text = []\n", + "article_count = soup.find_all(\"bdi\")\n", + "scraper_text(article_count, article_count_text)" + ] + }, { "cell_type": "markdown", - "id": "split-cartridge", "metadata": {}, "source": [ "### A list with the different kind of datasets available in [data.gov.uk](data.gov.uk)\n" @@ -119,17 +487,45 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "actual-parallel", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "url = 'https://data.gov.uk/'" + ] + }, + { + "cell_type": "code", + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ - "url = 'https://data.gov.uk/" + "response = requests.get(url)\n", + "response.status_code\n", + "soup = BeautifulSoup(response.content, \"html.parser\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['Business and economy', 'Crime and justice', 'Defence', 'Education', 'Environment', 'Government', 'Government spending', 'Health', 'Mapping', 'Society', 'Towns and cities', 'Transport', 'Digital service performance', 'Government reference data']\n" + ] + } + ], + "source": [ + "data_gov_text = []\n", + "data_gov = soup.find_all('h3')\n", + "scraper_text(data_gov, data_gov_text)" ] }, { "cell_type": "markdown", - "id": "potential-malpractice", "metadata": {}, "source": [ "### Display the top 10 languages by number of native speakers stored in a pandas dataframe" @@ -137,13 +533,44 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "adaptive-calculator", + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ "url = 'https://en.wikipedia.org/wiki/List_of_languages_by_number_of_native_speakers'" ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.get(url)\n", + "response.status_code\n", + "soup = BeautifulSoup(response.content, \"html.parser\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": {