diff --git a/01_Billboard.ipynb b/01_Billboard.ipynb index 779a38c..db000c5 100644 --- a/01_Billboard.ipynb +++ b/01_Billboard.ipynb @@ -41,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "promotional-algorithm", "metadata": {}, "outputs": [], @@ -59,11 +59,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "civic-broad", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ + "response = requests.get(url)\n", + "response.status_code\n", "# 200 status code means OK!" ] }, @@ -77,11 +90,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "revised-digest", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "soup = BeautifulSoup(response.content, \"html.parser\")" + ] }, { "cell_type": "markdown", @@ -93,20 +108,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "id": "falling-chambers", "metadata": {}, "outputs": [], "source": [ "# song titles\n", + "songs = soup.find_all(\"span\", class_=\"chart-element__information__song text--truncate color--primary\")\n", "\n", "# artists\n", + "artists = soup.find_all(\"span\", class_=\"chart-element__information__artist text--truncate color--secondary\")\n", + "\n", + "# this week\n", + "this_week_ranks = soup.find_all(\"span\", class_=\"chart-element__rank__number\")\n", "\n", "# last week\n", + "last_week_ranks = soup.find_all(\"span\", class_= \"chart-element__meta text--center color--secondary text--last\")\n", "\n", "# peak rank\n", + "peak_ranks = soup.find_all(\"span\", class_= \"chart-element__meta text--center color--secondary text--peak\")\n", "\n", - "# weeks on chart\n" + "# weeks on chart\n", + "weeks_on_chart = soup.find_all(\"span\", class_= \"chart-element__meta text--center color--secondary text--week\")" ] }, { @@ -119,11 +142,36 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, + "id": "020cc006", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Industry Baby'" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "songs[2].getText()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, "id": "amateur-protocol", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "for i in [songs, artists, this_week_ranks, last_week_ranks, peak_ranks, weeks_on_chart]:\n", + " for j in range(len(i)):\n", + " i[j] = i[j].getText()" + ] }, { "cell_type": "markdown", @@ -135,10 +183,206 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "external-instrumentation", "metadata": {}, "outputs": [], + "source": [ + "billboard = pd.DataFrame(\n", + " {\"song\": songs,\n", + " \"artist\": artists,\n", + " \"current_rank\": this_week_ranks,\n", + " \"last_week_rank\": last_week_ranks,\n", + " \"peak_rank\": peak_ranks,\n", + " \"weeks_on_chart\": weeks_on_chart}\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "7755df6d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
songartistcurrent_ranklast_week_rankpeak_rankweeks_on_chart
0My UniverseColdplay x BTS1-11
1StayThe Kid LAROI & Justin Bieber21112
2Industry BabyLil Nas X & Jack Harlow32210
3Way 2 SexyDrake Featuring Future & Young Thug4314
4Fancy LikeWalker Hayes55515
.....................
95Pipe DownDrake9668144
96Papi's HomeDrake976684
97ChosenBlxst & Tyga Featuring Ty Dolla $ign98-981
98Toxic PunkYoungBoy Never Broke Again99-991
99MoonKanye West10076175
\n", + "

100 rows × 6 columns

\n", + "
" + ], + "text/plain": [ + " song artist current_rank \\\n", + "0 My Universe Coldplay x BTS 1 \n", + "1 Stay The Kid LAROI & Justin Bieber 2 \n", + "2 Industry Baby Lil Nas X & Jack Harlow 3 \n", + "3 Way 2 Sexy Drake Featuring Future & Young Thug 4 \n", + "4 Fancy Like Walker Hayes 5 \n", + ".. ... ... ... \n", + "95 Pipe Down Drake 96 \n", + "96 Papi's Home Drake 97 \n", + "97 Chosen Blxst & Tyga Featuring Ty Dolla $ign 98 \n", + "98 Toxic Punk YoungBoy Never Broke Again 99 \n", + "99 Moon Kanye West 100 \n", + "\n", + " last_week_rank peak_rank weeks_on_chart \n", + "0 - 1 1 \n", + "1 1 1 12 \n", + "2 2 2 10 \n", + "3 3 1 4 \n", + "4 5 5 15 \n", + ".. ... ... ... \n", + "95 68 14 4 \n", + "96 66 8 4 \n", + "97 - 98 1 \n", + "98 - 99 1 \n", + "99 76 17 5 \n", + "\n", + "[100 rows x 6 columns]" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "billboard" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1bdcea94", + "metadata": {}, + "outputs": [], "source": [] } ], @@ -158,7 +402,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.8.8" }, "toc": { "base_numbering": 1, diff --git a/02_Further_questions.ipynb b/02_Further_questions.ipynb index 66fb224..fa2dd71 100644 --- a/02_Further_questions.ipynb +++ b/02_Further_questions.ipynb @@ -29,7 +29,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 68, + "id": "fc5c633c", + "metadata": {}, + "outputs": [], + "source": [ + "from bs4 import BeautifulSoup\n", + "import pandas as pd\n", + "import requests\n", + "import re" + ] + }, + { + "cell_type": "code", + "execution_count": 2, "id": "appreciated-bubble", "metadata": {}, "outputs": [], @@ -37,6 +50,117 @@ "url ='https://en.wikipedia.org/wiki/Python'" ] }, + { + "cell_type": "code", + "execution_count": 3, + "id": "76665d70", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "r = requests.get(url)\n", + "r.status_code" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "f5e9d13c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['/wiki/Pythonidae',\n", + " '/wiki/Python_(genus)',\n", + " '/w/index.php?title=Python&action=edit§ion=1',\n", + " '/wiki/Python_(programming_language)',\n", + " '/wiki/CMU_Common_Lisp',\n", + " '/wiki/PERQ#PERQ_3',\n", + " '/w/index.php?title=Python&action=edit§ion=2',\n", + " '/wiki/Python_of_Aenus',\n", + " '/wiki/Python_(painter)',\n", + " '/wiki/Python_of_Byzantium',\n", + " '/wiki/Python_of_Catana',\n", + " '/wiki/Python_Anghelo',\n", + " '/w/index.php?title=Python&action=edit§ion=3',\n", + " '/wiki/Python_(Efteling)',\n", + " '/wiki/Python_(Busch_Gardens_Tampa_Bay)',\n", + " '/wiki/Python_(Coney_Island,_Cincinnati,_Ohio)',\n", + " '/w/index.php?title=Python&action=edit§ion=4',\n", + " '/wiki/Python_(automobile_maker)',\n", + " '/wiki/Python_(Ford_prototype)',\n", + " '/w/index.php?title=Python&action=edit§ion=5',\n", + " '/wiki/Python_(missile)',\n", + " '/wiki/Python_(nuclear_primary)',\n", + " '/wiki/Colt_Python',\n", + " '/w/index.php?title=Python&action=edit§ion=6',\n", + " '/wiki/PYTHON',\n", + " '/wiki/Python_(film)',\n", + " '/wiki/Python_(mythology)',\n", + " '/wiki/Monty_Python',\n", + " '/wiki/Python_(Monty)_Pictures',\n", + " '/w/index.php?title=Python&action=edit§ion=7',\n", + " '/wiki/Cython',\n", + " '/wiki/Pyton',\n", + " '/wiki/Pithon',\n", + " '/wiki/File:Disambig_gray.svg',\n", + " '/wiki/Help:Disambiguation']" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "soup = BeautifulSoup(r.content, 'html.parser')\n", + "\n", + "links = []\n", + "\n", + "elems = soup.find('div', class_=\"mw-parser-output\").find_all('a')\n", + "for i in elems:\n", + " link = i.get('href')\n", + " if link.startswith('/'):\n", + " links.append(link)\n", + " \n", + "links" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "da68e029", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04860e0b", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fae36e41", + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "id": "relevant-performer", @@ -47,14 +171,63 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "id": "scenic-surgeon", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "url = 'http://uscode.house.gov/download/download.shtml'\n", + "r = requests.get(url)\n", + "r.status_code" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "id": "529a6a1e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "8" + ] + }, + "execution_count": 47, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "url = 'http://uscode.house.gov/download/download.shtml'" + "soup = BeautifulSoup(r.content, 'html.parser')\n", + "\n", + "changed_code = []\n", + "elems = soup.find_all('div', class_=\"usctitlechanged\")\n", + "for i in elems:\n", + " changed_code.append(i.get_text().strip())\n", + "\n", + "len(changed_code)" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "34b5ef8a", + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "id": "acute-necessity", @@ -65,14 +238,73 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 48, "id": "starting-blackberry", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 48, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "url = 'https://www.fbi.gov/wanted/topten" + "url = 'https://www.fbi.gov/wanted/topten'\n", + "\n", + "r = requests.get(url)\n", + "r.status_code" ] }, + { + "cell_type": "code", + "execution_count": 53, + "id": "e337b5b2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['JASON DEREK BROWN',\n", + " 'ALEXIS FLORES',\n", + " 'JOSE RODOLFO VILLARREAL-HERNANDEZ',\n", + " 'OCTAVIANO JUAREZ-CORRO',\n", + " 'EUGENE PALMER',\n", + " 'RAFAEL CARO-QUINTERO',\n", + " 'BHADRESHKUMAR CHETANBHAI PATEL',\n", + " 'ALEJANDRO ROSALES CASTILLO',\n", + " 'ROBERT WILLIAM FISHER',\n", + " 'ARNOLDO JIMENEZ']" + ] + }, + "execution_count": 53, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "soup = BeautifulSoup(r.content, 'html.parser')\n", + "\n", + "most_wanted = []\n", + "elems = soup.find_all('li', class_=\"portal-type-person castle-grid-block-item\")\n", + "for i in elems:\n", + " most_wanted.append(i.get_text().strip())\n", + "\n", + "most_wanted" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "78aa0cc2", + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "id": "joined-induction", @@ -83,12 +315,315 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 136, "id": "copyrighted-taiwan", "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 136, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "url = 'https://www.emsc-csem.org/Earthquake/'\n", + "\n", + "r = requests.get(url)\n", + "r.status_code" + ] + }, + { + "cell_type": "code", + "execution_count": 163, + "id": "f3e47876", + "metadata": {}, + "outputs": [], + "source": [ + "date = []\n", + "time= []\n", + "latitude = []\n", + "longitude = []\n", + "region = []\n", + "\n", + "soup = BeautifulSoup(r.content, 'html.parser')\n", + "elems = soup.find_all('tr')[14:34]\n" + ] + }, + { + "cell_type": "code", + "execution_count": 164, + "id": "97ce6e40", + "metadata": {}, + "outputs": [], + "source": [ + "for i in elems:\n", + "\n", + " date.append(i.find('a').getText().split()[0])\n", + " time.append(i.find('a').getText().split()[1])\n", + "\n", + " lat1 = i.find_all('td', class_= 'tabev1')[0].get_text()[:-1]\n", + " lat2 = i.find_all('td', class_= 'tabev2')[0].get_text()[:-2]\n", + " latitude.append(lat1 + lat2)\n", + "\n", + " lon1 = i.find_all('td', class_= 'tabev1')[1].get_text()[:-1]\n", + " lon2 = i.find_all('td', class_= 'tabev2')[1].get_text()[:-2]\n", + " longitude.append(lon1 + lon2)\n", + "\n", + " region.append(i.find('td', class_=\"tb_region\").get_text()[1:])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26359e3b", + "metadata": {}, "outputs": [], "source": [ - "url = 'https://www.emsc-csem.org/Earthquake/'" + "(href=re.compile('/Earthquake/earthquake.php'))" + ] + }, + { + "cell_type": "code", + "execution_count": 165, + "id": "b0b08ae2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
datetimelatitudelongituderegion
02021-10-0907:31:49.019.22N155.40WISLAND OF HAWAII, HAWAII
12021-10-0907:30:50.228.58N17.85WCANARY ISLANDS, SPAIN REGION
22021-10-0907:26:41.021.43S68.43WANTOFAGASTA, CHILE
32021-10-0907:19:26.550.00N149.89ESEA OF OKHOTSK
42021-10-0907:18:43.719.22N155.41WISLAND OF HAWAII, HAWAII
52021-10-0907:15:12.321.78S68.86WANTOFAGASTA, CHILE
62021-10-0907:12:26.628.54N17.80WCANARY ISLANDS, SPAIN REGION
72021-10-0907:05:04.637.16N37.18ECENTRAL TURKEY
82021-10-0906:56:27.528.59N17.88WCANARY ISLANDS, SPAIN REGION
92021-10-0906:54:43.828.55N17.84WCANARY ISLANDS, SPAIN REGION
102021-10-0906:54:11.963.32N158.40EMAGADANSKAYA OBLAST', RUSSIA
112021-10-0906:49:48.628.57N17.86WCANARY ISLANDS, SPAIN REGION
122021-10-0906:38:32.944.32N115.08WSOUTHERN IDAHO
132021-10-0906:28:20.928.55N17.84WCANARY ISLANDS, SPAIN REGION
142021-10-0906:25:53.428.55N17.86WCANARY ISLANDS, SPAIN REGION
152021-10-0906:25:07.015.63S71.93WSOUTHERN PERU
162021-10-0906:23:34.128.56N17.85WCANARY ISLANDS, SPAIN REGION
172021-10-0906:12:50.015.70S71.99WSOUTHERN PERU
182021-10-0906:08:45.119.16N155.48WISLAND OF HAWAII, HAWAII
192021-10-0906:08:30.228.56N17.86WCANARY ISLANDS, SPAIN REGION
\n", + "
" + ], + "text/plain": [ + " date time latitude longitude region\n", + "0 2021-10-09 07:31:49.0 19.22N 155.40W ISLAND OF HAWAII, HAWAII\n", + "1 2021-10-09 07:30:50.2 28.58N 17.85W CANARY ISLANDS, SPAIN REGION\n", + "2 2021-10-09 07:26:41.0 21.43S 68.43W ANTOFAGASTA, CHILE\n", + "3 2021-10-09 07:19:26.5 50.00N 149.89E SEA OF OKHOTSK\n", + "4 2021-10-09 07:18:43.7 19.22N 155.41W ISLAND OF HAWAII, HAWAII\n", + "5 2021-10-09 07:15:12.3 21.78S 68.86W ANTOFAGASTA, CHILE\n", + "6 2021-10-09 07:12:26.6 28.54N 17.80W CANARY ISLANDS, SPAIN REGION\n", + "7 2021-10-09 07:05:04.6 37.16N 37.18E CENTRAL TURKEY\n", + "8 2021-10-09 06:56:27.5 28.59N 17.88W CANARY ISLANDS, SPAIN REGION\n", + "9 2021-10-09 06:54:43.8 28.55N 17.84W CANARY ISLANDS, SPAIN REGION\n", + "10 2021-10-09 06:54:11.9 63.32N 158.40E MAGADANSKAYA OBLAST', RUSSIA\n", + "11 2021-10-09 06:49:48.6 28.57N 17.86W CANARY ISLANDS, SPAIN REGION\n", + "12 2021-10-09 06:38:32.9 44.32N 115.08W SOUTHERN IDAHO\n", + "13 2021-10-09 06:28:20.9 28.55N 17.84W CANARY ISLANDS, SPAIN REGION\n", + "14 2021-10-09 06:25:53.4 28.55N 17.86W CANARY ISLANDS, SPAIN REGION\n", + "15 2021-10-09 06:25:07.0 15.63S 71.93W SOUTHERN PERU\n", + "16 2021-10-09 06:23:34.1 28.56N 17.85W CANARY ISLANDS, SPAIN REGION\n", + "17 2021-10-09 06:12:50.0 15.70S 71.99W SOUTHERN PERU\n", + "18 2021-10-09 06:08:45.1 19.16N 155.48W ISLAND OF HAWAII, HAWAII\n", + "19 2021-10-09 06:08:30.2 28.56N 17.86W CANARY ISLANDS, SPAIN REGION" + ] + }, + "execution_count": 165, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "earthquakes = pd.DataFrame(\n", + " {\"date\": date,\n", + " \"time\": time,\n", + " \"latitude\": latitude,\n", + " \"longitude\": longitude,\n", + " \"region\": region}\n", + ")\n", + "earthquakes" ] }, { @@ -101,12 +636,221 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 166, "id": "metric-vertex", "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 166, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "url = 'https://www.wikipedia.org/'\n", + "\n", + "r = requests.get(url)\n", + "r.status_code" + ] + }, + { + "cell_type": "code", + "execution_count": 271, + "id": "06a5b5b3", + "metadata": {}, "outputs": [], "source": [ - "url = 'https://www.wikipedia.org/'" + "language = []\n", + "articles = []\n", + "\n", + "soup = BeautifulSoup(r.content, 'html.parser')" + ] + }, + { + "cell_type": "code", + "execution_count": 272, + "id": "0012eb24", + "metadata": {}, + "outputs": [], + "source": [ + "lst1= ['1000000+', '100000+']\n", + "lst2= ['10000+', '1000+']\n", + "lst3= ['100+']\n", + "\n", + "elem_1 = soup.find_all('div', class_=\"langlist langlist-large hlist\")\n", + "for x in enumerate(elem_1):\n", + " lang = x[1].find_all('li')\n", + " a = x[0]\n", + " for i in lang:\n", + " language.append(i.getText())\n", + " articles.append(lst1[a])\n", + " \n", + "elem_2 = soup.find_all('div', class_=\"langlist hlist\")\n", + "for x in enumerate(elem_2):\n", + " lang = x[1].find_all('li')\n", + " a = x[0]\n", + " for i in lang:\n", + " language.append(i.getText())\n", + " articles.append(lst2[a])\n", + "\n", + "elem_3 = soup.find('div', class_=\"langlist langlist-tiny hlist\")\n", + "lang = elem_3.find_all('li')\n", + "for i in lang:\n", + " language.append(i.getText())\n", + " articles.append('100+')" + ] + }, + { + "cell_type": "code", + "execution_count": 273, + "id": "e120d1d6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
languagearticles
0Polski1000000+
1العربية1000000+
2Deutsch1000000+
3English1000000+
4Español1000000+
.........
279Tshivenḓa100+
280Xitsonga100+
281chiTumbuka100+
282Twi100+
283ትግርኛ100+
\n", + "

284 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " language articles\n", + "0 Polski 1000000+\n", + "1 العربية 1000000+\n", + "2 Deutsch 1000000+\n", + "3 English 1000000+\n", + "4 Español 1000000+\n", + ".. ... ...\n", + "279 Tshivenḓa 100+\n", + "280 Xitsonga 100+\n", + "281 chiTumbuka 100+\n", + "282 Twi 100+\n", + "283 ትግርኛ 100+\n", + "\n", + "[284 rows x 2 columns]" + ] + }, + "execution_count": 273, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "wiki_list = pd.DataFrame(\n", + " {\"language\": language,\n", + " \"articles\": articles}\n", + " )\n", + "wiki_list" + ] + }, + { + "cell_type": "code", + "execution_count": 270, + "id": "147324be", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "284" + ] + }, + "execution_count": 270, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# checking if the total number of language is right or not\n", + "\n", + "soup = BeautifulSoup(r.content, 'html.parser')\n", + "elem = soup.find('div', class_=\"lang-list-content\")\n", + "elem1 = elem.find_all('li')\n", + "len(elem1)" ] }, { @@ -119,12 +863,67 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 276, "id": "actual-parallel", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 276, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "url = 'https://data.gov.uk/" + "url = 'https://data.gov.uk/'\n", + "r = requests.get(url)\n", + "r.status_code" + ] + }, + { + "cell_type": "code", + "execution_count": 284, + "id": "052148d4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['Business and economy',\n", + " 'Crime and justice',\n", + " 'Defence',\n", + " 'Education',\n", + " 'Environment',\n", + " 'Government',\n", + " 'Government spending',\n", + " 'Health',\n", + " 'Mapping',\n", + " 'Society',\n", + " 'Towns and cities',\n", + " 'Transport',\n", + " 'Digital service performance',\n", + " 'Government reference data']" + ] + }, + "execution_count": 284, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "database_category = []\n", + "\n", + "soup = BeautifulSoup(r.content, 'html.parser')\n", + "elem_1 = soup.find('div', class_=\"govuk-grid-column-full\")\n", + "elem_2 = elem_1.find_all('a')\n", + "for i in elem_2:\n", + " database_category.append(i.getText())\n", + "\n", + "database_category" ] }, { @@ -137,13 +936,75 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 285, "id": "adaptive-calculator", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 285, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "url = 'https://en.wikipedia.org/wiki/List_of_languages_by_number_of_native_speakers'" + "url = 'https://en.wikipedia.org/wiki/List_of_languages_by_number_of_native_speakers'\n", + "r = requests.get(url)\n", + "r.status_code" ] + }, + { + "cell_type": "code", + "execution_count": 325, + "id": "760d7e8c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['Mandarin Chinese',\n", + " 'Spanish',\n", + " 'English',\n", + " 'Hindi (sanskritised Hindustani)[9]',\n", + " 'Bengali',\n", + " 'Portuguese',\n", + " 'Russian',\n", + " 'Japanese',\n", + " 'Western Punjabi[10]',\n", + " 'Marathi']" + ] + }, + "execution_count": 325, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "top10_languages = []\n", + "\n", + "soup = BeautifulSoup(r.content, 'html.parser')\n", + "elem_1 = soup.find(\"table\", class_='wikitable sortable')\n", + "elem_2 = elem_1.find_all('tr')[1:11]\n", + "\n", + "for i in elem_2:\n", + " elem_3 = i.find_all('td')[1]\n", + " lang = elem_3.getText().strip()\n", + " top10_languages.append(lang)\n", + "\n", + "top10_languages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8a004f09", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -162,7 +1023,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.8.8" }, "toc": { "base_numbering": 1,