diff --git a/01_Billboard.ipynb b/01_Billboard.ipynb index 779a38c..6795edb 100644 --- a/01_Billboard.ipynb +++ b/01_Billboard.ipynb @@ -21,7 +21,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "id": "little-prophet", "metadata": {}, "outputs": [], @@ -31,17 +31,27 @@ "import pandas as pd" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "a52ad9ba-5f3c-4760-995d-3e57933d0a52", + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "id": "political-wright", "metadata": {}, "source": [ + "# Billboard Hot 100 data\n", + "\n", "## Store the hot-100 songs list from billboard url in a variable" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "promotional-algorithm", "metadata": {}, "outputs": [], @@ -59,11 +69,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "civic-broad", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ + "response = requests.get(url)\n", + "response.status_code\n", "# 200 status code means OK!" ] }, @@ -77,11 +100,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "revised-digest", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "soup = BeautifulSoup(response.content, \"html.parser\")" + ] }, { "cell_type": "markdown", @@ -93,20 +118,145 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "falling-chambers", "metadata": {}, "outputs": [], "source": [ - "# song titles\n", + "# songs\n", + "songs = soup.find_all(\"span\", class_=\"chart-element__information__song text--truncate color--primary\")\n", "\n", "# artists\n", + "artists = soup.find_all(\"span\", class_=\"chart-element__information__artist text--truncate color--secondary\")\n", "\n", "# last week\n", + "last_week_ranks = soup.find_all(\"span\", class_= \"chart-element__meta text--center color--secondary text--last\")\n", "\n", "# peak rank\n", + "peak_ranks = soup.find_all(\"span\", class_= \"chart-element__meta text--center color--secondary text--peak\")\n", "\n", - "# weeks on chart\n" + "# weeks on chart\n", + "weeks_on_chart = soup.find_all(\"span\", class_= \"chart-element__meta text--center color--secondary text--week\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f2ecd39f-81da-4386-8e8d-bef7e9b4ceaf", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[My Universe,\n", + " Stay,\n", + " Industry Baby,\n", + " Way 2 Sexy,\n", + " Fancy Like,\n", + " Bad Habits,\n", + " Good 4 U,\n", + " Kiss Me More,\n", + " Knife Talk,\n", + " Levitating,\n", + " Essence,\n", + " Save Your Tears,\n", + " Montero (Call Me By Your Name),\n", + " Shivers,\n", + " Heat Waves,\n", + " Need To Know,\n", + " Girls Want Girls,\n", + " You Right,\n", + " Beggin',\n", + " Wockesha,\n", + " If I Didn't Love You,\n", + " Take My Breath,\n", + " Fair Trade,\n", + " Thats What I Want,\n", + " Traitor,\n", + " Cold Beer Calling My Name,\n", + " Chasing After You,\n", + " Bad Morning,\n", + " Pepas,\n", + " Deja Vu,\n", + " Happier Than Ever,\n", + " Your Heart,\n", + " Leave The Door Open,\n", + " Hurricane,\n", + " A-O-K,\n", + " Butter,\n", + " On My Side,\n", + " Too Easy,\n", + " Leave Before You Love Me,\n", + " No Where,\n", + " Peaches,\n", + " Family Ties,\n", + " Memory I Don't Mess With,\n", + " Late At Night,\n", + " Things A Man Oughta Know,\n", + " Cold Heart (PNAU Remix),\n", + " Meet Me At Our Spot,\n", + " Life Support,\n", + " I Was On A Boat That Day,\n", + " Love Nwantiti (Ah Ah Ah),\n", + " Wild Side,\n", + " Whole Lotta Money,\n", + " Hold Me Down,\n", + " Champagne Poetry,\n", + " Buy Dirt,\n", + " Have Mercy,\n", + " Love Again,\n", + " Nevada,\n", + " 50 Shots,\n", + " No Friends In The Industry,\n", + " Smoke Strong,\n", + " Break Or Make Me,\n", + " You Time,\n", + " Gyalis,\n", + " Cold As You,\n", + " My Boy,\n", + " Sincerely,\n", + " Drunk (And I Don't Wanna Go Home),\n", + " I Can't Take It Back,\n", + " Waves,\n", + " Sharing Locations,\n", + " In The Bible,\n", + " Thot Shit,\n", + " Rumors,\n", + " Woman,\n", + " Thinking 'Bout You,\n", + " Baddest,\n", + " 2055,\n", + " Memory,\n", + " Forgiato,\n", + " You Should Probably Leave,\n", + " Drinkin' Beer. Talkin' God. Amen.,\n", + " Rich Shit,\n", + " TSU,\n", + " Volvi,\n", + " Todo de Ti,\n", + " Love All,\n", + " N 2 Deep,\n", + " Yonaguni,\n", + " Knowing You,\n", + " For Tonight,\n", + " Baddest Thing,\n", + " Summer Of Love,\n", + " Get Into It (Yuh),\n", + " Same Boat,\n", + " Pipe Down,\n", + " Papi's Home,\n", + " Chosen,\n", + " Toxic Punk,\n", + " Moon]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "songs" ] }, { @@ -119,11 +269,135 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "amateur-protocol", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "for item in [songs, artists, last_week_ranks, peak_ranks, weeks_on_chart]:\n", + " for txt in range(len(item)):\n", + " item[txt] = item[txt].getText()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "b63b1793-e70d-40ab-af97-00a0fa8ca7d1", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['My Universe',\n", + " 'Stay',\n", + " 'Industry Baby',\n", + " 'Way 2 Sexy',\n", + " 'Fancy Like',\n", + " 'Bad Habits',\n", + " 'Good 4 U',\n", + " 'Kiss Me More',\n", + " 'Knife Talk',\n", + " 'Levitating',\n", + " 'Essence',\n", + " 'Save Your Tears',\n", + " 'Montero (Call Me By Your Name)',\n", + " 'Shivers',\n", + " 'Heat Waves',\n", + " 'Need To Know',\n", + " 'Girls Want Girls',\n", + " 'You Right',\n", + " \"Beggin'\",\n", + " 'Wockesha',\n", + " \"If I Didn't Love You\",\n", + " 'Take My Breath',\n", + " 'Fair Trade',\n", + " 'Thats What I Want',\n", + " 'Traitor',\n", + " 'Cold Beer Calling My Name',\n", + " 'Chasing After You',\n", + " 'Bad Morning',\n", + " 'Pepas',\n", + " 'Deja Vu',\n", + " 'Happier Than Ever',\n", + " 'Your Heart',\n", + " 'Leave The Door Open',\n", + " 'Hurricane',\n", + " 'A-O-K',\n", + " 'Butter',\n", + " 'On My Side',\n", + " 'Too Easy',\n", + " 'Leave Before You Love Me',\n", + " 'No Where',\n", + " 'Peaches',\n", + " 'Family Ties',\n", + " \"Memory I Don't Mess With\",\n", + " 'Late At Night',\n", + " 'Things A Man Oughta Know',\n", + " 'Cold Heart (PNAU Remix)',\n", + " 'Meet Me At Our Spot',\n", + " 'Life Support',\n", + " 'I Was On A Boat That Day',\n", + " 'Love Nwantiti (Ah Ah Ah)',\n", + " 'Wild Side',\n", + " 'Whole Lotta Money',\n", + " 'Hold Me Down',\n", + " 'Champagne Poetry',\n", + " 'Buy Dirt',\n", + " 'Have Mercy',\n", + " 'Love Again',\n", + " 'Nevada',\n", + " '50 Shots',\n", + " 'No Friends In The Industry',\n", + " 'Smoke Strong',\n", + " 'Break Or Make Me',\n", + " 'You Time',\n", + " 'Gyalis',\n", + " 'Cold As You',\n", + " 'My Boy',\n", + " 'Sincerely',\n", + " \"Drunk (And I Don't Wanna Go Home)\",\n", + " \"I Can't Take It Back\",\n", + " 'Waves',\n", + " 'Sharing Locations',\n", + " 'In The Bible',\n", + " 'Thot Shit',\n", + " 'Rumors',\n", + " 'Woman',\n", + " \"Thinking 'Bout You\",\n", + " 'Baddest',\n", + " '2055',\n", + " 'Memory',\n", + " 'Forgiato',\n", + " 'You Should Probably Leave',\n", + " \"Drinkin' Beer. Talkin' God. Amen.\",\n", + " 'Rich Shit',\n", + " 'TSU',\n", + " 'Volvi',\n", + " 'Todo de Ti',\n", + " 'Love All',\n", + " 'N 2 Deep',\n", + " 'Yonaguni',\n", + " 'Knowing You',\n", + " 'For Tonight',\n", + " 'Baddest Thing',\n", + " 'Summer Of Love',\n", + " 'Get Into It (Yuh)',\n", + " 'Same Boat',\n", + " 'Pipe Down',\n", + " \"Papi's Home\",\n", + " 'Chosen',\n", + " 'Toxic Punk',\n", + " 'Moon']" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "songs" + ] }, { "cell_type": "markdown", @@ -135,10 +409,1052 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "external-instrumentation", "metadata": {}, "outputs": [], + "source": [ + "hot100 = pd.DataFrame(\n", + " {\"song\": songs,\n", + " \"artist\": artists,\n", + " \"last_week_rank\": last_week_ranks,\n", + " \"peak_rank\": peak_ranks,\n", + " \"weeks_on_chart\": weeks_on_chart})" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "dbc8fbe3-e422-47ba-982c-5cb9fa8beea9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
songartistlast_week_rankpeak_rankweeks_on_chart
0My UniverseColdplay x BTS-11
1StayThe Kid LAROI & Justin Bieber1112
2Industry BabyLil Nas X & Jack Harlow2210
3Way 2 SexyDrake Featuring Future & Young Thug314
4Fancy LikeWalker Hayes5515
\n", + "
" + ], + "text/plain": [ + " song artist last_week_rank \\\n", + "0 My Universe Coldplay x BTS - \n", + "1 Stay The Kid LAROI & Justin Bieber 1 \n", + "2 Industry Baby Lil Nas X & Jack Harlow 2 \n", + "3 Way 2 Sexy Drake Featuring Future & Young Thug 3 \n", + "4 Fancy Like Walker Hayes 5 \n", + "\n", + " peak_rank weeks_on_chart \n", + "0 1 1 \n", + "1 1 12 \n", + "2 2 10 \n", + "3 1 4 \n", + "4 5 15 " + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hot100.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df961efa-93a5-453f-a47d-e1d9e83d51c8", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "2b8de2bc-ed0b-43c0-848e-d4b344d6cd92", + "metadata": {}, + "source": [ + "# World Singles Top 100 data" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "55049c18-398c-4b7a-8e10-1e47dcb0b3ec", + "metadata": {}, + "outputs": [], + "source": [ + "url_ws100 = \"https://top40-charts.com/chart.php?cid=35\"" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "f40b44b4-b68e-4622-b55a-ec3f5a518671", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response = requests.get(url_ws100)\n", + "response.status_code" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "id": "f80755c2-09b7-4706-8f69-7d2b616ce83c", + "metadata": {}, + "outputs": [], + "source": [ + "soup_ws100 = BeautifulSoup(response.content, \"html.parser\")\n", + "\n", + "ws100_page = soup_ws100.find_all('tr', attrs={'class': 'latc_song'})" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "id": "0e0855af-1e10-4542-83a4-67a0f922fa31", + "metadata": {}, + "outputs": [], + "source": [ + "ws100_songs, ws100_artists, ws100_last_week_ranks, ws100_peak_ranks, ws100_weeks_on_chart = [], [], [], [], []\n", + "\n", + "for song in ws100_page:\n", + " ws100_songs.append(song.find_all('a')[2].get_text())\n", + " ws100_artists.append(song.find_all('a')[3].get_text())\n", + " ws100_last_week_ranks.append(song.find_all('td', attrs={'class': 'text-nowrap text-center'})[2].get_text())\n", + " ws100_peak_ranks.append(song.find_all('td', attrs={'align': 'right'})[3].find('font').get_text())\n", + " ws100_weeks_on_chart.append(song.find_all('td', attrs={'align': 'right'})[4].find('font').get_text())\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "id": "bf1db00b-b7a4-46ce-b6ad-9b171c1d1cff", + "metadata": {}, + "outputs": [], + "source": [ + "ws100 = pd.DataFrame(\n", + " {\"song\": ws100_songs,\n", + " \"artist\": ws100_artists,\n", + " \"last_week_rank\": ws100_last_week_ranks,\n", + " \"peak_rank\": ws100_peak_ranks,\n", + " \"weeks_on_chart\": ws100_weeks_on_chart})" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "id": "f8f8a5fa-d42b-4cb8-bebd-6c8ebff138d7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
songartistlast_week_rankpeak_rankweeks_on_chart
0StayKid Laroi & Justin Bieber1112
1Bad HabitsEd Sheeran6114
2Industry BabyLil Nas X & Jack Harlow5210
3ShiversEd Sheeran223
4Thats What I WantLil Nas X332
..................
95Remote ControlKanye West62622
96Tu Mi Hai CapitoMadame & Sfera EbbastaRE214
97Todo De TiRauw Alejandro100577
98Ik Ga ZwemmenMart HoogkamerRE624
99LuccioleBlancoRE213
\n", + "

100 rows × 5 columns

\n", + "
" + ], + "text/plain": [ + " song artist last_week_rank peak_rank \\\n", + "0 Stay Kid Laroi & Justin Bieber 1 1 \n", + "1 Bad Habits Ed Sheeran 6 1 \n", + "2 Industry Baby Lil Nas X & Jack Harlow 5 2 \n", + "3 Shivers Ed Sheeran 2 2 \n", + "4 Thats What I Want Lil Nas X 3 3 \n", + ".. ... ... ... ... \n", + "95 Remote Control Kanye West 62 62 \n", + "96 Tu Mi Hai Capito Madame & Sfera Ebbasta RE 2 \n", + "97 Todo De Ti Rauw Alejandro 100 57 \n", + "98 Ik Ga Zwemmen Mart Hoogkamer RE 6 \n", + "99 Lucciole Blanco RE 2 \n", + "\n", + " weeks_on_chart \n", + "0 12 \n", + "1 14 \n", + "2 10 \n", + "3 3 \n", + "4 2 \n", + ".. ... \n", + "95 2 \n", + "96 14 \n", + "97 7 \n", + "98 24 \n", + "99 13 \n", + "\n", + "[100 rows x 5 columns]" + ] + }, + "execution_count": 57, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ws100" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "889ddf71-76c6-44f9-a5e4-0abccd3113f3", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "e352a865-899b-44e4-9f1d-c5aabddf6310", + "metadata": {}, + "source": [ + "# Dance Top 50 Germany data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "741c79e7-bf66-41a1-a787-64046d511318", + "metadata": {}, + "outputs": [], + "source": [ + "url_dance50ger = \"https://www.mix1.de/charts/dance50.htm\"" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "12f5b077-a0a2-4fbb-af60-026d4b93e61f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response = requests.get(url_dance50ger)\n", + "response.status_code" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "2390e435-bf33-4eb4-b7f1-86c9ac9cf117", + "metadata": {}, + "outputs": [], + "source": [ + "soup_dance50ger = BeautifulSoup(response.content, \"html.parser\")\n", + "\n", + "charts_dance50ger = soup_dance50ger.find_all('div', attrs={'class': 'charts-main-block'})" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "27db01a2-5d7a-4332-851f-e56fb35b8d92", + "metadata": {}, + "outputs": [], + "source": [ + "# songs\n", + "dance50ger_songs = soup_dance50ger.find_all(\"div\", attrs={'class': 'charts-single-title'})\n", + "dance50ger_songs = dance50ger_songs[::2]\n", + "\n", + "# artists\n", + "dance50ger_artists = soup_dance50ger.find_all(\"div\", attrs={'class': 'charts-single-interpret'})\n", + "dance50ger_artists = dance50ger_artists[::2]\n", + "\n", + "# last week\n", + "dance50ger_last_week_ranks = soup_dance50ger.find_all(\"div\", attrs={'class': 'charts-position-old'})\n", + "\n", + "# weeks on chart\n", + "dance50ger_weeks_on_chart = soup_dance50ger.find_all(\"div\", attrs={'class': 'charts-position-week'})" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "1778ef6a-6704-48e9-bcb9-14c115fc799c", + "metadata": {}, + "outputs": [], + "source": [ + "for item in [dance50ger_songs, dance50ger_artists, dance50ger_last_week_ranks, dance50ger_weeks_on_chart]:\n", + " for txt in range(len(item)):\n", + " item[txt] = item[txt].getText().strip().replace('. Woche', '')" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "4258086b-a4f6-438f-bf19-61fbf92346fd", + "metadata": {}, + "outputs": [], + "source": [ + "dance50ger = pd.DataFrame(\n", + " {\"song\": dance50ger_songs,\n", + " \"artist\": dance50ger_artists,\n", + " \"last_week_rank\": dance50ger_last_week_ranks,\n", + " \"weeks_on_chart\": dance50ger_weeks_on_chart})" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "95056e13-270d-46ed-8935-21bd8aab5032", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
songartistlast_week_rankweeks_on_chart
0DopaminePurple Disco Machine feat. Eyelar15
1Better Off AloneAncalima & Tom Franke63
2I Got A FeelingFelix Jaehn, Robin Schulz feat. Georgia Ku74
3Like ThatATB feat. Ben Samama35
4Rich In Paradise (Da Clubbmaster Edit)F. P. I. Project510
5The Age Of Love (Charlotte de Witte & Enrico S...Age Of Love107
6Titanium [David Guetta & MORTEN Future Rave Re...David Guetta feat. Sia164
7Cry a Little (Laurent Simeca Remix)Sean Finn & DJ Blackstone136
8Lost in SpaceArmin van Buuren & Jorn van Deynhoven88
9You To Me Are EverythingDisco Culture vs. Greg & Gregory313
10'O SarracinoAquagen128
11Stupidisco (David Penn Remix)Junior Jack119
12Another ChanceRoger Sanchez x Oliver Heldens403
13All the Things2 Electronic Souls176
14Dare Me (Yvvan Back Remix)Sean Finn & Paul Jockey147
15House Is the NationDream Sound Masters, Ezequiel Asencio256
16Never Knew Love Like This BeforeDa Clubbmaster & Tom Pulse452
17Richtung SonneTimster, Ninth & Seaside Clubbers157
18Love Meditation 2.0Andy Jay Powell422
19Paradise (Summer Mix)No Hopes214
20VommuliEDX205
21Sweet DreamsAlan Walker & Imanbek2210
22Anything Can Happen (Benny Benassi Remix)5 West198
23Never Going HomeKungs344
24You Want MeRuesche236
25I've Been Gone So LongTalla 2xlc & Christina Novelli276
26Run AwaySean Finn & The Soundlovers1810
27So Fly (Ogazumu Remix)Kendra Erika249
28Rise Up 2021Yves Larock x Steff Da Campo feat. Jaba304
29LegworkBootmasters feat. Gee Jay & DJ Beeman2610
30I Wanna GoRené De La Moné482
31You For MeSigala & Rita Ora2910
32GirlfriendKryder & B Jones355
33I Begin To WonderJoseph Sinatra & Djsmark feat. Emma325
34WeekendMufasa & Hypeman & Dopamine373
35Silent Tears 2021Beam & Michelle Aragon288
36AnimoAdam Hofer & Disscut383
37BodyrockBeam & Lara MCallenNeueinstieg
38The Way That It WasLaura Bryna364
39Party Of The NightMario Beck502
40Blinded by the LightRoyal Melody vs. Chris Rockford & Miq PuentesNeueinstieg
41Fade AwayAndrew Spencer feat. Jorik Burema419
42Love YouMario Beck3310
43C'est la VieWestbam/ML feat. Richard Judge392
44Stop This FlameCeleste & MK477
45ElectricityJonny Docherty feat. Alexandria Anderton439
46Get This Party StartedWestend449
47Music Sounds Better With You (Chris Geka Rework)StardustNeueinstieg
48PositanoNari466
49FreeVintage Culture & Fancy Inc & Roland Clark4910
\n", + "
" + ], + "text/plain": [ + " song \\\n", + "0 Dopamine \n", + "1 Better Off Alone \n", + "2 I Got A Feeling \n", + "3 Like That \n", + "4 Rich In Paradise (Da Clubbmaster Edit) \n", + "5 The Age Of Love (Charlotte de Witte & Enrico S... \n", + "6 Titanium [David Guetta & MORTEN Future Rave Re... \n", + "7 Cry a Little (Laurent Simeca Remix) \n", + "8 Lost in Space \n", + "9 You To Me Are Everything \n", + "10 'O Sarracino \n", + "11 Stupidisco (David Penn Remix) \n", + "12 Another Chance \n", + "13 All the Things \n", + "14 Dare Me (Yvvan Back Remix) \n", + "15 House Is the Nation \n", + "16 Never Knew Love Like This Before \n", + "17 Richtung Sonne \n", + "18 Love Meditation 2.0 \n", + "19 Paradise (Summer Mix) \n", + "20 Vommuli \n", + "21 Sweet Dreams \n", + "22 Anything Can Happen (Benny Benassi Remix) \n", + "23 Never Going Home \n", + "24 You Want Me \n", + "25 I've Been Gone So Long \n", + "26 Run Away \n", + "27 So Fly (Ogazumu Remix) \n", + "28 Rise Up 2021 \n", + "29 Legwork \n", + "30 I Wanna Go \n", + "31 You For Me \n", + "32 Girlfriend \n", + "33 I Begin To Wonder \n", + "34 Weekend \n", + "35 Silent Tears 2021 \n", + "36 Animo \n", + "37 Bodyrock \n", + "38 The Way That It Was \n", + "39 Party Of The Night \n", + "40 Blinded by the Light \n", + "41 Fade Away \n", + "42 Love You \n", + "43 C'est la Vie \n", + "44 Stop This Flame \n", + "45 Electricity \n", + "46 Get This Party Started \n", + "47 Music Sounds Better With You (Chris Geka Rework) \n", + "48 Positano \n", + "49 Free \n", + "\n", + " artist last_week_rank \\\n", + "0 Purple Disco Machine feat. Eyelar 1 \n", + "1 Ancalima & Tom Franke 6 \n", + "2 Felix Jaehn, Robin Schulz feat. Georgia Ku 7 \n", + "3 ATB feat. Ben Samama 3 \n", + "4 F. P. I. Project 5 \n", + "5 Age Of Love 10 \n", + "6 David Guetta feat. Sia 16 \n", + "7 Sean Finn & DJ Blackstone 13 \n", + "8 Armin van Buuren & Jorn van Deynhoven 8 \n", + "9 Disco Culture vs. Greg & Gregory 31 \n", + "10 Aquagen 12 \n", + "11 Junior Jack 11 \n", + "12 Roger Sanchez x Oliver Heldens 40 \n", + "13 2 Electronic Souls 17 \n", + "14 Sean Finn & Paul Jockey 14 \n", + "15 Dream Sound Masters, Ezequiel Asencio 25 \n", + "16 Da Clubbmaster & Tom Pulse 45 \n", + "17 Timster, Ninth & Seaside Clubbers 15 \n", + "18 Andy Jay Powell 42 \n", + "19 No Hopes 21 \n", + "20 EDX 20 \n", + "21 Alan Walker & Imanbek 22 \n", + "22 5 West 19 \n", + "23 Kungs 34 \n", + "24 Ruesche 23 \n", + "25 Talla 2xlc & Christina Novelli 27 \n", + "26 Sean Finn & The Soundlovers 18 \n", + "27 Kendra Erika 24 \n", + "28 Yves Larock x Steff Da Campo feat. Jaba 30 \n", + "29 Bootmasters feat. Gee Jay & DJ Beeman 26 \n", + "30 René De La Moné 48 \n", + "31 Sigala & Rita Ora 29 \n", + "32 Kryder & B Jones 35 \n", + "33 Joseph Sinatra & Djsmark feat. Emma 32 \n", + "34 Mufasa & Hypeman & Dopamine 37 \n", + "35 Beam & Michelle Aragon 28 \n", + "36 Adam Hofer & Disscut 38 \n", + "37 Beam & Lara MCallen \n", + "38 Laura Bryna 36 \n", + "39 Mario Beck 50 \n", + "40 Royal Melody vs. Chris Rockford & Miq Puentes \n", + "41 Andrew Spencer feat. Jorik Burema 41 \n", + "42 Mario Beck 33 \n", + "43 Westbam/ML feat. Richard Judge 39 \n", + "44 Celeste & MK 47 \n", + "45 Jonny Docherty feat. Alexandria Anderton 43 \n", + "46 Westend 44 \n", + "47 Stardust \n", + "48 Nari 46 \n", + "49 Vintage Culture & Fancy Inc & Roland Clark 49 \n", + "\n", + " weeks_on_chart \n", + "0 5 \n", + "1 3 \n", + "2 4 \n", + "3 5 \n", + "4 10 \n", + "5 7 \n", + "6 4 \n", + "7 6 \n", + "8 8 \n", + "9 3 \n", + "10 8 \n", + "11 9 \n", + "12 3 \n", + "13 6 \n", + "14 7 \n", + "15 6 \n", + "16 2 \n", + "17 7 \n", + "18 2 \n", + "19 4 \n", + "20 5 \n", + "21 10 \n", + "22 8 \n", + "23 4 \n", + "24 6 \n", + "25 6 \n", + "26 10 \n", + "27 9 \n", + "28 4 \n", + "29 10 \n", + "30 2 \n", + "31 10 \n", + "32 5 \n", + "33 5 \n", + "34 3 \n", + "35 8 \n", + "36 3 \n", + "37 Neueinstieg \n", + "38 4 \n", + "39 2 \n", + "40 Neueinstieg \n", + "41 9 \n", + "42 10 \n", + "43 2 \n", + "44 7 \n", + "45 9 \n", + "46 9 \n", + "47 Neueinstieg \n", + "48 6 \n", + "49 10 " + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dance50ger" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7a6a450b-1945-4dff-b734-82eb4ccad0ee", + "metadata": {}, + "outputs": [], "source": [] } ], @@ -158,7 +1474,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.9.4" }, "toc": { "base_numbering": 1, diff --git a/02_Further_questions.ipynb b/02_Further_questions.ipynb index 66fb224..e6de9b6 100644 --- a/02_Further_questions.ipynb +++ b/02_Further_questions.ipynb @@ -19,6 +19,19 @@ "As you've seen, scraping the internet is a skill that can get you all sorts of information. Here are some little challenges to gain more experience in the field" ] }, + { + "cell_type": "code", + "execution_count": 22, + "id": "3d4d2375-1271-40f8-a89b-88ef41258a10", + "metadata": {}, + "outputs": [], + "source": [ + "from bs4 import BeautifulSoup\n", + "import requests\n", + "import pandas as pd\n", + "import numpy as np" + ] + }, { "cell_type": "markdown", "id": "express-introduction", @@ -29,14 +42,159 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 125, "id": "appreciated-bubble", "metadata": {}, "outputs": [], "source": [ - "url ='https://en.wikipedia.org/wiki/Python'" + "url_python ='https://en.wikipedia.org/wiki/Python'" + ] + }, + { + "cell_type": "code", + "execution_count": 126, + "id": "43399097-38d1-46d0-a1ec-7128fafa9f76", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 126, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response = requests.get(url_python)\n", + "response.status_code" + ] + }, + { + "cell_type": "code", + "execution_count": 127, + "id": "47e40808-d59e-4522-8aa6-a5a0299d2810", + "metadata": {}, + "outputs": [], + "source": [ + "soup_python = BeautifulSoup(response.content, \"html.parser\")\n" ] }, + { + "cell_type": "code", + "execution_count": 134, + "id": "5bdb774e-5cc0-48f0-beed-ed791eb8238d", + "metadata": {}, + "outputs": [], + "source": [ + "list_a = soup_python.find('div', attrs={'class': 'mw-parser-output'}).find_all('a')" + ] + }, + { + "cell_type": "code", + "execution_count": 138, + "id": "5514936f-f51d-446f-ad3f-067d7d9a1c6e", + "metadata": {}, + "outputs": [], + "source": [ + "links = []\n", + "targets = []\n", + "\n", + "for lnk in list_a:\n", + " links.append(lnk.get_text())\n", + " targets.append(lnk.get('href'))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 139, + "id": "900eb55c-e17b-416d-910c-abcd0d9a6d85", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
linkurl
0Pythonhttps://en.wiktionary.org/wiki/Python
1pythonhttps://en.wiktionary.org/wiki/python
2Pythonidae/wiki/Pythonidae
3Python (genus)/wiki/Python_(genus)
41 Computing#Computing
\n", + "
" + ], + "text/plain": [ + " link url\n", + "0 Python https://en.wiktionary.org/wiki/Python\n", + "1 python https://en.wiktionary.org/wiki/python\n", + "2 Pythonidae /wiki/Pythonidae\n", + "3 Python (genus) /wiki/Python_(genus)\n", + "4 1 Computing #Computing" + ] + }, + "execution_count": 139, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_wiki_python = pd.DataFrame({'link': links, 'url': targets})\n", + "data_wiki_python.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6ccc9d62-19ec-4ad2-b1da-981d0f0fdf7b", + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "id": "relevant-performer", @@ -47,14 +205,186 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "scenic-surgeon", "metadata": {}, "outputs": [], "source": [ - "url = 'http://uscode.house.gov/download/download.shtml'" + "url_uscode = 'http://uscode.house.gov/download/download.shtml'" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "6940b792-2507-416f-ab48-aea37b47d6e6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response = requests.get(url_uscode)\n", + "response.status_code" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "cfb36744-6efa-4dd5-a824-b8548d2290eb", + "metadata": {}, + "outputs": [], + "source": [ + "soup_uscode = BeautifulSoup(response.content, \"html.parser\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "95bd1b45-1a12-4190-9015-ea928ca3b72b", + "metadata": {}, + "outputs": [], + "source": [ + "usctitleschanged = soup_uscode.find_all('div', attrs={'class': 'usctitlechanged'})" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "a5f6cdb4-2166-43d3-b239-0d6a6cbc8fa9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['Title 1 - General Provisions٭',\n", + " 'Title 5 - Government Organization and Employees٭',\n", + " 'Title 8 - Aliens and Nationality',\n", + " 'Title 11 - Bankruptcy٭',\n", + " 'Title 18 - Crimes and Criminal Procedure٭',\n", + " 'Title 23 - Highways٭',\n", + " 'Title 26 - Internal Revenue Code',\n", + " 'Title 42 - The Public Health and Welfare']" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "lst_usctitleschanged =[]\n", + "\n", + "for title in usctitleschanged:\n", + " lst_usctitleschanged.append(title.get_text(strip=True))\n", + " \n", + "lst_usctitleschanged" ] }, + { + "cell_type": "code", + "execution_count": 41, + "id": "26c69d06-5bc8-429c-8b95-2b162b1bcaaa", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
usc_titles_changed
0Title 1 - General Provisions٭
1Title 5 - Government Organization and Employees٭
2Title 8 - Aliens and Nationality
3Title 11 - Bankruptcy٭
4Title 18 - Crimes and Criminal Procedure٭
5Title 23 - Highways٭
6Title 26 - Internal Revenue Code
7Title 42 - The Public Health and Welfare
\n", + "
" + ], + "text/plain": [ + " usc_titles_changed\n", + "0 Title 1 - General Provisions٭\n", + "1 Title 5 - Government Organization and Employees٭\n", + "2 Title 8 - Aliens and Nationality\n", + "3 Title 11 - Bankruptcy٭\n", + "4 Title 18 - Crimes and Criminal Procedure٭\n", + "5 Title 23 - Highways٭\n", + "6 Title 26 - Internal Revenue Code\n", + "7 Title 42 - The Public Health and Welfare" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_usctitleschanged = pd.DataFrame({'usc_titles_changed': lst_usctitleschanged})\n", + "data_usctitleschanged" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "42df7659-6b37-4ed7-b177-e863479c5d6d", + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "id": "acute-necessity", @@ -65,14 +395,99 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 43, "id": "starting-blackberry", "metadata": {}, "outputs": [], "source": [ - "url = 'https://www.fbi.gov/wanted/topten" + "url_fbi = 'https://www.fbi.gov/wanted/topten'" ] }, + { + "cell_type": "code", + "execution_count": 53, + "id": "fe79d5bd-3d71-4085-b61b-489237fd8b8c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 53, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response = requests.get(url_fbi)\n", + "response.status_code" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "id": "82a925d4-7e1a-44aa-9db3-a6b26965613c", + "metadata": {}, + "outputs": [], + "source": [ + "soup_fbi = BeautifulSoup(response.content, \"html.parser\")" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "id": "e50600b0-122e-4b50-8695-9572c2bfd4b1", + "metadata": {}, + "outputs": [], + "source": [ + "most_wanted = soup_fbi.find_all('li', attrs={'class': 'portal-type-person castle-grid-block-item'})" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "id": "98f7c8a6-3e6c-4f76-9bdf-9da6c925cc66", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['JASON DEREK BROWN',\n", + " 'ALEXIS FLORES',\n", + " 'JOSE RODOLFO VILLARREAL-HERNANDEZ',\n", + " 'OCTAVIANO JUAREZ-CORRO',\n", + " 'EUGENE PALMER',\n", + " 'RAFAEL CARO-QUINTERO',\n", + " 'BHADRESHKUMAR CHETANBHAI PATEL',\n", + " 'ALEJANDRO ROSALES CASTILLO',\n", + " 'ROBERT WILLIAM FISHER',\n", + " 'ARNOLDO JIMENEZ']" + ] + }, + "execution_count": 61, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "top10_mw =[]\n", + "\n", + "for element in most_wanted:\n", + " top10_mw.append(element.find('h3').find('a').get_text())\n", + "\n", + "top10_mw" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "52bfb0ac-62df-4b9f-8405-3e0b0fa77903", + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "id": "joined-induction", @@ -83,14 +498,352 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "copyrighted-taiwan", "metadata": {}, "outputs": [], "source": [ - "url = 'https://www.emsc-csem.org/Earthquake/'" + "url_earthquake = 'https://www.emsc-csem.org/Earthquake/'" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "fc71ef66-e048-4597-a19a-cfdf39166f5c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response = requests.get(url_earthquake)\n", + "response.status_code" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "20b3e987-62f6-4bc5-83df-00d0c446e4f4", + "metadata": {}, + "outputs": [], + "source": [ + "soup_earthquake = BeautifulSoup(response.content, \"html.parser\")" ] }, + { + "cell_type": "code", + "execution_count": 6, + "id": "1588e325-90e5-4b56-8e66-3e06ada4b572", + "metadata": {}, + "outputs": [], + "source": [ + "eq_recent1 = soup_earthquake.find_all('tr', attrs={'class': 'ligne1 normal'})\n", + "eq_recent2 = soup_earthquake.find_all('tr', attrs={'class': 'ligne2 normal'})\n", + "eq_recent = []\n", + "\n", + "for i in range(len(eq_recent1)):\n", + " eq_recent.append(eq_recent1[i])\n", + " eq_recent.append(eq_recent2[i])\n", + "\n", + "eq_recent = eq_recent[:20]" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "id": "5c5aa41a-9801-4614-8fa3-d21faa111897", + "metadata": {}, + "outputs": [], + "source": [ + "eq_datetime = []\n", + "eq_lat = []\n", + "eq_lon = []\n", + "eq_region = []\n", + "\n", + "for eq in eq_recent:\n", + " eq_datetime.append(eq.find('td', attrs={'class': 'tabev6'}).find('a').get_text().replace('\\xa0\\xa0\\xa0', ' '))\n", + " lat_value = eq.find_all('td', attrs={'class': 'tabev1'})[0].get_text(strip=True)\n", + " lat_ns = eq.find_all('td', attrs={'class': 'tabev2'})[0].get_text(strip=True)\n", + " eq_lat.append(lat_value + lat_ns)\n", + " lon_value = eq.find_all('td', attrs={'class': 'tabev1'})[1].get_text(strip=True)\n", + " lon_ns = eq.find_all('td', attrs={'class': 'tabev2'})[1].get_text(strip=True)\n", + " eq_lon.append(lon_value + lon_ns)\n", + " eq_region.append(eq.find('td', attrs={'class': 'tb_region'}).get_text(strip=True))\n", + "\n", + "data_eq_recent = pd.DataFrame({'date_time': eq_datetime, 'latitude': eq_lat, 'longitude': eq_lon, 'region': eq_region})" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "id": "82bcc933-b3cc-4a13-8eda-9e088ec757fd", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.9/site-packages/pandas/core/indexing.py:1637: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " self._setitem_single_block(indexer, value, name)\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
datetimelatitudelongituderegion
02021-10-0912:29:5810.02S118.35ESOUTH OF SUMBAWA, INDONESIA
12021-10-0912:19:4912.78N82.86WCARIBBEAN SEA
22021-10-0912:17:1128.57N17.85WCANARY ISLANDS, SPAIN REGION
32021-10-0912:13:0012.37N87.69WNEAR COAST OF NICARAGUA
42021-10-0911:53:1028.58N17.86WCANARY ISLANDS, SPAIN REGION
52021-10-0911:40:4836.58N27.24EDODECANESE IS.-TURKEY BORDER REG
62021-10-0911:25:5428.56N17.84WCANARY ISLANDS, SPAIN REGION
72021-10-0911:25:0915.64S71.94WSOUTHERN PERU
82021-10-0911:17:3428.54N17.86WCANARY ISLANDS, SPAIN REGION
92021-10-0911:17:1346.15N11.80ENORTHERN ITALY
102021-10-0911:10:497.92N77.55WPANAMA-COLOMBIA BORDER REGION
112021-10-0910:58:4335.43S141.10EVICTORIA, AUSTRALIA
122021-10-0910:39:144.50S153.41ENEW IRELAND REGION, P.N.G.
132021-10-0910:55:0231.52S71.73WOFFSHORE COQUIMBO, CHILE
142021-10-0910:32:1728.57N17.86WCANARY ISLANDS, SPAIN REGION
152021-10-0910:33:3628.57N17.84WCANARY ISLANDS, SPAIN REGION
162021-10-0910:22:3631.53S71.72WOFFSHORE COQUIMBO, CHILE
172021-10-0910:27:0928.57N17.87WCANARY ISLANDS, SPAIN REGION
182021-10-0910:14:0241.15N20.06EALBANIA
192021-10-0910:20:2721.16S68.86WTARAPACA, CHILE
\n", + "
" + ], + "text/plain": [ + " date time latitude longitude region\n", + "0 2021-10-09 12:29:58 10.02S 118.35E SOUTH OF SUMBAWA, INDONESIA\n", + "1 2021-10-09 12:19:49 12.78N 82.86W CARIBBEAN SEA\n", + "2 2021-10-09 12:17:11 28.57N 17.85W CANARY ISLANDS, SPAIN REGION\n", + "3 2021-10-09 12:13:00 12.37N 87.69W NEAR COAST OF NICARAGUA\n", + "4 2021-10-09 11:53:10 28.58N 17.86W CANARY ISLANDS, SPAIN REGION\n", + "5 2021-10-09 11:40:48 36.58N 27.24E DODECANESE IS.-TURKEY BORDER REG\n", + "6 2021-10-09 11:25:54 28.56N 17.84W CANARY ISLANDS, SPAIN REGION\n", + "7 2021-10-09 11:25:09 15.64S 71.94W SOUTHERN PERU\n", + "8 2021-10-09 11:17:34 28.54N 17.86W CANARY ISLANDS, SPAIN REGION\n", + "9 2021-10-09 11:17:13 46.15N 11.80E NORTHERN ITALY\n", + "10 2021-10-09 11:10:49 7.92N 77.55W PANAMA-COLOMBIA BORDER REGION\n", + "11 2021-10-09 10:58:43 35.43S 141.10E VICTORIA, AUSTRALIA\n", + "12 2021-10-09 10:39:14 4.50S 153.41E NEW IRELAND REGION, P.N.G.\n", + "13 2021-10-09 10:55:02 31.52S 71.73W OFFSHORE COQUIMBO, CHILE\n", + "14 2021-10-09 10:32:17 28.57N 17.86W CANARY ISLANDS, SPAIN REGION\n", + "15 2021-10-09 10:33:36 28.57N 17.84W CANARY ISLANDS, SPAIN REGION\n", + "16 2021-10-09 10:22:36 31.53S 71.72W OFFSHORE COQUIMBO, CHILE\n", + "17 2021-10-09 10:27:09 28.57N 17.87W CANARY ISLANDS, SPAIN REGION\n", + "18 2021-10-09 10:14:02 41.15N 20.06E ALBANIA\n", + "19 2021-10-09 10:20:27 21.16S 68.86W TARAPACA, CHILE" + ] + }, + "execution_count": 73, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import datetime\n", + "\n", + "data_eq_recent['date_time'] = pd.to_datetime(data_eq_recent['date_time'], errors='coerce').round('1s')\n", + "\n", + "data_eq_recent.insert(1, 'date', np.array(data_eq_recent['date_time']), True)\n", + "data_eq_recent.insert(2, 'time', np.array(data_eq_recent['date_time']), True)\n", + "\n", + "for i in range(len(data_eq_recent)):\n", + " data_eq_recent['date'].iloc[i] = data_eq_recent['date'].iloc[i].date()\n", + " data_eq_recent['time'].iloc[i] = data_eq_recent['time'].iloc[i].time()\n", + "\n", + "data_eq_recent = data_eq_recent.drop(['date_time'], axis=1)\n", + "data_eq_recent" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f100256-a763-4e59-af17-ca46f6b9956a", + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "id": "dominican-defeat", @@ -101,14 +854,496 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 94, "id": "metric-vertex", "metadata": {}, "outputs": [], "source": [ - "url = 'https://www.wikipedia.org/'" + "url_wiki = 'https://www.wikipedia.org/'" + ] + }, + { + "cell_type": "code", + "execution_count": 95, + "id": "8f788a39-efd3-4c55-b8ac-4731ec6b990c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 95, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response = requests.get(url_wiki)\n", + "response.status_code" + ] + }, + { + "cell_type": "code", + "execution_count": 96, + "id": "1ac5922d-81c2-419b-a8ba-bcca700bb3f6", + "metadata": {}, + "outputs": [], + "source": [ + "soup_wiki = BeautifulSoup(response.content, \"html.parser\")" + ] + }, + { + "cell_type": "code", + "execution_count": 99, + "id": "9f80b3e2-a493-4eb6-bcfc-317e5ada7164", + "metadata": {}, + "outputs": [], + "source": [ + "wiki_lang_box = soup_wiki.find('div', attrs={'class': 'lang-list-content'}).find_all('a')" + ] + }, + { + "cell_type": "code", + "execution_count": 106, + "id": "5951d2f9-fb70-4de7-94f0-3ba5e0ab8d80", + "metadata": {}, + "outputs": [], + "source": [ + "languages = []\n", + "links = []\n", + "\n", + "for lang in wiki_lang_box:\n", + " languages.append(lang.get_text())\n", + " links.append(lang.get('href'))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 107, + "id": "f6a14b5f-7d49-4c11-9ae2-b053d4e4727d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "283\n", + "283\n" + ] + } + ], + "source": [ + "languages.pop()\n", + "links.pop()\n", + "\n", + "print(len(languages))\n", + "print(len(links))" + ] + }, + { + "cell_type": "code", + "execution_count": 123, + "id": "bfece3c1-f482-4459-a098-25a174a42aff", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['1492364',\n", + " '1139345',\n", + " '2621137',\n", + " '6390612',\n", + " '1720717',\n", + " '2365476',\n", + " '1720285',\n", + " '1344762',\n", + " '2068443',\n", + " '1293875',\n", + " '1074970',\n", + " '1759782',\n", + " '5978282',\n", + " '2910327',\n", + " '1117249',\n", + " '1269206',\n", + " '1265517',\n", + " '1234298',\n", + " '100472',\n", + " '237732',\n", + " '128117',\n", + " '178974',\n", + " '275428',\n", + " '430925',\n", + " '১১৩৮৭২',\n", + " '209479',\n", + " '688960',\n", + " '489527',\n", + " '133678',\n", + " '269980',\n", + " '222423',\n", + " '198025',\n", + " '304417',\n", + " '381061',\n", + " '۸۳۸۷۲۸',\n", + " '176793',\n", + " '288319',\n", + " '149800',\n", + " '209873',\n", + " '598814',\n", + " '304081',\n", + " '156854',\n", + " '136103',\n", + " '109097',\n", + " '199845',\n", + " '492878',\n", + " '117310',\n", + " '354093',\n", + " '224799',\n", + " '566042',\n", + " '159794',\n", + " '434648',\n", + " '139838',\n", + " '229202',\n", + " '423443',\n", + " '196952',\n", + " '173744',\n", + " '650052',\n", + " '455681',\n", + " '517514',\n", + " '141338',\n", + " '319993',\n", + " '141081',\n", + " '104330',\n", + " '۲۴۰۸۳۷',\n", + " '429527',\n", + " '165280',\n", + " '127007',\n", + " '119954',\n", + " '၁၀၂၁၁၆',\n", + " '554763',\n", + " '12520',\n", + " '27992',\n", + " '14987',\n", + " '40520',\n", + " '13707',\n", + " '58630',\n", + " '76165',\n", + " '11404',\n", + " '২৫০৮৮',\n", + " '31700',\n", + " '87628',\n", + " '70062',\n", + " '46817',\n", + " '17119',\n", + " '12923',\n", + " '13630',\n", + " '46598',\n", + " '55670',\n", + " '15534',\n", + " '૨૯૬૮૯',\n", + " '12531',\n", + " '13762',\n", + " '30493',\n", + " '15349',\n", + " '23234',\n", + " '14158',\n", + " '53279',\n", + " '64405',\n", + " '೨೭೩೦೩',\n", + " '63375',\n", + " '50172',\n", + " '٣٤٨٩٣',\n", + " '80859',\n", + " '10291',\n", + " '60253',\n", + " '13490',\n", + " '49362',\n", + " '10776',\n", + " '१३६८६',\n", + " '94205',\n", + " '75780',\n", + " '一一〇〇五',\n", + " '८०७३०',\n", + " '16607',\n", + " '۱۳۳۶۲',\n", + " '15504',\n", + " '20828',\n", + " '७२३४८',\n", + " '३२१६८',\n", + " '14686',\n", + " '14203',\n", + " '86906',\n", + " '10242',\n", + " '୧୫୮୬୦',\n", + " '36361',\n", + " '۶۳۹۶۶',\n", + " '۱۲۴۵۰',\n", + " '65857',\n", + " '82741',\n", + " '12006',\n", + " '22937',\n", + " '११५२९',\n", + " '13417',\n", + " '40452',\n", + " '83983',\n", + " '26180',\n", + " '17445',\n", + " '١٤٧١٦',\n", + " '54364',\n", + " '60964',\n", + " '67336',\n", + " '43438',\n", + " '73138',\n", + " '14144',\n", + " '68650',\n", + " '11281',\n", + " '42673',\n", + " '15284',\n", + " '33480',\n", + " '40045',\n", + " '16914',\n", + " '10129',\n", + " '1592',\n", + " '3369',\n", + " '1429',\n", + " '5807',\n", + " '9065',\n", + " '1264',\n", + " '5165',\n", + " '2347',\n", + " '1779',\n", + " '4400',\n", + " '2544',\n", + " '4960',\n", + " '9659',\n", + " '4404',\n", + " '7559',\n", + " '1386',\n", + " '༥༩༦༣',\n", + " '2737',\n", + " '3150',\n", + " '6068',\n", + " '1974',\n", + " '7770',\n", + " '1926',\n", + " '3303',\n", + " '6582',\n", + " '3299',\n", + " '10076',\n", + " '3450',\n", + " '5048',\n", + " '2794',\n", + " '1501',\n", + " '۶۳۴۲',\n", + " '6454',\n", + " '9422',\n", + " '2094',\n", + " '2391',\n", + " '8411',\n", + " '8864',\n", + " '5394',\n", + " '4732',\n", + " '៨៣៥៨',\n", + " '2271',\n", + " '5461',\n", + " '1229',\n", + " '3543',\n", + " '1046',\n", + " '໓໘໖໔',\n", + " '3605',\n", + " '1234',\n", + " '1010',\n", + " '4213',\n", + " '3234',\n", + " '1273',\n", + " '1362',\n", + " '4271',\n", + " '1214',\n", + " '7235',\n", + " '3862',\n", + " '1491',\n", + " '1218',\n", + " '7016',\n", + " '1652',\n", + " '7374',\n", + " '4581',\n", + " '1519',\n", + " '1078',\n", + " '9684',\n", + " '२५६२',\n", + " '2563',\n", + " '2222',\n", + " '3445',\n", + " '2732',\n", + " '5252',\n", + " '2046',\n", + " '1906',\n", + " '2911',\n", + " '3’720',\n", + " '8186',\n", + " '1017',\n", + " '7094',\n", + " '4048',\n", + " '8441',\n", + " '7352',\n", + " '6610',\n", + " '1089',\n", + " '6208',\n", + " '9305',\n", + " '1490',\n", + " '1633',\n", + " '1769',\n", + " '6178',\n", + " '3282',\n", + " '5114',\n", + " '5087',\n", + " '6676',\n", + " '5877',\n", + " '7477',\n", + " '1649',\n", + " '1211',\n", + " '4761',\n", + " '3009',\n", + " '߁߀߄߈',\n", + " '720',\n", + " '530',\n", + " '826',\n", + " '382',\n", + " '360',\n", + " '848',\n", + " '574',\n", + " '399',\n", + " '810',\n", + " '۸۶۸',\n", + " '157',\n", + " '879',\n", + " '485',\n", + " '༢༢༢',\n", + " '703',\n", + " '620',\n", + " '278',\n", + " '819',\n", + " '722',\n", + " '798',\n", + " '542',\n", + " '977',\n", + " '621',\n", + " '634',\n", + " '709',\n", + " '598',\n", + " '950',\n", + " '215']" + ] + }, + "execution_count": 123, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "no_of_articles = []\n", + "\n", + "for lnk in links:\n", + " stats_page = 'https:' + lnk + 'wiki/Special:Statistics'\n", + " response = requests.get(stats_page)\n", + " soup_stats = BeautifulSoup(response.content, \"html.parser\")\n", + " no_of_articles.append(soup_stats.find('tr', attrs={'class': 'mw-statistics-articles'}).find('td', attrs={'class': 'mw-statistics-numbers'}).get_text().replace('\\xa0', '').replace(',', '').replace('.', '').replace('٬', ''))\n", + "\n", + "no_of_articles\n", + " " ] }, + { + "cell_type": "code", + "execution_count": 124, + "id": "44908453-1082-440b-82dc-4e625b6b48b5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
languageno_of_articles
0Polski1492364
1العربية1139345
2Deutsch2621137
3English6390612
4Español1720717
\n", + "
" + ], + "text/plain": [ + " language no_of_articles\n", + "0 Polski 1492364\n", + "1 العربية 1139345\n", + "2 Deutsch 2621137\n", + "3 English 6390612\n", + "4 Español 1720717" + ] + }, + "execution_count": 124, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_wiki_languages = pd.DataFrame({'language': languages, 'no_of_articles': no_of_articles})\n", + "data_wiki_languages.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aaf83e5f-e7b1-404a-b896-641984179b69", + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "id": "split-cartridge", @@ -119,14 +1354,211 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 141, "id": "actual-parallel", "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 141, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "url_govuk = 'https://data.gov.uk/'\n", + "\n", + "response = requests.get(url_govuk)\n", + "response.status_code" + ] + }, + { + "cell_type": "code", + "execution_count": 146, + "id": "0b4dedc2-ec17-4bbe-a777-de162fd04fa4", + "metadata": {}, + "outputs": [], + "source": [ + "soup_govuk = BeautifulSoup(response.content, \"html.parser\")\n", + "\n", + "datasets_categories = soup_govuk.find('ul', attrs={'class': 'govuk-list dgu-topics__list'}).find_all('a', attrs={'class': 'govuk-link'})\n", + "datasets_info = soup_govuk.find('ul', attrs={'class': 'govuk-list dgu-topics__list'}).find_all('p', attrs={'class': 'govuk-body'})" + ] + }, + { + "cell_type": "code", + "execution_count": 149, + "id": "fb56c735-430b-4e77-98d6-d407876b64e5", + "metadata": {}, "outputs": [], "source": [ - "url = 'https://data.gov.uk/" + "for i in range(len(datasets_categories)):\n", + " datasets_categories[i] = datasets_categories[i].get_text()\n", + " datasets_info[i] = datasets_info[i].get_text()" + ] + }, + { + "cell_type": "code", + "execution_count": 151, + "id": "b13a430d-2767-46cc-9800-d10e72ecbd05", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
datasetsdescription
0Business and economySmall businesses, industry, imports, exports a...
1Crime and justiceCourts, police, prison, offenders, borders and...
2DefenceArmed forces, health and safety, search and re...
3EducationStudents, training, qualifications and the Nat...
4EnvironmentWeather, flooding, rivers, air quality, geolog...
5GovernmentStaff numbers and pay, local councillors and d...
6Government spendingIncludes all payments by government department...
7HealthIncludes smoking, drugs, alcohol, medicine per...
8MappingAddresses, boundaries, land ownership, aerial ...
9SocietyEmployment, benefits, household finances, pove...
10Towns and citiesIncludes housing, urban planning, leisure, was...
11TransportAirports, roads, freight, electric vehicles, p...
12Digital service performanceCost, usage, completion rate, digital take-up,...
13Government reference dataTrusted data that is referenced and shared acr...
\n", + "
" + ], + "text/plain": [ + " datasets \\\n", + "0 Business and economy \n", + "1 Crime and justice \n", + "2 Defence \n", + "3 Education \n", + "4 Environment \n", + "5 Government \n", + "6 Government spending \n", + "7 Health \n", + "8 Mapping \n", + "9 Society \n", + "10 Towns and cities \n", + "11 Transport \n", + "12 Digital service performance \n", + "13 Government reference data \n", + "\n", + " description \n", + "0 Small businesses, industry, imports, exports a... \n", + "1 Courts, police, prison, offenders, borders and... \n", + "2 Armed forces, health and safety, search and re... \n", + "3 Students, training, qualifications and the Nat... \n", + "4 Weather, flooding, rivers, air quality, geolog... \n", + "5 Staff numbers and pay, local councillors and d... \n", + "6 Includes all payments by government department... \n", + "7 Includes smoking, drugs, alcohol, medicine per... \n", + "8 Addresses, boundaries, land ownership, aerial ... \n", + "9 Employment, benefits, household finances, pove... \n", + "10 Includes housing, urban planning, leisure, was... \n", + "11 Airports, roads, freight, electric vehicles, p... \n", + "12 Cost, usage, completion rate, digital take-up,... \n", + "13 Trusted data that is referenced and shared acr... " + ] + }, + "execution_count": 151, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_govuk = pd.DataFrame({'datasets': datasets_categories, 'description': datasets_info})\n", + "data_govuk" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "deea2a38-fb76-48d1-9f21-b8b9b6db841a", + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "id": "potential-malpractice", @@ -137,20 +1569,178 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "adaptive-calculator", "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "url_top10lang = 'https://en.wikipedia.org/wiki/List_of_languages_by_number_of_native_speakers'\n", + "\n", + "response = requests.get(url_top10lang)\n", + "response.status_code" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "2f2dac48-86b6-4727-8aa4-d48c9f55c67a", + "metadata": {}, "outputs": [], "source": [ - "url = 'https://en.wikipedia.org/wiki/List_of_languages_by_number_of_native_speakers'" + "soup_top10lang = BeautifulSoup(response.content, \"html.parser\")\n", + "\n", + "table_languages = soup_top10lang.find_all('table')[1].find('tbody').find_all('tr')\n" ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "96f5ffca-bb7b-466b-aa6e-ae5f15185b7b", + "metadata": {}, + "outputs": [], + "source": [ + "lang_native, lang_no_of_speakers = [], []\n", + "\n", + "for i in range(1, 11):\n", + " lang_native.append(table_languages[i].find_all('td')[1].get_text())\n", + " lang_no_of_speakers.append(table_languages[i].find_all('td')[2].get_text().replace('\\n',''))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "506057f1-87b1-4b4e-8847-257bbd74d3c6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
languageno_of_native_speakers_millions
0Mandarin Chinese\\n918
1Spanish\\n480
2English\\n379
3Hindi (sanskritised Hindustani)[9]\\n341
4Bengali\\n300
5Portuguese\\n221
6Russian\\n154
7Japanese\\n128
8Western Punjabi[10]\\n92.7
9Marathi\\n83.1
\n", + "
" + ], + "text/plain": [ + " language no_of_native_speakers_millions\n", + "0 Mandarin Chinese\\n 918\n", + "1 Spanish\\n 480\n", + "2 English\\n 379\n", + "3 Hindi (sanskritised Hindustani)[9]\\n 341\n", + "4 Bengali\\n 300\n", + "5 Portuguese\\n 221\n", + "6 Russian\\n 154\n", + "7 Japanese\\n 128\n", + "8 Western Punjabi[10]\\n 92.7\n", + "9 Marathi\\n 83.1" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_top10lang = pd.DataFrame({'language': lang_native, 'no_of_native_speakers_millions': lang_no_of_speakers})\n", + "data_top10lang" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c749a767-354c-444a-bd54-8326bb13593e", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python3Brew", "language": "python", - "name": "python3" + "name": "python3brew" }, "language_info": { "codemirror_mode": { @@ -162,7 +1752,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.9.4" }, "toc": { "base_numbering": 1,