diff --git a/8.05_Lab_API_wrappers.ipynb b/8.05_Lab_API_wrappers.ipynb new file mode 100644 index 0000000..cc38000 --- /dev/null +++ b/8.05_Lab_API_wrappers.ipynb @@ -0,0 +1,725 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "e2841f2a-fa89-4d98-a96a-6704a645752d", + "metadata": {}, + "source": [ + "# Lab | API wrappers - Create your collection of songs & audio features\n", + "\n", + "**Instructions**\n", + "\n", + "To move forward with the project, you need to create a collection of songs with their audio features - as large as possible!\n", + "\n", + "These are the songs that we will cluster. And, later, when the user inputs a song, we will find the cluster to which the song belongs and recommend a song from the same cluster. The more songs you have, the more accurate and diverse recommendations you'll be able to give. Although... you might want to make sure the collected songs are \"curated\" in a certain way. Try to find playlists of songs that are diverse, but also that meet certain standards.\n", + "\n", + "The process of sending hundreds or thousands of requests can take some time - it's normal if you have to wait a few minutes (or, if you're ambitious, even hours) to get all the data you need.\n", + "\n", + "An idea for collecting as many songs as possible is to start with all the songs of a big, diverse playlist and then go to every artist present in the playlist and grab every song of every album of that artist. The amount of songs you'll be collecting per playlist will grow exponentially!" + ] + }, + { + "cell_type": "markdown", + "id": "eb827ee6-feb1-4d06-a523-abb4960489d5", + "metadata": {}, + "source": [ + "### 1. Libary imports" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "id": "74dd2ed8-96bb-4eee-94df-cb30e1093dcc", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import requests\n", + "import json\n", + "import time\n", + "import random\n", + "import spotipy\n", + "from spotipy.oauth2 import SpotifyClientCredentials" + ] + }, + { + "cell_type": "markdown", + "id": "60e993c2-d258-41c8-b56e-5293bc7eec0f", + "metadata": {}, + "source": [ + "### 2. Authentification" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "id": "8ffbbbdf-5123-4741-b787-80866c5d32c7", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize SpotiPy with user credentias\n", + "\n", + "sp = spotipy.Spotify(auth_manager=SpotifyClientCredentials(client_id=\"af3a4e21d9974f798b0ddef081728f2b\",\n", + " client_secret=\"99a65d20eff04d64bcf24b11824dffc4\"))\n" + ] + }, + { + "cell_type": "markdown", + "id": "975e0932-87e4-4822-8ae6-3e4acb7d1e3a", + "metadata": {}, + "source": [ + "### 3. Playlists fetching\n", + "### 3.1. Featured playlists" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "id": "c49c0b92-fd42-491a-a09d-8895da687093", + "metadata": {}, + "outputs": [], + "source": [ + "# get featured playlists from spotify \n", + "# offset not worlking, maybe only 10 playlists availible atm\n", + "# inputs: number of playlists, outputs: list of playlist ids, df with playlist names and ids\n", + "\n", + "def get_featured_playlists(n_playlists):\n", + " playlist_names, playlist_ids, sublist_names, sublist_ids = [], [], [], []\n", + " # fetch 10 playlists at a time via api\n", + " for ofs in range(0,n_playlists,10):\n", + " featured_pl_10 = sp.featured_playlists(locale='en_US', country='US', offset=ofs)\n", + " \n", + " # get all playlists of each fetch\n", + " for i in range(len(featured_pl_10['playlists']['items'])):\n", + " sublist_ids.append(featured_pl_10['playlists']['items'][i]['id'])\n", + " sublist_names.append(featured_pl_10['playlists']['items'][i]['name'])\n", + " \n", + " # fill output lists\n", + " playlist_ids += sublist_ids\n", + " playlist_names += sublist_names\n", + " time.sleep(2)\n", + " \n", + " # overview df\n", + " featured_playlists_total = pd.DataFrame({'name': playlist_names, 'ids': playlist_ids})\n", + " return playlist_ids, featured_playlists_total" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "id": "5ee649a8-1451-42cc-95a5-c62fdc630811", + "metadata": {}, + "outputs": [], + "source": [ + "featured_10_ids, featured_10 = get_featured_playlists(n_playlists=10)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "id": "f978e0ef-9c02-468c-9f7d-333b20074d9c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['37i9dQZF1DX4JAvHpjipBk',\n", + " '37i9dQZF1DX3LyU0mhfqgP',\n", + " '37i9dQZF1DXcRXFNfZr7Tp',\n", + " '37i9dQZF1DWUxHPh2rEiHr',\n", + " '37i9dQZF1DWZryfp6NSvtz',\n", + " '37i9dQZF1DX8CopunbDxgW',\n", + " '37i9dQZF1DX0bUGQdz5BJG',\n", + " '37i9dQZF1DWUzFXarNiofw',\n", + " '37i9dQZF1DWT6SJaitNDax',\n", + " '37i9dQZF1DWZoF06RIo9el']" + ] + }, + "execution_count": 64, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "featured_10_ids" + ] + }, + { + "cell_type": "markdown", + "id": "57af16ff-cb82-4e58-9b4b-ced7bec9c2b6", + "metadata": {}, + "source": [ + "### 3.2 User playlists (alternative)" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "id": "11cb011f-33d9-4aff-a647-b91863f57730", + "metadata": {}, + "outputs": [], + "source": [ + "# get playlists from spotify user\n", + "# offset not worlking, maybe only 10 playlists availible atm\n", + "# inputs: user id, optional: number of playlists, outputs: list of playlist ids, df with playlist names and ids\n", + "\n", + "def get_user_playlists(user_id, n_playlists=-1):\n", + " playlist_names, playlist_ids = [], []\n", + " \n", + " # fetch first 20 playlists \n", + " results = sp.user_playlists(user=user_id, limit=20)\n", + " user_playlists = results['items']\n", + " time.sleep(2)\n", + " \n", + " # fetch playlist 21 - end\n", + " for ofs in range(20,n_playlists,20):\n", + " results = sp.user_playlists(user=user_id, limit=20, offset=ofs)\n", + " user_playlists += results['items']\n", + " time.sleep(2)\n", + " \n", + " # option to limit the no of playlists retreived\n", + " if n_playlists > 0:\n", + " if n_playlists < results['total']:\n", + " user_playlists = user_playlists[:n_playlists]\n", + " \n", + " # extract name and id\n", + " for pl in user_playlists:\n", + " playlist_ids.append(pl['id'])\n", + " playlist_names.append(pl['name'])\n", + " playlist_ids = playlist_ids[:n_playlists]\n", + " playlist_names = playlist_names[:n_playlists]\n", + " \n", + " # overview df\n", + " user_playlists_total = pd.DataFrame({'name': playlist_names, 'ids': playlist_ids})\n", + " return playlist_ids, user_playlists_total" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "id": "0d423a90-7dc9-4907-aab9-6c2afc8a948e", + "metadata": {}, + "outputs": [], + "source": [ + "# get offiacal spotify playlists\n", + "# number of playlists, outputs: list of playlist ids, df with playlist names and ids\n", + "\n", + "def get_spotify_playlists(n_playlists):\n", + " spo_playlist_ids, spo_playlists_total = get_user_playlists(user_id='spotify', n_playlists=n_playlists)\n", + " return spo_playlist_ids, spo_playlists_total" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "id": "96e3ad2d-4e2a-446d-b29b-9aabbd0c8326", + "metadata": {}, + "outputs": [], + "source": [ + "spotify_playlist_ids, spotify_playlists_total = get_spotify_playlists(n_playlists=30)" + ] + }, + { + "cell_type": "code", + "execution_count": 75, + "id": "0624f5ac-f69f-4f44-a420-f9e7f1a53c2b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
nameids
0Today's Top Hits37i9dQZF1DXcBWIGoYBM5M
1RapCaviar37i9dQZF1DX0XUsuxWHRQd
2Hot Country37i9dQZF1DX1lVhptIYRda
3¡Viva Latino!37i9dQZF1DX10zKzsJ2jva
4New Music Friday37i9dQZF1DX4JAvHpjipBk
5Peaceful Piano37i9dQZF1DX4sWSpwq3LiO
6Are & Be37i9dQZF1DX4SBhb3fqCJd
7Rock Classics37i9dQZF1DWXRqgorJj26U
8mint37i9dQZF1DX4dyzvuaRJ0n
9Rock This37i9dQZF1DXcF6B6QPhFDv
10New Music Friday37i9dQZF1DWXJfnUiYjUKT
11just hits37i9dQZF1DXcRXFNfZr7Tp
12All Out 00s37i9dQZF1DX4o1oenSJRJd
13All Out 90s37i9dQZF1DXbTxeAdrVG2l
14All Out 80s37i9dQZF1DX4UtSsGT1Sbe
15All Out 70s37i9dQZF1DWTJ7xPn4vNaz
16All Out 60s37i9dQZF1DXaKIA8E7WcJj
17All Out 50s37i9dQZF1DWSV3Tk4GO2fq
18Soft Pop Hits37i9dQZF1DWTwnEm1IYyoj
19Signed XOXO37i9dQZF1DX2A29LI7xHn1
20Most Necessary37i9dQZF1DX2RxBh64BHjQ
21Gold School37i9dQZF1DWVA1Gq4XHa6U
22Get Turnt37i9dQZF1DWY4xHQp97fN6
23B.A.E.37i9dQZF1DWX3387IZmjNa
24African Heat37i9dQZF1DWYkaDif7Ztbp
25Mind Right37i9dQZF1DX5hR0J49CmXC
26Dancehall Official37i9dQZF1DXan38dNVDdl4
27Westside Story37i9dQZF1DWSvKsRPPnv5o
28Power Workout37i9dQZF1DWUVpAXiEPK8P
29No Cap37i9dQZF1DX0Tkc6ltcBfU
\n", + "
" + ], + "text/plain": [ + " name ids\n", + "0 Today's Top Hits 37i9dQZF1DXcBWIGoYBM5M\n", + "1 RapCaviar 37i9dQZF1DX0XUsuxWHRQd\n", + "2 Hot Country 37i9dQZF1DX1lVhptIYRda\n", + "3 ¡Viva Latino! 37i9dQZF1DX10zKzsJ2jva\n", + "4 New Music Friday 37i9dQZF1DX4JAvHpjipBk\n", + "5 Peaceful Piano 37i9dQZF1DX4sWSpwq3LiO\n", + "6 Are & Be 37i9dQZF1DX4SBhb3fqCJd\n", + "7 Rock Classics 37i9dQZF1DWXRqgorJj26U\n", + "8 mint 37i9dQZF1DX4dyzvuaRJ0n\n", + "9 Rock This 37i9dQZF1DXcF6B6QPhFDv\n", + "10 New Music Friday 37i9dQZF1DWXJfnUiYjUKT\n", + "11 just hits 37i9dQZF1DXcRXFNfZr7Tp\n", + "12 All Out 00s 37i9dQZF1DX4o1oenSJRJd\n", + "13 All Out 90s 37i9dQZF1DXbTxeAdrVG2l\n", + "14 All Out 80s 37i9dQZF1DX4UtSsGT1Sbe\n", + "15 All Out 70s 37i9dQZF1DWTJ7xPn4vNaz\n", + "16 All Out 60s 37i9dQZF1DXaKIA8E7WcJj\n", + "17 All Out 50s 37i9dQZF1DWSV3Tk4GO2fq\n", + "18 Soft Pop Hits 37i9dQZF1DWTwnEm1IYyoj\n", + "19 Signed XOXO 37i9dQZF1DX2A29LI7xHn1\n", + "20 Most Necessary 37i9dQZF1DX2RxBh64BHjQ\n", + "21 Gold School 37i9dQZF1DWVA1Gq4XHa6U\n", + "22 Get Turnt 37i9dQZF1DWY4xHQp97fN6\n", + "23 B.A.E. 37i9dQZF1DWX3387IZmjNa\n", + "24 African Heat 37i9dQZF1DWYkaDif7Ztbp\n", + "25 Mind Right 37i9dQZF1DX5hR0J49CmXC\n", + "26 Dancehall Official 37i9dQZF1DXan38dNVDdl4\n", + "27 Westside Story 37i9dQZF1DWSvKsRPPnv5o\n", + "28 Power Workout 37i9dQZF1DWUVpAXiEPK8P\n", + "29 No Cap 37i9dQZF1DX0Tkc6ltcBfU" + ] + }, + "execution_count": 75, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "spotify_playlists_total" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ede93298-e0c4-440b-8dbb-241a9bf48a79", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "e0a8eb83-9529-4501-a5ad-3b42f3a3cb55", + "metadata": {}, + "source": [ + "### 4. Artists fetching" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "id": "b9bf0300-bea3-4f6b-a99e-ea60a079ed35", + "metadata": {}, + "outputs": [], + "source": [ + "# fetch all artists present in a playlist\n", + "# inputs: playlist id, outputs: list of artist ids\n", + "\n", + "def get_artists_from_playlist(plst_id):\n", + " # get first 100 tracks from playlist\n", + " playlist = sp.playlist_tracks(playlist_id=plst_id)\n", + " tracks = playlist['items']\n", + " artist_ids = []\n", + "\n", + " # get track 101 - end from playlist\n", + " for ofs in range(100,playlist['total'],100):\n", + " playlist = sp.playlist_tracks(playlist_id=plst_id, offset=ofs)\n", + " time.sleep(2)\n", + " tracks += playlist['items']\n", + " \n", + " # extract artist ids\n", + " for track in tracks:\n", + " for a in track['track']['artists']:\n", + " artist_ids.append(a['id'])\n", + " \n", + " return list(set(artist_ids))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "id": "5fe02500-a5e4-4a18-b080-ca31b773a18f", + "metadata": {}, + "outputs": [], + "source": [ + "artist_ids = []\n", + "\n", + "for playlist_id in spotify_playlist_ids[11:15]:\n", + " pl_artist_ids = get_artists_from_playlist(plst_id=playlist_id)\n", + " artist_ids += pl_artist_ids\n" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "id": "fa2da1ab-12b6-4d7e-a909-a1e7638cdb19", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "468" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(artist_ids)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13ec4f70-13c5-4ab0-9529-ee13a58654e3", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "54229c81-b4fd-45de-a7b0-3004ae3479ff", + "metadata": {}, + "source": [ + "### 5. Fetching Tracks from Artists" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "id": "ec09a122-bf54-4a63-b584-03c9d5289ad3", + "metadata": {}, + "outputs": [], + "source": [ + "# pick random items from multiple lists of the same length, pick items at the same positions in each list\n", + "# inputs: list of lists, number of items, outputs: random list\n", + "\n", + "def select_random_list_items(lists, n_items):\n", + " if len(lists[0]) > n_items:\n", + " new_lists = []\n", + " items_selected = random.sample(range(0, len(lists[0])-1), n_items)\n", + " for l in lists:\n", + " list_selected = []\n", + " for x in items_selected:\n", + " list_selected.append(l[x])\n", + " l = list_selected\n", + " new_lists.append(l)\n", + " return new_lists" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "id": "ce9d9a5a-a222-4a3d-b00a-5131107c336b", + "metadata": {}, + "outputs": [], + "source": [ + "# fetch albums from artists (ToDo: max. 20 per artist)\n", + "# inputs: artist id, outputs: list of albums\n", + "\n", + "def get_albums_from_artists(art_id):\n", + " # get first 20 albums from artist\n", + " results = sp.artist_albums(artist_id=art_id, album_type='album', limit=20)\n", + " albums = results['items']\n", + " album_ids, album_names = [], []\n", + "\n", + " # get album 21 - end from artist\n", + " for ofs in range(20,results['total'],20):\n", + " results = sp.artist_albums(artist_id=art_id, album_type='album', limit=20, offset=ofs)\n", + " albums += results['items']\n", + " time.sleep(3)\n", + " \n", + " # extract album ids\n", + " for idx, album in enumerate(albums):\n", + " if albums[idx]['id'] not in album_ids:\n", + " album_ids.append(album['id'])\n", + " album_names.append(album['name'])\n", + " \n", + " return album_ids" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "ce7fdb98-ff61-4f6c-8cac-36adc05c13be", + "metadata": {}, + "outputs": [], + "source": [ + "# fetch tracks from artists albums\n", + "# inputs: album id, outputs: list of albums\n", + "\n", + "def get_tracks_from_artists_albums(art_id, n_items=-1):\n", + " # lists for tracks fetch and outputs\n", + " tracks, track_ids, track_names, artists = [], [], [], []\n", + " # fetch albums from artists\n", + " artist_albums_list = get_albums_from_artists(art_id=art_id)\n", + " \n", + " for abm_id in artist_albums_list:\n", + " # get first 50 tracks from album\n", + " results = sp.album_tracks(album_id=abm_id, limit=50)\n", + " tracks += results['items']\n", + " time.sleep(2)\n", + " \n", + " # get track 51 - end from album\n", + " for ofs in range(50,results['total'],50):\n", + " results = sp.album_tracks(album_id=abm_id, limit=50, offset=ofs)\n", + " tracks += results['items']\n", + " time.sleep(2)\n", + "\n", + " # extract track info\n", + " if len(artist_albums_list) > 0:\n", + " for idx, track in enumerate(tracks):\n", + " if tracks[idx]['name'] not in track_names:\n", + " # track id\n", + " track_ids.append(track['id'])\n", + " # track name\n", + " track_names.append(track['name'])\n", + " # track artists\n", + " artist_name = ''\n", + " for a in track['artists']:\n", + " artist_name += (a['name'] + ', ')\n", + " artist_name.strip().strip(',')\n", + " artists.append(artist_name)\n", + " \n", + " # df overview\n", + " tracklist_overview = pd.DataFrame({'artists': artists, 'track_name': track_names, 'track_id': track_ids})\n", + " \n", + " # option to select random number of tracks from each artist\n", + " if n_items > 0:\n", + " if n_items < len(tracklist_overview):\n", + " tracklist_overview = tracklist_overview.sample(n_items)\n", + " \n", + " return track_ids, tracklist_overview" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "id": "b6b0f965-3527-4701-afa8-a05df357ac09", + "metadata": {}, + "outputs": [], + "source": [ + "track_ids_10_20, track_names_10_20, artists_10_20 = [], [], []\n", + "many_tracks_overview_10_20 = pd.DataFrame({'artists': artists_10_20, 'track_name': track_names_10_20, 'track_id': track_ids_10_20})\n", + "\n", + "for arst_id in artist_ids[50:70]:\n", + " track_ids, many_tracks_overview = get_tracks_from_artists_albums(art_id=arst_id, n_items=100)\n", + " track_ids_10_20 += track_ids\n", + " many_tracks_overview_10_20 = many_tracks_overview_10_20.append(many_tracks_overview)\n", + " \n", + "many_tracks_overview_10_20 = many_tracks_overview_10_20.reset_index(drop=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "id": "a6f97c6a-1041-47c4-949a-3d60c1d8d28b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "577" + ] + }, + "execution_count": 56, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(track_ids_10_20)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "341118eb-7bf6-490f-8694-b3c7e545f1dc", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python3Brew", + "language": "python", + "name": "python3brew" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}