From ff84a13d3548352b1d584a6d031b6b12d281bd5b Mon Sep 17 00:00:00 2001 From: John42506176Linux Date: Wed, 7 Apr 2021 12:54:44 -0400 Subject: [PATCH] Refactoring+Adding comments + Adding user_genres --- Aptfile | 2 +- Genre_preprocessor.py | 47 ++- app.py | 456 ++++++++++------------------ database/DB.sql | 186 +++++++----- database/ml_database.py | 333 +++++++++++--------- database/spotify_database.py | 358 ++++++++++++++++++++++ spotify_data/genre_ml_classifier.py | 53 +++- spotify_data/spotify.py | 73 ++++- taskrunner/app.py | 35 --- taskrunner/sun_time.py | 14 +- taskrunner/task_app.py | 11 + weather_data/weather.py | 49 ++- 12 files changed, 996 insertions(+), 621 deletions(-) create mode 100644 database/spotify_database.py delete mode 100644 taskrunner/app.py create mode 100644 taskrunner/task_app.py diff --git a/Aptfile b/Aptfile index 35b4544..4b1df2b 100644 --- a/Aptfile +++ b/Aptfile @@ -1,4 +1,4 @@ +python-pyodbc unixodbc unixodbc-dev -python-pyodbc libsqliteodbc \ No newline at end of file diff --git a/Genre_preprocessor.py b/Genre_preprocessor.py index 739428e..2c74ec9 100644 --- a/Genre_preprocessor.py +++ b/Genre_preprocessor.py @@ -12,10 +12,6 @@ from sklearn.neural_network import MLPClassifier from sklearn.base import BaseEstimator,TransformerMixin -column_order = ['popularity', 'acousticness', 'danceability', 'duration_ms', 'energy', - 'instrumentalness', 'key', 'liveness', 'loudness', 'mode', - 'speechiness', 'tempo', 'time_signature', 'valence', 'is_explicit', - 'release_year'] class DropTransformer(BaseEstimator,TransformerMixin): def __init__(self,drop=None): @@ -50,7 +46,7 @@ def top3predictions(preds,model,encoder): def train_pipeline(): data = pd.read_csv('CleanData.csv') full_prep_pipeline = Pipeline([ - ('Drop',DropTransformer('loudness')), + ('Drop',DropTransformer('loudness')), # During the data analysis portion we found loudness was a collinear feature that wasn't really needed so we drop it. ('Scaler',StandardScaler()), ('MLPModel',MLPClassifier(max_iter=300)) ]) @@ -64,7 +60,7 @@ def train_pipeline(): y_pred_label = encoder.inverse_transform(y_pred) y_test_label = encoder.inverse_transform(y_test) metric= metrics.classification_report (y_pred_label,y_test_label) - dump(encoder,'Encoder.joblib') + dump(encoder,'Encoder.joblib') # We have so save the encoder to get back the actual values as the model just prints out integers. dump(full_prep_pipeline,'MLPModelPipeline.joblib') def test_saved_pipeline(): @@ -85,27 +81,30 @@ def test_saved_pipeline(): ''' song_info = { - "danceability": 0.5160, - "energy": 0.2380, - "key": float(4), - "loudness": -18.7100, - "mode": float(0), - "speechiness": 0.0343, - "acousticness": 0.8310, - "instrumentalness": 0.8510, - "liveness": 0.0934, - "valence": 0.0614, - "tempo": float(115), - "duration_ms": float(134507), - "time_signature": float(4), - "Song_name": "Yamanaiame", - "popularity": float(30), - "release_year": float(2018), - "is_explicit": float(0) + "danceability": float, + "energy": float, + "key": float, + "loudness": float, + "mode": int, + "speechiness": float, + "acousticness": float, + "instrumentalness": float, + "liveness": float, + "valence": float, + "tempo": float, + "duration_ms": float, + "time_signature": int, + "Song_name": Str, + "popularity": int, + "release_year": int, + "is_explicit": int } ''' def get_prediction(song_info): - global column_order + column_order = ['popularity', 'acousticness', 'danceability', 'duration_ms', 'energy', + 'instrumentalness', 'key', 'liveness', 'loudness', 'mode', + 'speechiness', 'tempo', 'time_signature', 'valence', 'is_explicit', + 'release_year'] # We need to keep the order in the same way we had it during the training process. pipeline = load('MLPModelPipeline.joblib') encoder = load('Encoder.joblib') if 'Song_name' in song_info: diff --git a/app.py b/app.py index f528efd..15801c4 100644 --- a/app.py +++ b/app.py @@ -4,56 +4,41 @@ import requests from weather_data import weather import json +from flask_apscheduler import APScheduler from taskrunner import sun_time -from database import ml_database +from database import ml_database,spotify_database import requests from datetime import datetime import time -from taskrunner import refresh_token +from taskrunner import refresh_token,task_app app = Flask(__name__) CORS(app) curr_date = datetime.utcnow().strftime("%m/%d/%Y") -@app.route('/api', methods=['GET']) -def api(): - return { - 'userID': 1, - 'title': 'Flask and React Application.', - 'completed': False - } -#Route Testing Data transfer from front end to Back end -@app.route('/data', methods = ['POST','GET']) -def data(): +scheduler = APScheduler() +# if you don't wanna use a config, you can set options here: +# scheduler.api_enabled = True +scheduler.api_enabled = True +scheduler.init_app(app) +scheduler.start() +scheduler.add_job(func=task_app.store,trigger='interval',id='Store',seconds=10) - if 'latitude' not in request.form or 'authKey' not in request.form or 'longitude' not in request.form : - response = app.response_class( - response=json.dumps({'success':False}), - status=404, - mimetype='application/json' - ) - return response - - latitude = request.form['latitude'] - longitude = request.form['longitude'] - authKey = request.form['authKey'] - responseD = { - 'latitude' : latitude, - 'authKey' : authKey, - 'longitude' : longitude - } +#TODO: We need to add in a method that recollects the users data every week. - response = app.response_class( - response=json.dumps(responseD), - status=200, - mimetype='application/json' - ) - response.headers.add('Access-Control-Allow-Origin', '*') - return response @app.route('/store_user', methods = ['POST']) def store_user(): + ''' + Stores Users: + -Location + -TimeZone + -Artists + -Recently Played Songs + -Playlists + -Liked Genres + ''' status = 404 success = False message = "Fields are incorrect" @@ -66,122 +51,37 @@ def store_user(): user_id = request.form['user_id'] zipcode = request.form['zipcode'] last_refresh = int(request.form['last_refresh']) - query = ml_database.Check_User_Query(user_id) + query = ml_database.Check_User(user_id) status = 200 success = True if query is None: + print(country) spotify_info = spotify.get_all_songs_table_info(access_token,country) + utc_day = datetime.utcnow().strftime("%m/%d/%Y") - ml_database.Insert_UTC_Day_Query(utc_day) - utc_day_id = ml_database.Get_UTC_Day_ID_Query(utc_day) - ml_database.Insert_Location_Query(zipcode,country) - Location_ID = ml_database.Get_Location_ID_Query(zipcode,country) + utc_day_id = spotify_database.Insert_UTC_Day_Info(utc_day) + + location_id = spotify_database.Insert_Location_Info(zipcode,country) + songs = spotify_info['Songs'] - user_table = { - 'Display_name' : name, - 'Last_Known_Location_ID' : Location_ID, - 'Home_Location_ID' : Location_ID, - 'numRuns' : 0, - 'numSongs' : len(songs), - 'numHours' : 0, - 'access_token' : access_token, - 'refresh_token' : refresh_token, - 'last_refreshed' : last_refresh, - 'User_ID' : user_id - } - ml_database.Insert_User_Query(user_table) - for song_id,song_info in songs.items(): - song_query = ml_database.Check_Song_Query(song_id) - if song_query is None: - song_table_info = song_info - song_table_info['Song_key'] = song_table_info['key'] # In the table we call it Song_key because you can't have key as a column name - del song_table_info['key'] - song_table_info.update({ - 'Song_ID' : song_id - }) - ml_database.Insert_Song_Query(song_table_info) - genres = genre_ml_classifier.get_genres(song_info) - for genre in set(genres.keys()): - ml_database.Insert_Genre_Query(genre) - for genre,confidence in genres.items(): - genre_id = ml_database.Get_Genre_ID_Query(genre) - song_genre_info = { - 'Song_ID' : song_id, - 'Genre_ID' : genre_id, - 'Confidence' : confidence - } - ml_database.Insert_Song_Genre_Query(song_genre_info) + spotify_database.Insert_Songs_Info(songs) + + spotify_database.Insert_User_Info(user_id,access_token,refresh_token,last_refresh,name,location_id,len(songs)) + + user_genres = genre_ml_classifier.get_user_genre(songs) + spotify_database.Insert_User_Genres_Info(user_id,user_genres,utc_day_id) + recently_played_songs = spotify_info['Recently_Played_Songs'] - for song in recently_played_songs: - recently_played_songs_user_table = { - 'Song_ID' : song, - 'User_ID' : user_id, - 'UTC_Day_ID' : utc_day_id - } - ml_database.Insert_Recently_Played_Songs_User_Query(recently_played_songs_user_table) + spotify_database.Insert_User_Recently_Played_Songs_Info(utc_day_id,user_id,recently_played_songs) + artists = spotify_info['Artists'] songs_artists = spotify_info['Songs_Artists'] - for artist_info in artists: - artist_table = { - 'Artist_ID' : artist_info[0], - 'Artist_name' : artist_info[1] - } - user_artist_table = { - 'User_ID' : user_id, - 'Artist_ID' : artist_info[0], - 'UTC_Day_ID' : utc_day_id - } - ml_database.Insert_Artist_Query(artist_table) - ml_database.Insert_Users_Artists_Query(user_artist_table) - genres_artists = genre_ml_classifier.get_artist_genre(songs_artists,artist_info[0]) - for genre,confidence in genres_artists.items(): - genre_id = ml_database.Get_Genre_ID_Query(genre) - genre_artist_table = { - 'Genre_ID' : genre_id, - 'Artist_ID' : artist_info[0], - 'Confidence' : confidence - } - ml_database.Insert_Genre_Artist_Query(genre_artist_table) - for song in songs_artists[artist_info[0]]: - song_artist_table = { - 'Song_ID' : song, - 'Artist_ID' : artist_info[0], - 'UTC_Day_ID' : utc_day_id - } - ml_database.Insert_Song_Artist_Query(song_artist_table) + spotify_database.Insert_All_User_Artists_Info(utc_day_id,artists,songs_artists,user_id) + playlists = spotify_info['Playlists'] songs_playlists = spotify_info['Songs_Playlists'] - for playlist_info in playlists: - playlist_table = { - 'Playlist_ID' : playlist_info[0], - 'Playlist_name' : playlist_info[1] - } - user_playlist_table = { - 'User_ID' : user_id, - 'Playlist_ID' : playlist_info[0], - 'UTC_Day_ID' : utc_day_id - } - ml_database.Insert_Playlist_Query(playlist_table) - ml_database.Insert_Users_Playlists_Query(user_playlist_table) - genres_playlists = genre_ml_classifier.get_playlist_genre(songs_playlists,playlist_info[0]) - if 'N/A' in genres_playlists: # This will only happen if the playlist is empty. - ml_database.Insert_Genre_Query('N/A') - for genre,confidence in genres_playlists.items(): - genre_id = ml_database.Get_Genre_ID_Query(genre) - genre_playlist_table = { - 'Genre_ID' : genre_id, - 'Playlist_ID' : playlist_info[0], - 'Confidence' : confidence - } - ml_database.Insert_Genre_Playlist_Query(genre_playlist_table) - if playlist_info[0] in songs_playlists: - for song in songs_playlists[playlist_info[0]]: - song_playlist_table = { - 'Song_ID' : song, - 'Playlist_ID' : playlist_info[0], - 'UTC_Day_ID' : utc_day_id - } - ml_database.Insert_Songs_Playlist_Query(song_playlist_table) + spotify_database.Insert_All_User_Playlists_Info(playlists,user_id,utc_day_id,songs_playlists) + message = "User inserted" else: message = "User already inserted" @@ -206,39 +106,29 @@ def get_data(): @app.route('/fill_weather',methods =['GET']) def fill_weather(): + ''' + This will be called once per day after all the user's track information. + Gets information for the previous day and fills in the weather for each hour. + This is done BY ZIPCODE not by user to avoid too many calls to the weather API. + ''' global curr_date - utc_day_id = ml_database.Get_UTC_Day_ID_Query(curr_date) + utc_day_id = ml_database.Get_UTC_Day_ID(curr_date) locations = ml_database.Get_Day_Locations(utc_day_id) for location_id in locations: zipcode,country = ml_database.Get_Location_Data(location_id) - longlat = sun_time.get_long_lat(zipcode, country) - prev_weather_data = weather.get_prev_day_data(longlat['lat'],longlat['lng']) - for hour_data in prev_weather_data: - date = datetime.utcfromtimestamp(hour_data["dt"]) - utc_hour = date.strftime("%H:00") - utc_day = date.strftime("%m/%d/%Y") - utc_day_id = ml_database.Get_UTC_Day_ID_Query(utc_day) - utc_hour_id = ml_database.Get_UTC_Hour_ID_Query(utc_hour) - weather_info = { - 'UTC_Day_ID' : 1, - 'UTC_Hour_ID' : 1, - 'Temperature' : hour_data['temp'], - 'Feels_like' : hour_data['feels_like'], - 'pressure' : hour_data['pressure'], - 'humidity' : 100.00 / float(hour_data['humidity']), - 'dew_point' : hour_data['dew_point'], - 'cloudiness' : 100.00 / float(hour_data['clouds']), - 'visibility' : hour_data['visibility'], - 'wind_speed' : hour_data['wind_speed'], - 'wind_gust' : hour_data['wind_gust'] if 'wind_gust' in hour_data else None, - 'rain' : hour_data['rain']['1h'] if 'rain' in hour_data else None, - 'snow' : hour_data['snow']['1h'] if 'snow' in hour_data else None, - 'conditions_id' :hour_data['weather'][0]['id'], - 'conditions' : hour_data['weather'][0]['description'] - } - ml_database.Insert_Weather_Info(weather_info) - weather_id = ml_database.Get_Weather_ID(location_id,utc_day_id,utc_hour_id) - ml_database.Update_Data_Hour_Per_Location_Weather(location_id,utc_day_id,utc_hour_id,weather_id) + if zipcode != 'ZipCode N/A': # We don't collect weather data if you don't have a zipcode + longlat = sun_time.get_long_lat(zipcode, country) + prev_weather_data = weather.get_prev_day_data(longlat['lat'],longlat['lng']) + for hour_data in prev_weather_data: + date = datetime.utcfromtimestamp(hour_data["dt"]) + utc_hour = date.strftime("%H:00") + utc_day = date.strftime("%m/%d/%Y") + utc_day_id = ml_database.Get_UTC_Day_ID(utc_day) + utc_hour_id = ml_database.Get_UTC_Hour_ID(utc_hour) + + weather_id = spotify_database.Insert_Weather_Info(hour_data,utc_day_id,utc_hour_id,location_id) + ml_database.Update_Data_Hour_Per_Location_Weather(location_id,utc_day_id,utc_hour_id,weather_id) + response = app.response_class( response=json.dumps({'success':True}), status=200, @@ -250,6 +140,9 @@ def fill_weather(): @app.route('/update_location', methods = ['POST']) def update_location(): + ''' + Used to update user's Last_Known_Location and updates home location if unknown. + ''' status = 404 success = False message = "Fields are incorrect" @@ -258,11 +151,14 @@ def update_location(): user_id = request.form['user_id'] zipcode = request.form['zipcode'] country = request.form['country'] - ml_database.Insert_Location_Query(zipcode,country) - Location_ID = ml_database.Get_Location_ID_Query(zipcode,country) - ml_database.Update_User_Last_Location(Location_ID,user_id) + + location_id = spotify_database.Insert_Location_Info(zipcode,country) + + ml_database.Update_User_Last_Location(location_id,user_id) + user_location = ml_database.Get_User_Home_Location(user_id) - if user_location[0] is 'ZipCode N/A': + + if user_location[0] == 'ZipCode N/A': # This is ml_database.Update_User_Home_Location(Location_ID,user_id) success = True status = 200 @@ -278,101 +174,68 @@ def update_location(): @app.route('/store_tracks', methods = ['GET']) def store_tracks(): + ''' + Stores users + -Track information + -Location + on a 15 minute interval called by the taskrunner into a single data collected per hour row. + + Creates new row in data collected per hour every hour. + Creates new row in data collected per day per day. + + Gets weather information at the end of the UTC Day. + ''' global curr_date users = ml_database.Get_Users() for user_id in users: - # if bool(ml_database.Get_User_NumRuns(user_id)) or (datetime.utcnow().minute == 15 and datetime.utcnow().hour == 0): # The first run per user must start at 0:00 UTC - curr_time = int((time.time() * 1000)) - access_info = ml_database.Get_Access_Information(user_id) - (new_access_token,new_last_refresh) = refresh_token.check_new_token(access_info['access_token'],access_info['refresh_token'],access_info['last_refreshed']) - ml_database.Update_Access_information(user_id,new_access_token,new_last_refresh) - recent_tracks = spotify.get_recent_tracks(new_access_token,after=curr_time-900000,id_only=True)# Did you FINISH any songs in the last 15 minutes. - numHours = ml_database.Get_User_NumHours(user_id) - numRuns = ml_database.Get_User_NumRuns(user_id) - if numRuns == 0: - ml_database.Insert_UTC_Day_Query(curr_date) - utc_day_id = ml_database.Get_UTC_Day_ID_Query(curr_date) - data_per_date = { - 'UTC_Day_ID' : utc_day_id, - 'User_ID' : user_id - } - ml_database.Insert_Day_Data(data_per_date) - if numRuns % 4 == 0 and numRuns % 96 != 0: # Every hour move to a new Dict, we start on the first hour so we don't need to increment. - numHours+= 1 - ml_database.Increment_NumHours(user_id) - curr_hour = datetime.utcnow().strftime("%H:00") - if ml_database.Check_DataCollectedPerHour(user_id,curr_date,curr_hour) is None: # We need to know the hour it was listened too so we can convert it to weather at the end. - ml_database.Insert_UTC_Hour_Query(curr_hour) - utc_hour_id = ml_database.Get_UTC_Hour_ID_Query(curr_hour) - location_id = ml_database.Get_User_Last_Location_ID(user_id) - utc_day_id = ml_database.Get_UTC_Day_ID_Query(curr_date) - data_date_id = ml_database.Get_User_Day_Data_ID(user_id,curr_date) - zipcode,country = ml_database.Get_User_Last_Location(user_id) - latlng = sun_time.get_long_lat(zipcode,country) - curr_datetime = datetime.utcnow().strftime("%m/%d/%Y : %H:%M") - timezone_id = ml_database.Get_TimeZone_ID_From_Location_Query(location_id) - sun_position = sun_time.get_SunPosition(latlng['lat'], latlng['lng'], curr_datetime) - time_period = sun_time.getTimePeriod(latlng['lat'], latlng['lng'], curr_datetime) - time_period_table = { - 'Time_Period' : time_period - } - ml_database.Insert_Time_Period(time_period_table) - time_period_id = ml_database.Get_Time_Period_ID(time_period) - timezone_hour_table = { - 'TimeZone_ID' : timezone_id, - 'UTC_Hour_ID' : utc_hour_id, - 'Time_Period_ID' : time_period_id - } - ml_database.Insert_TimeZone_Hour(timezone_hour_table) - sun_position_table = { - 'Sun_Position' : sun_position - } - ml_database.Insert_Sun_Position(sun_position_table) - sun_position_id = ml_database.Get_Sun_Position_ID(sun_position) - data_hour_table = { - 'User_ID' : user_id, - 'Location_ID' : location_id, - 'UTC_Hour_ID' : utc_hour_id, - 'UTC_Day_ID' : utc_day_id, - 'SunPosition_ID' : sun_position_id, - 'DataCollectedPerDate_ID' : data_date_id, - 'Weather_Characteristics_ID' : None - } - ml_database.Insert_Hour_Data(data_hour_table) - if bool(recent_tracks): - for (track_id,track) in recent_tracks.items(): - is_new = ml_database.Check_User_Playlist_Songs(user_id,track_id) is None and ml_database.Check_User_Recent_Songs(user_id,track_id) is None - curr_hour = datetime.utcnow().strftime("%H:00") - data_hour_id = ml_database.Get_User_Hour_Data_ID(user_id,curr_hour,curr_date) - if ml_database.Check_Data_Hour_Songs(user_id,track_id,data_hour_id) is not None: - ml_database.Increment_Data_Hour_Song_Played(data_hour_id,track_id) - else: - data_hour_songs_table = { - 'DataCollectedPerHour_ID' : data_hour_id, - 'Song_ID' :track_id, - 'numPlayed' : 1, - 'isNew' : int(is_new) - } - ml_database.Insert_Data_Hour_Song(data_hour_songs_table) - # We're only storing the songs we heard, the other songs the user in the user's playlist will be recorded as 0. - numRuns += 1 - ml_database.Increment_NumRuns(user_id) - if numRuns % 96 == 0 and numRuns != 0: # Every 15 minutes a day it incruments so in theory this should create a new date every day - curr_date = datetime.utcnow().strftime("%m/%d/%Y") - ml_database.Insert_UTC_Day_Query(curr_date) - utc_day_id = ml_database.Get_UTC_Day_ID_Query(curr_date) - data_per_date = { - 'UTC_Day_ID' : utc_day_id, - 'User_ID' : user_id - } - ml_database.Insert_Day_Data(data_per_date) - ml_database.Set0_NumHours(user_id) - try: - resp = requests.get("http://localhost:5000/fill_weather") - resp.raise_for_status() - except requests.exceptions.HTTPError as err: - print("ERR:" + str(err)) - numHours = 0 + if bool(ml_database.Get_User_NumRuns(user_id)) or (datetime.utcnow().minute == 15 and datetime.utcnow().hour == 0): # The first run per user must start at 0:00 UTC + curr_time = int((time.time() * 1000)) + + access_info = ml_database.Get_Access_Information(user_id) + (new_access_token,new_last_refresh) = refresh_token.check_new_token(access_info['access_token'],access_info['refresh_token'],access_info['last_refreshed']) + ml_database.Update_Access_information(user_id,new_access_token,new_last_refresh) + + recent_tracks = spotify.get_recent_tracks(new_access_token,after=curr_time-900000,id_only=True)# Did you FINISH any songs in the last 15 minutes. + + numHours = ml_database.Get_User_NumHours(user_id) + numRuns = ml_database.Get_User_NumRuns(user_id) + + if numRuns == 0: + utc_day_id = spotify_database.Insert_UTC_Day_Info(curr_date) + spotify_database.Insert_Data_Date_Info(utc_day_id,user_id) + + if numRuns % 4 == 0 and numRuns % 96 != 0: # Every hour move to a new Dict, we start on the first hour so we don't need to increment. + numHours+= 1 + ml_database.Increment_NumHours(user_id) + curr_hour = datetime.utcnow().strftime("%H:00") + if ml_database.Check_DataCollectedPerHour(user_id,curr_date,curr_hour) is None: # We need to know the hour it was listened too so we can convert it to weather at the end. + utc_hour_id = spotify_database.Insert_UTC_Hour_Info(curr_hour) + + location_id = ml_database.Get_User_Last_Location_ID(user_id) + utc_day_id = spotify_database.Insert_UTC_Day_Info(curr_date) + data_date_id = ml_database.Get_User_Day_Data_ID(user_id,utc_day_id) + zipcode,country = ml_database.Get_User_Last_Location(user_id) + + spotify_database.Insert_Data_Hour_Info(user_id,location_id,utc_hour_id,data_date_id,utc_day_id) + + if zipcode != 'ZipCode N/A': + spotify_database.Insert_Time_Data_Info(zipcode,country,location_id,utc_day_id,utc_hour_id) + if bool(recent_tracks): + spotify_database.Insert_User_Recent_Songs(recent_tracks,user_id,curr_date) + numRuns += 1 + ml_database.Increment_NumRuns(user_id) + if numRuns % 96 == 0 and numRuns != 0: # Every 15 minutes a day it incruments so in theory this should create a new date every day + curr_date = datetime.utcnow().strftime("%m/%d/%Y") + utc_day_id = spotify_database.Insert_UTC_Day_Info(curr_date) + spotify_database.Insert_Data_Date_Info(utc_day_id,user_id) + + ml_database.Set0_NumHours(user_id) + try: # We only get the weather at the end of the day as we can get all of the days weather in a single call which is important to keep under the rate limit. + resp = requests.get("http://localhost:5000/fill_weather") + resp.raise_for_status() + except requests.exceptions.HTTPError as err: + print("ERR:" + str(err)) + numHours = 0 response = app.response_class( response=json.dumps({'success':True}), status=200, @@ -414,42 +277,33 @@ def spotify_user_playlists(): data = spotify.get_current_user_playlists(access_token) return data -# ''' -# Related Playlist -# ''' -# @app.route('/relatedplaylist', methods=['GET']) -# def spotify_user_playlists(): -# access_token = request.args.get('access_token') -# data = spotify.get_current_user_playlists(access_token) -# return data - -@app.route('/weather') -def get_weather(): - zipcode = weather.get_zipcode() - data = weather.get_current_weather() - if "ERROR" not in data: - avg_temp = weather.get_avg_temp(data) - pressure = weather.get_pressure(data) - humidity = weather.get_humidity(data) - weather_id = weather.get_weather_id(data) - wind_speed = weather.get_wind_speed(data) - wind_direction = weather.get_wind_direction(data) - cloudiness = weather.get_cloudiness(data) - precipitation = weather.get_precipitation(data) - last_updated = weather.get_last_updated(data) - response = app.response_class( - response=json.dumps(data), - status=200, - mimetype='application/json' - ) - return response - else: - response = app.response_class( - response=json.dumps({"ERROR": "Refer to console for more information"}), - status=406, - mimetype='application/json' - ) - return response +# @app.route('/weather') +# def get_weather(): +# zipcode = '07305' +# data = weather.get_current_weather() +# if "ERROR" not in data: +# avg_temp = weather.get_avg_temp(data) +# pressure = weather.get_pressure(data) +# humidity = weather.get_humidity(data) +# weather_id = weather.get_weather_id(data) +# wind_speed = weather.get_wind_speed(data) +# wind_direction = weather.get_wind_direction(data) +# cloudiness = weather.get_cloudiness(data) +# precipitation = weather.get_precipitation(data) +# last_updated = weather.get_last_updated(data) +# response = app.response_class( +# response=json.dumps(data), +# status=200, +# mimetype='application/json' +# ) +# return response +# else: +# response = app.response_class( +# response=json.dumps({"ERROR": "Refer to console for more information"}), +# status=406, +# mimetype='application/json' +# ) +# return response if __name__ == '__main__': diff --git a/database/DB.sql b/database/DB.sql index ef8a925..05422fb 100644 --- a/database/DB.sql +++ b/database/DB.sql @@ -1,7 +1,7 @@ -- We're not creating nonclustered indexes for things that use the spotify_id (if the ID is a varchar(255) variable) or if it's intermediate table-- - +-- TODO: We need to start removing the ignore_dup_keys and handling it in the code instead-- CREATE TABLE Genres( - Genre varchar(255) NOT NULL, + Genre varchar(50) NOT NULL, Genre_ID int NOT NULL IDENTITY, PRIMARY KEY (Genre_ID)); @@ -18,12 +18,12 @@ CREATE TABLE Users( access_token text NOT NULL, refresh_token text NOT NULL, last_refreshed bigint NOT NULL, - User_ID varchar(255) NOT NULL, + User_ID varchar(100) NOT NULL, PRIMARY KEY (User_ID) WITH (IGNORE_DUP_KEY = ON)); CREATE TABLE Songs_Artists( - Song_ID varchar(255) NOT NULL, - Artist_ID varchar(255) NOT NULL, + Song_ID varchar(100) NOT NULL, + Artist_ID varchar(100) NOT NULL, UTC_Day_ID int NOT NULL, PRIMARY KEY (Song_ID,Artist_ID) WITH (IGNORE_DUP_KEY = ON)); @@ -32,13 +32,14 @@ CREATE NONCLUSTERED INDEX IX_Songs_Artists CREATE TABLE Genres_Artists( Genre_ID int NOT NULL, - Artist_ID varchar(255) NOT NULL, - Confidence numeric(18,7) NOT NULL, - PRIMARY KEY (Artist_ID,Genre_ID) WITH (IGNORE_DUP_KEY = ON)); + Artist_ID varchar(100) NOT NULL, + Confidence numeric(3,2) NOT NULL, + UTC_Day_ID int NOT NULL, + PRIMARY KEY (Artist_ID,Genre_ID,UTC_Day_ID) WITH (IGNORE_DUP_KEY = ON)); CREATE TABLE Songs_Playlists( - Song_ID varchar(255) NOT NULL, - Playlist_ID varchar(255) NOT NULL, + Song_ID varchar(100) NOT NULL, + Playlist_ID varchar(100) NOT NULL, UTC_Day_ID int NOT NULL, PRIMARY KEY (Song_ID,Playlist_ID) WITH (IGNORE_DUP_KEY = ON)); @@ -48,9 +49,9 @@ CREATE NONCLUSTERED INDEX IX_Songs_Playlists CREATE TABLE Users_Playlists( - User_ID varchar(255) NOT NULL, + User_ID varchar(100) NOT NULL, UTC_Day_ID int NOT NULL, - Playlist_ID varchar(255) NOT NULL, + Playlist_ID varchar(100) NOT NULL, PRIMARY KEY (User_ID,Playlist_ID) WITH (IGNORE_DUP_KEY = ON)); @@ -59,9 +60,9 @@ CREATE NONCLUSTERED INDEX IX_Users_Playlists CREATE TABLE Users_Artists( - User_ID varchar(255) NOT NULL, + User_ID varchar(100) NOT NULL, UTC_Day_ID int NOT NULL, - Artist_ID varchar(255) NOT NULL, + Artist_ID varchar(100) NOT NULL, PRIMARY KEY (User_ID,Artist_ID) WITH (IGNORE_DUP_KEY = ON)); @@ -70,8 +71,8 @@ CREATE NONCLUSTERED INDEX IX_Users_Artists CREATE TABLE RecentlyPlayedSongs_User( - Song_ID varchar(255) NOT NULL, - User_ID varchar(255) NOT NULL, + Song_ID varchar(100) NOT NULL, + User_ID varchar(100) NOT NULL, UTC_Day_ID int NOT NULL, PRIMARY KEY (Song_ID,User_ID) WITH (IGNORE_DUP_KEY = ON)); @@ -82,9 +83,9 @@ CREATE NONCLUSTERED INDEX IX_RecentlyPlayedSongs_User CREATE TABLE Location( Location_ID int NOT NULL IDENTITY, - Zipcode varchar(255) NOT NULL, - TimeZone_ID int NOT NULL, - Country varchar(255) NOT NULL, + Zipcode varchar(10) NOT NULL, + TimeZone_ID int, + Country varchar(5) NOT NULL, PRIMARY KEY (Location_ID)); CREATE UNIQUE NONCLUSTERED INDEX IX_Location @@ -92,19 +93,18 @@ CREATE UNIQUE NONCLUSTERED INDEX IX_Location CREATE TABLE TimeZone( TimeZone_ID int NOT NULL IDENTITY, - TimeZone_name varchar(255) NOT NULL, + TimeZone_name varchar(50) NOT NULL, PRIMARY KEY (TimeZone_ID) WITH (IGNORE_DUP_KEY = ON)); CREATE UNIQUE NONCLUSTERED INDEX IX_TimeZone ON TimeZone (TimeZone_name) WITH (IGNORE_DUP_KEY = ON); CREATE TABLE DataCollectedPerHour( - User_ID varchar(255) NOT NULL, + User_ID varchar(100) NOT NULL, Location_ID int NOT NULL, UTC_Hour_ID int NOT NULL, UTC_Day_ID int NOT NULL, DataCollectedPerHour_ID int NOT NULL IDENTITY, - SunPosition_ID int NOT NULL, DataCollectedPerDate_ID int NOT NULL, Weather_Characteristics_ID int , PRIMARY KEY (DataCollectedPerHour_ID)); @@ -115,14 +115,14 @@ CREATE UNIQUE NONCLUSTERED INDEX IX_DataCollectedPerHour CREATE TABLE DataCollectedPerDate( DataCollectedPerDate_ID int NOT NULL IDENTITY, UTC_Day_ID int NOT NULL, - User_ID varchar(255) NOT NULL, + User_ID varchar(100) NOT NULL, PRIMARY KEY (DataCollectedPerDate_ID)); CREATE UNIQUE NONCLUSTERED INDEX IX_DataCollectedPerDate ON DataCollectedPerDate (User_ID,UTC_Day_ID) WITH (IGNORE_DUP_KEY = ON); CREATE TABLE UTC_Hour( - UTC_Hour varchar(255) NOT NULL, + UTC_Hour varchar(20) NOT NULL, UTC_Hour_ID int NOT NULL IDENTITY, PRIMARY KEY (UTC_Hour_ID)); @@ -130,7 +130,7 @@ CREATE UNIQUE NONCLUSTERED INDEX IX_UTC_Hour ON UTC_Hour (UTC_Hour) WITH (IGNORE_DUP_KEY = ON); CREATE TABLE UTC_Day( - UTC_Day varchar(255) NOT NULL, + UTC_Day varchar(20) NOT NULL, UTC_Day_ID int NOT NULL IDENTITY, PRIMARY KEY (UTC_Day_ID)); @@ -139,15 +139,22 @@ CREATE UNIQUE NONCLUSTERED INDEX IX_UTC_Day CREATE TABLE Time_Period( Time_Period_ID int NOT NULL IDENTITY, - Time_Period varchar(255) NOT NULL, + Time_Period varchar(20) NOT NULL, PRIMARY KEY (Time_Period_ID)); CREATE UNIQUE NONCLUSTERED INDEX IX_Time_Period ON Time_Period (Time_Period) WITH (IGNORE_DUP_KEY = ON); +CREATE TABLE Location_UTC_Day_Hour( + Location_ID int NOT NULL, + UTC_Day_ID int NOT NULL, + UTC_Hour_ID int NOT NULL, + Sun_Position_ID smallint NOT NULL, + PRIMARY KEY (Location_ID,UTC_Day_ID,UTC_Hour_ID)); + CREATE TABLE Sun_Position( - Sun_Position varchar(255) NOT NULL, - Sun_Position_ID int NOT NULL IDENTITY, + Sun_Position varchar(20) NOT NULL, + Sun_Position_ID smallint NOT NULL IDENTITY, PRIMARY KEY (Sun_Position_ID)); CREATE UNIQUE NONCLUSTERED INDEX IX_Sun_Position @@ -160,65 +167,66 @@ CREATE TABLE TimeZone_UTC_Hour( PRIMARY KEY (TimeZone_ID,UTC_Hour_ID) WITH (IGNORE_DUP_KEY = ON)); CREATE TABLE Songs( - Song_ID varchar(255) NOT NULL, - danceability numeric(18,7) NOT NULL, - acousticness numeric(18,7) NOT NULL, - energy numeric(18,7) NOT NULL, + Song_ID varchar(100) NOT NULL, + danceability numeric(6,5) NOT NULL, + acousticness numeric(6,5) NOT NULL, + energy numeric(6,5) NOT NULL, Song_key smallint NOT NULL, - loudness numeric(18,7) NOT NULL, - mode smallint NOT NULL, - speechiness numeric(18,7) NOT NULL, - instrumentalness numeric(18,7) NOT NULL, - liveness numeric(18,7) NOT NULL, - valence numeric(18,7) NOT NULL, - tempo numeric(18,7) NOT NULL, + loudness numeric(7,5) NOT NULL, + mode bit NOT NULL, + speechiness numeric(6,5) NOT NULL, + instrumentalness numeric(6,5) NOT NULL, + liveness numeric(6,5) NOT NULL, + valence numeric(6,5) NOT NULL, + tempo numeric(8,5) NOT NULL, duration_ms bigint NOT NULL, time_signature smallint NOT NULL, Song_name text NOT NULL, - is_explicit smallint not NULL, + is_explicit bit not NULL, release_year smallint not NULL, popularity smallint NOT NULL, PRIMARY KEY (Song_ID) WITH (IGNORE_DUP_KEY = ON)); CREATE TABLE Song_Genres( - Song_ID varchar(255) NOT NULL, + Song_ID varchar(100) NOT NULL, Genre_ID int NOT NULL, - Confidence numeric(18,7) NOT NULL, + Confidence numeric(7,5) NOT NULL, PRIMARY KEY (Song_ID,Genre_ID) WITH (IGNORE_DUP_KEY = ON)); CREATE TABLE Artists( Artist_name text NOT NULL, - Artist_ID varchar(255) NOT NULL, + Artist_ID varchar(100) NOT NULL, PRIMARY KEY (Artist_ID) WITH (IGNORE_DUP_KEY = ON)); CREATE TABLE Playlists( Playlist_name text NOT NULL, - Playlist_ID varchar(255) NOT NULL, + Playlist_ID varchar(100) NOT NULL, PRIMARY KEY (Playlist_ID) WITH (IGNORE_DUP_KEY = ON)); CREATE TABLE Genres_Playlists( - Playlist_ID varchar(255) NOT NULL, + Playlist_ID varchar(100) NOT NULL, Genre_ID int NOT NULL, - Confidence numeric(18,7) NOT NULL, - PRIMARY KEY (Playlist_ID,Genre_ID) WITH (IGNORE_DUP_KEY = ON)); + Confidence numeric(7,5) NOT NULL, + UTC_Day_ID int NOT NULL, + PRIMARY KEY (Playlist_ID,Genre_ID,UTC_Day_ID) WITH (IGNORE_DUP_KEY = ON)); CREATE TABLE Weather_Characteristics( Weather_Characteristics_ID int NOT NULL IDENTITY, UTC_Day_ID int NOT NULL, UTC_Hour_ID int NOT NULL, Location_ID int NOT NULL, - Temperature numeric(18,7) NOT NULL, - Feels_like numeric(18,7) NOT NULL, - pressure bigint NOT NULL, - humidity numeric(18,7) NOT NULL, - dew_point numeric(18,7) NOT NULL, - cloudiness numeric(18,7) NOT NULL, - visibility bigint NOT NULL, - wind_speed numeric(18,7) NOT NULL, - wind_gust numeric(18,7) , - wind_deg numeric(18,7) NOT NULL, - rain numeric(18,7) , - snow numeric(18,7) , + Temperature numeric(5,2) NOT NULL, + Feels_like numeric(5,2) NOT NULL, + pressure int NOT NULL, + humidity numeric(5,2) NOT NULL, + dew_point numeric(5,2) NOT NULL, + cloudiness numeric(5,2) NOT NULL, + visibility int NOT NULL, + wind_speed numeric(5,2) NOT NULL, + wind_gust numeric(5,2) , + wind_deg numeric(5,2) NOT NULL, + rain numeric(5,2) , + snow numeric(5,2) , conditions_id smallint NOT NULL, conditions text NOT NULL, PRIMARY KEY (Weather_Characteristics_ID)); @@ -228,7 +236,7 @@ CREATE UNIQUE NONCLUSTERED INDEX IX_Weather_Characteristics CREATE TABLE DataCollectedPerHour_Songs( DataCollectedPerHour_ID int NOT NULL, - Song_ID varchar(255) NOT NULL, + Song_ID varchar(100) NOT NULL, isNew bit NOT NULL, numPlayed smallint NOT NULL, PRIMARY KEY (DataCollectedPerHour_ID,Song_ID)); @@ -237,10 +245,35 @@ CREATE NONCLUSTERED INDEX IX_DataCollectedPerHour_Songs ON DataCollectedPerHour_Songs (isNew); CREATE TABLE Users_Genres( - User_ID varchar(255) NOT NULL, + User_ID varchar(100) NOT NULL, Genre_ID int NOT NULL, - Percent_liked numeric(18,7) NOT NULL - PRIMARY KEY (User_ID,Genre_ID) WITH (IGNORE_DUP_KEY = ON)); + UTC_Day_ID int NOT NULL, + Confidence numeric(7,5) NOT NULL + PRIMARY KEY (User_ID,Genre_ID,UTC_Day_ID) WITH (IGNORE_DUP_KEY = ON)); + + +ALTER TABLE Location_UTC_Day_Hour + ADD FOREIGN KEY (UTC_Day_ID) + REFERENCES UTC_Day (UTC_Day_ID); + + +ALTER TABLE Location_UTC_Day_Hour + ADD FOREIGN KEY (UTC_Hour_ID) + REFERENCES UTC_Hour (UTC_Hour_ID); + + +ALTER TABLE Location_UTC_Day_Hour + ADD FOREIGN KEY (Location_ID) + REFERENCES Location (Location_ID); + +ALTER TABLE Location_UTC_Day_Hour + ADD FOREIGN KEY (Sun_Position_ID) + REFERENCES Sun_Position (Sun_Position_ID); + + +ALTER TABLE Genres_Playlists + ADD FOREIGN KEY (UTC_Day_ID) + REFERENCES UTC_Day (UTC_Day_ID); @@ -262,6 +295,17 @@ ALTER TABLE Genres_Artists +ALTER TABLE Genres_Artists + ADD FOREIGN KEY (Artist_ID) + REFERENCES Artists (Artist_ID); + + + +ALTER TABLE Genres_Artists + ADD FOREIGN KEY (UTC_Day_ID) + REFERENCES UTC_Day (UTC_Day_ID); + + ALTER TABLE Users_Playlists ADD FOREIGN KEY (User_ID) REFERENCES Users (User_ID); @@ -408,15 +452,12 @@ ALTER TABLE Weather_Characteristics ADD FOREIGN KEY (Location_ID) REFERENCES Location (Location_ID); - -ALTER TABLE Genres_Artists - ADD FOREIGN KEY (Artist_ID) - REFERENCES Artists (Artist_ID); ALTER TABLE Song_Genres ADD FOREIGN KEY (Song_ID) REFERENCES Songs (Song_ID); + ALTER TABLE Song_Genres ADD FOREIGN KEY (Genre_ID) REFERENCES Genres (Genre_ID); @@ -427,6 +468,7 @@ ALTER TABLE Users_Playlists REFERENCES Playlists (Playlist_ID); + ALTER TABLE Users_Playlists ADD FOREIGN KEY (UTC_Day_ID) REFERENCES UTC_Day (UTC_Day_ID); @@ -448,11 +490,6 @@ ALTER TABLE DataCollectedPerHour ADD FOREIGN KEY (Weather_Characteristics_ID) REFERENCES Weather_Characteristics (Weather_Characteristics_ID); - - -ALTER TABLE DataCollectedPerHour - ADD FOREIGN KEY (SunPosition_ID) - REFERENCES Sun_Position (Sun_Position_ID); @@ -470,3 +507,8 @@ ALTER TABLE Users_Genres ADD FOREIGN KEY (Genre_ID) REFERENCES Genres (Genre_ID); + +ALTER TABLE Users_Genres + ADD FOREIGN KEY (UTC_Day_ID) + REFERENCES UTC_Day (UTC_Day_ID); + diff --git a/database/ml_database.py b/database/ml_database.py index 4df2280..447755b 100644 --- a/database/ml_database.py +++ b/database/ml_database.py @@ -28,158 +28,150 @@ GOOGLE_MAP_KEY = os.getenv('GOOGLE_MAP_KEY') +# TODO: Potentially use an ORM Like SQL Alchemy to turn this into an object structure instead. +# TODO: UNIT TEST insert methods on tables with no foreign keys +# TODO: Integration tests for all the other methods. with pyodbc.connect('DRIVER='+driver+';SERVER='+server+';PORT=1433;DATABASE='+database+';UID='+username+';PWD='+ password,autocommit=True) as conn: - def Insert_Location_Query(zipcode,country): - latlong = sun_time.get_long_lat(zipcode, country) - timezone_name = sun_time.getTimeZone(latlong['lat'],latlong['lng']) - Insert_TimeZone_Query(timezone_name) - TimeZone_ID = Get_TimeZone_ID_Query(timezone_name) - query_string = f"INSERT INTO Location (Zipcode,Country,TimeZone_ID) VALUES ('{zipcode}','{country}','{TimeZone_ID}');" - with conn.cursor() as cursor: - cursor.execute(query_string) - - def Get_TimeZone_ID_Query(timezone_name): - query_string = f"SELECT TimeZone_ID FROM TimeZone WHERE TimeZone_name = '{timezone_name}';" - with conn.cursor() as cursor: - cursor.execute(query_string) - row = cursor.fetchone() - return row[0] def Update_Data_Hour_Per_Location_Weather(Location_ID,UTC_Day_ID,UTC_Hour_ID,Weather_Characteristics_ID): query_string = f"UPDATE DataCollectedPerHour SET Weather_Characteristics_ID = {Weather_Characteristics_ID} WHERE UTC_Day_ID = {UTC_Day_ID} AND UTC_Hour_ID = {UTC_Hour_ID} AND Location_ID = {Location_ID}" with conn.cursor() as cursor: cursor.execute(query_string) - - def Get_TimeZone_ID_From_Location_Query(location_id): - query_string = f'SELECT TimeZone_ID FROM Location WHERE Location_ID = {location_id};' - with conn.cursor() as cursor: - cursor.execute(query_string) - row = cursor.fetchone() - return row[0] def Update_User_Last_Location(Location_ID,User_ID): query_string = f"UPDATE Users SET Last_Known_Location_ID = {Location_ID} WHERE User_ID = '{User_ID}'; " with conn.cursor() as cursor: cursor.execute(query_string) - def Get_Users(): - query_string = f"SELECT USER_ID FROM Users;" + def Update_Access_information(User_ID,access_token,last_refreshed): + query_string = f"UPDATE Users SET access_token = '{access_token}',last_refreshed = {last_refreshed} WHERE User_ID = '{User_ID}'; " with conn.cursor() as cursor: cursor.execute(query_string) - users = [row[0] for row in cursor.fetchall()] - return users - - def Get_Location_Data(Location_ID): - location_query_string = f"SELECT Zipcode,Country FROM Location WHERE Location_ID = {Location_ID}" + + + def Update_User_Home_Location(Location_ID,User_ID): + query_string = f"UPDATE Users SET Home_Location_ID = {Location_ID} WHERE User_ID = '{User_ID}'; " with conn.cursor() as cursor: - cursor.execute(location_query_string) - row = cursor.fetchone() - return row + cursor.execute(query_string) - def Get_User_NumRuns(User_ID): - query_string = f"SELECT numRuns FROM Users WHERE User_ID='{User_ID}';" + def Set0_NumHours(User_ID): + query_string = f"UPDATE Users SET numHours= 0 WHERE User_ID = '{User_ID}'; " with conn.cursor() as cursor: cursor.execute(query_string) - row = cursor.fetchone() - return row[0] - def Get_User_Day_Data_ID(User_ID,date): - query_string = f"SELECT A.DataCollectedPerDate_ID FROM DataCollectedPerDate A inner join UTC_Day B on A.UTC_Day_ID = B.UTC_Day_ID WHERE A.User_ID = '{User_ID}' AND B.UTC_Day = '{date}';" + def Increment_NumHours(User_ID): + query_string = f"UPDATE Users SET numHours=numHours + 1 WHERE User_ID = '{User_ID}'; " with conn.cursor() as cursor: cursor.execute(query_string) - row = cursor.fetchone() - return row[0] - def Get_User_Hour_Data_ID(User_ID,hour,date): - query_string = f"SELECT A.DataCollectedPerHour_ID FROM DataCollectedPerHour A inner join UTC_Hour B on A.UTC_Hour_ID = B.UTC_Hour_ID inner join UTC_Day C on C.UTC_Day_ID = A.UTC_Day_ID WHERE A.User_ID = '{User_ID}' AND B.UTC_Hour = '{hour}' AND C.UTC_Day = '{date}';" + def Increment_NumRuns(User_ID): + query_string = f"UPDATE Users SET numRuns=numRuns + 1 WHERE User_ID = '{User_ID}'; " with conn.cursor() as cursor: cursor.execute(query_string) - row = cursor.fetchone() - return row[0] - - def Get_User_NumHours(User_ID): - query_string = f"SELECT numHours FROM Users WHERE User_ID='{User_ID}';" + + def Increment_Data_Hour_Song_Played(DataCollectedPerHour_ID,Song_ID): + query_string = f"UPDATE DataCollectedPerHour_Songs SET numPlayed = numPlayed + 1 WHERE DataCollectedPerHour_ID = {DataCollectedPerHour_ID} AND Song_ID = '{Song_ID}'; " with conn.cursor() as cursor: cursor.execute(query_string) - row = cursor.fetchone() - return row[0] - def Get_Access_Information(User_ID): - query_string = f"SELECT access_token,refresh_token,last_refreshed FROM Users WHERE User_ID='{User_ID}';" + + def Check_User(user): + query_string = f"SELECT User_ID FROM Users WHERE User_ID = '{user}'" with conn.cursor() as cursor: cursor.execute(query_string) row = cursor.fetchone() - access_info = { - 'access_token' : row[0], - 'refresh_token' : row[1], - 'last_refreshed': row[2] - } - return access_info + return row - def Get_Day_Locations(UTC_Day_ID): - query_string = f"SELECT DISTINCT Location_ID FROM DataCollectedPerHour WHERE UTC_DAY_ID = {UTC_Day_ID}" + def Check_Song(song): + query_string = f"SELECT Song_ID FROM Songs WHERE Song_ID = '{song}'" with conn.cursor() as cursor: cursor.execute(query_string) - locations = [row[0] for row in cursor.fetchall()] - return locations + row = cursor.fetchone() + return row - def Increment_NumHours(User_ID): - query_string = f"UPDATE Users SET numHours=numHours + 1 WHERE User_ID = '{User_ID}'; " + def Check_DataCollectedPerHour(User_ID,date,hour): + query_string = f"SELECT * FROM DataCollectedPerHour A inner join UTC_Day B on A.UTC_Day_ID = B.UTC_Day_ID inner join UTC_Hour C on A.UTC_Hour_ID = C.UTC_Hour_ID WHERE C.UTC_Hour = '{hour}' AND B.UTC_Day = '{date}' AND A.User_ID = '{User_ID}'" with conn.cursor() as cursor: cursor.execute(query_string) + row = cursor.fetchone() + return row - def Set0_NumHours(User_ID): - query_string = f"UPDATE Users SET numHours= 0 WHERE User_ID = '{User_ID}'; " + def Check_User_Playlist_Songs(User_ID,Song_ID): + query_string = f"SELECT B.Song_ID FROM Users_Playlists A inner join Songs_Playlists B on A.Playlist_ID = B.Playlist_ID WHERE A.User_ID = '{User_ID}' AND B.Song_ID ='{Song_ID}';" with conn.cursor() as cursor: cursor.execute(query_string) - - def Increment_NumRuns(User_ID): - query_string = f"UPDATE Users SET numRuns=numRuns + 1 WHERE User_ID = '{User_ID}'; " + row = cursor.fetchone() + return row + + def Check_User_Recent_Songs(User_ID,Song_ID): + query_string = f"SELECT Song_ID FROM RecentlyPlayedSongs_User WHERE User_ID = '{User_ID}' AND Song_ID = '{Song_ID}';" with conn.cursor() as cursor: cursor.execute(query_string) + row = cursor.fetchone() + return row - def Update_Access_information(User_ID,access_token,last_refreshed): - query_string = f"UPDATE Users SET access_token = '{access_token}',last_refreshed = {last_refreshed} WHERE User_ID = '{User_ID}'; " + def Check_Data_Hour_Songs(User_ID,Song_ID,DataCollectedPerHour_ID): + query_string = f"SELECT Song_ID FROM DataCollectedPerHour_Songs WHERE DataCollectedPerHour_ID = {DataCollectedPerHour_ID} AND Song_ID = '{Song_ID}';" with conn.cursor() as cursor: cursor.execute(query_string) + row = cursor.fetchone() + return row - def Get_User_Playlist_Songs(User_ID): query_string = f"SELECT B.Song_ID FROM Users_Playlists A inner join Songs_Playlists B on A.Playlist_ID = B.Playlist_ID WHERE A.User_ID = '{User_ID}';" with conn.cursor() as cursor: cursor.execute(query_string) playlist_songs = [row[0] for row in cursor.fetchall()] return playlist_songs - - def Check_DataCollectedPerHour(User_ID,date,hour): - query_string = f"SELECT * FROM DataCollectedPerHour A inner join UTC_Day B on A.UTC_Day_ID = B.UTC_Day_ID inner join UTC_Hour C on A.UTC_Hour_ID = C.UTC_Hour_ID WHERE C.UTC_Hour = '{hour}' AND B.UTC_Day = '{date}' AND A.User_ID = '{User_ID}'" + + def Get_Users(): + query_string = f"SELECT USER_ID FROM Users;" with conn.cursor() as cursor: cursor.execute(query_string) + users = [row[0] for row in cursor.fetchall()] + return users + + def Get_Location_Data(Location_ID): + location_query_string = f"SELECT Zipcode,Country FROM Location WHERE Location_ID = {Location_ID}" + with conn.cursor() as cursor: + cursor.execute(location_query_string) row = cursor.fetchone() return row - def Check_User_Playlist_Songs(User_ID,Song_ID): - query_string = f"SELECT B.Song_ID FROM Users_Playlists A inner join Songs_Playlists B on A.Playlist_ID = B.Playlist_ID WHERE A.User_ID = '{User_ID}' AND B.Song_ID ='{Song_ID}';" + def Get_User_NumRuns(User_ID): + query_string = f"SELECT numRuns FROM Users WHERE User_ID='{User_ID}';" with conn.cursor() as cursor: cursor.execute(query_string) row = cursor.fetchone() - return row + return row[0] - def Check_User_Recent_Songs(User_ID,Song_ID): - query_string = f"SELECT Song_ID FROM RecentlyPlayedSongs_User WHERE User_ID = '{User_ID}' AND Song_ID = '{Song_ID}';" + def Get_User_NumHours(User_ID): + query_string = f"SELECT numHours FROM Users WHERE User_ID='{User_ID}';" with conn.cursor() as cursor: cursor.execute(query_string) row = cursor.fetchone() - return row + return row[0] - def Check_Data_Hour_Songs(User_ID,Song_ID,DataCollectedPerHour_ID): - query_string = f"SELECT Song_ID FROM DataCollectedPerHour_Songs WHERE DataCollectedPerHour_ID = {DataCollectedPerHour_ID} AND Song_ID = '{Song_ID}';" + def Get_Access_Information(User_ID): + query_string = f"SELECT access_token,refresh_token,last_refreshed FROM Users WHERE User_ID='{User_ID}';" with conn.cursor() as cursor: cursor.execute(query_string) row = cursor.fetchone() - return row + access_info = { + 'access_token' : row[0], + 'refresh_token' : row[1], + 'last_refreshed': row[2] + } + return access_info - def Get_Song_Genre_Query(song_id): + def Get_Day_Locations(UTC_Day_ID): + query_string = f"SELECT DISTINCT Location_ID FROM DataCollectedPerHour WHERE UTC_DAY_ID = {UTC_Day_ID}" + with conn.cursor() as cursor: + cursor.execute(query_string) + locations = [row[0] for row in cursor.fetchall()] + return locations + + def Get_Song_Genre(song_id): query_string = f"SELECT A.Song_ID,B.Genre,A.Confidence FROM Song_Genres A inner join Genres B on A.Genre_ID = B.Genre_ID WHERE Song_ID='{song_id}';" with conn.cursor() as cursor: cursor.execute(query_string) @@ -196,13 +188,6 @@ def Get_User_Home_Location(User_ID): location_data = cursor.fetchone() return location_data - def Get_User_Last_Location_ID(User_ID): - query_string = f"SELECT Last_Known_Location_ID FROM Users WHERE User_ID = '{User_ID}'" - with conn.cursor() as cursor: - cursor.execute(query_string) - location_id = cursor.fetchone() - return location_id[0] - def Get_User_Last_Location(User_ID): query_string = f"SELECT Last_Known_Location_ID FROM Users WHERE User_ID = '{User_ID}'" with conn.cursor() as cursor: @@ -213,72 +198,64 @@ def Get_User_Last_Location(User_ID): location_data = cursor.fetchone() return location_data - def Update_User_Home_Location(Location_ID,User_ID): - query_string = f"UPDATE Users SET Home_Location_ID = {Location_ID} WHERE User_ID = '{User_ID}'; " + def Get_TimeZone_ID(timezone_name): + query_string = f"SELECT TimeZone_ID FROM TimeZone WHERE TimeZone_name = '{timezone_name}';" with conn.cursor() as cursor: cursor.execute(query_string) + row = cursor.fetchone() + return row[0] - def Increment_Data_Hour_Song_Played(DataCollectedPerHour_ID,Song_ID): - query_string = f"UPDATE DataCollectedPerHour_Songs SET numPlayed = numPlayed + 1 WHERE DataCollectedPerHour_ID = {DataCollectedPerHour_ID} AND Song_ID = '{Song_ID}'; " + def Get_TimeZone_ID_From_Location(location_id): + query_string = f'SELECT TimeZone_ID FROM Location WHERE Location_ID = {location_id};' with conn.cursor() as cursor: cursor.execute(query_string) + row = cursor.fetchone() + return row[0] - def Get_Location_ID_Query(zipcode,country): - query_string = f"SELECT Location_ID FROM Location WHERE zipcode = '{zipcode}' AND country = '{country}';" + def Get_User_Day_Data_ID(User_ID,date_id): + query_string = f"SELECT DataCollectedPerDate_ID FROM DataCollectedPerDate WHERE User_ID = '{User_ID}' AND UTC_Day_ID = '{date_id}';" with conn.cursor() as cursor: cursor.execute(query_string) row = cursor.fetchone() return row[0] - - def Insert_TimeZone_Query(timezone_name): - query_string = f"INSERT INTO TimeZone (TimeZone_name) VALUES ('{timezone_name}');" - with conn.cursor() as cursor: - cursor.execute(query_string) - - def Check_User_Query(user): - query_string = f"SELECT User_ID FROM Users WHERE User_ID = '{user}'" + + def Get_User_Hour_Data_ID(User_ID,hour_id,date_id): + query_string = f"SELECT DataCollectedPerHour_ID FROM DataCollectedPerHour WHERE User_ID = '{User_ID}' AND UTC_Hour_ID = '{hour_id}' AND UTC_Day_ID = '{date_id}';" with conn.cursor() as cursor: cursor.execute(query_string) row = cursor.fetchone() - return row + return row[0] - def Check_Song_Query(song): - query_string = f"SELECT Song_ID FROM Songs WHERE Song_ID = '{song}'" + def Get_User_Last_Location_ID(User_ID): + query_string = f"SELECT Last_Known_Location_ID FROM Users WHERE User_ID = '{User_ID}'" with conn.cursor() as cursor: cursor.execute(query_string) - row = cursor.fetchone() - return row + location_id = cursor.fetchone() + return location_id[0] + - def Get_UTC_Day_ID_Query(utc_day): - query_string = f"SELECT UTC_Day_ID FROM UTC_Day WHERE UTC_Day = '{utc_day}'" + def Get_Location_ID(zipcode,country): + query_string = f"SELECT Location_ID FROM Location WHERE zipcode = '{zipcode}' AND country = '{country}';" with conn.cursor() as cursor: cursor.execute(query_string) row = cursor.fetchone() return row[0] - def Insert_UTC_Day_Query(utc_day): - query_string = f"INSERT INTO UTC_Day (UTC_Day) VALUES ('{utc_day}');" + def Get_UTC_Day_ID(utc_day): + query_string = f"SELECT UTC_Day_ID FROM UTC_Day WHERE UTC_Day = '{utc_day}'" with conn.cursor() as cursor: cursor.execute(query_string) + row = cursor.fetchone() + return row[0] - def Get_UTC_Hour_ID_Query(utc_hour): + def Get_UTC_Hour_ID(utc_hour): query_string = f"SELECT UTC_Hour_ID FROM UTC_Hour WHERE UTC_Hour = '{utc_hour}'" with conn.cursor() as cursor: cursor.execute(query_string) row = cursor.fetchone() return row[0] - def Insert_UTC_Hour_Query(utc_hour): - query_string = f"INSERT INTO UTC_Hour (UTC_Hour) VALUES ('{utc_hour}');" - with conn.cursor() as cursor: - cursor.execute(query_string) - - def Insert_Genre_Query(genre): - query_string = f"INSERT INTO Genres (Genre) VALUES ('{genre}');" - with conn.cursor() as cursor: - cursor.execute(query_string) - - def Get_Genre_ID_Query(genre): + def Get_Genre_ID(genre): query_string = f"SELECT Genre_ID FROM Genres WHERE Genre = '{genre}'" with conn.cursor() as cursor: cursor.execute(query_string) @@ -306,7 +283,52 @@ def Get_Weather_ID(Location_ID,UTC_Day_ID,UTC_Hour_ID): row = cursor.fetchone() return row[0] - def Insert_Weather_Info(weather_info): + def Insert_Location(location_info): + columns = ', '.join(location_info.keys()) + placeholders = ', '.join('?' * len(location_info.keys())) + columns = columns.replace("'","") + values = [x for x in location_info.values()] + query_string = f"INSERT INTO Location ({columns}) VALUES ({placeholders});" + with conn.cursor() as cursor: + cursor.execute(query_string,values) + + def Insert_TimeZone(timezone_info): + columns = ', '.join(timezone_info.keys()) + placeholders = ', '.join('?' * len(timezone_info.keys())) + columns = columns.replace("'","") + values = [x for x in timezone_info.values()] + query_string = f"INSERT INTO TimeZone ({columns}) VALUES ({placeholders});" + with conn.cursor() as cursor: + cursor.execute(query_string,values) + + def Insert_UTC_Day(utc_day_info): + columns = ', '.join(utc_day_info.keys()) + placeholders = ', '.join('?' * len(utc_day_info.keys())) + columns = columns.replace("'","") + values = [x for x in utc_day_info.values()] + query_string = f"INSERT INTO UTC_Day ({columns}) VALUES ({placeholders});" + with conn.cursor() as cursor: + cursor.execute(query_string,values) + + def Insert_UTC_Hour(utc_hour_info): + columns = ', '.join(utc_hour_info.keys()) + placeholders = ', '.join('?' * len(utc_hour_info.keys())) + columns = columns.replace("'","") + values = [x for x in utc_hour_info.values()] + query_string = f"INSERT INTO UTC_Hour ({columns}) VALUES ({placeholders});" + with conn.cursor() as cursor: + cursor.execute(query_string,values) + + def Insert_Genre(genre_info): + columns = ', '.join(genre_info.keys()) + placeholders = ', '.join('?' * len(genre_info.keys())) + columns = columns.replace("'","") + values = [x for x in genre_info.values()] + query_string = f"INSERT INTO Genres ({columns}) VALUES ({placeholders});" + with conn.cursor() as cursor: + cursor.execute(query_string,values) + + def Insert_Weather(weather_info): columns = ', '.join(weather_info.keys()) placeholders = ', '.join('?' * len(weather_info.keys())) columns = columns.replace("'","") @@ -341,6 +363,15 @@ def Insert_Sun_Position(sun_position_info): query_string = f"INSERT INTO Sun_Position ({columns}) VALUES ({placeholders});" with conn.cursor() as cursor: cursor.execute(query_string,values) + + def Insert_Location_UTC_Day_Hour(location_day_hour_info): + columns = ', '.join(location_day_hour_info.keys()) + placeholders = ', '.join('?' * len(location_day_hour_info.keys())) + columns = columns.replace("'","") + values = [x for x in location_day_hour_info.values()] + query_string = f"INSERT INTO Location_UTC_Day_Hour ({columns}) VALUES ({placeholders});" + with conn.cursor() as cursor: + cursor.execute(query_string,values) def Insert_Time_Period(time_period_info): columns = ', '.join(time_period_info.keys()) @@ -368,17 +399,8 @@ def Insert_Hour_Data(hour_info): query_string = f"INSERT INTO DataCollectedPerHour ({columns}) VALUES ({placeholders});" with conn.cursor() as cursor: cursor.execute(query_string,values) - - def Insert_Weather_Data(weather_info): - columns = ', '.join(weather_info.keys()) - placeholders = ', '.join('?' * len(weather_info.keys())) - columns = columns.replace("'","") - values = [x for x in weather_info.values()] - query_string = f"INSERT INTO Weather_Characteristics ({columns}) VALUES ({placeholders});" - with conn.cursor() as cursor: - cursor.execute(query_string,values) - def Insert_User_Query(user_information): + def Insert_User(user_information): columns = ', '.join(user_information.keys()) placeholders = ', '.join('?' * len(user_information.keys())) columns = columns.replace("'","") @@ -387,16 +409,16 @@ def Insert_User_Query(user_information): with conn.cursor() as cursor: cursor.execute(query_string,values) - def Insert_Song_Query(song_information): - columns = ', '.join(song_information.keys()) - placeholders = ', '.join('?' * len(song_information.keys())) + def Insert_Song(song_info): + columns = ', '.join(song_info.keys()) + placeholders = ', '.join('?' * len(song_info.keys())) columns = columns.replace("'","") - values = [x for x in song_information.values()] + values = [x for x in song_info.values()] query_string = f"INSERT INTO Songs ({columns}) VALUES ({placeholders});" with conn.cursor() as cursor: cursor.execute(query_string,values) - def Insert_Song_Genre_Query(song_genre_info): + def Insert_Song_Genre(song_genre_info): columns = ', '.join(song_genre_info.keys()) placeholders = ', '.join('?' * len(song_genre_info.keys())) columns = columns.replace("'","") @@ -405,7 +427,7 @@ def Insert_Song_Genre_Query(song_genre_info): with conn.cursor() as cursor: cursor.execute(query_string,values) - def Insert_Song_Artist_Query(song_artist_info): + def Insert_Song_Artist(song_artist_info): columns = ', '.join(song_artist_info.keys()) placeholders = ', '.join('?' * len(song_artist_info.keys())) columns = columns.replace("'","") @@ -414,7 +436,7 @@ def Insert_Song_Artist_Query(song_artist_info): with conn.cursor() as cursor: cursor.execute(query_string,values) - def Insert_Songs_Playlist_Query(song_playlist_info): + def Insert_Songs_Playlist(song_playlist_info): columns = ', '.join(song_playlist_info.keys()) placeholders = ', '.join('?' * len(song_playlist_info.keys())) columns = columns.replace("'","") @@ -423,7 +445,7 @@ def Insert_Songs_Playlist_Query(song_playlist_info): with conn.cursor() as cursor: cursor.execute(query_string,values) - def Insert_Artist_Query(artist_info): + def Insert_Artist(artist_info): columns = ', '.join(artist_info.keys()) placeholders = ', '.join('?' * len(artist_info.keys())) columns = columns.replace("'","") @@ -432,7 +454,7 @@ def Insert_Artist_Query(artist_info): with conn.cursor() as cursor: cursor.execute(query_string,values) - def Insert_Genre_Artist_Query(genre_artist_info): + def Insert_Genre_Artist(genre_artist_info): columns = ', '.join(genre_artist_info.keys()) placeholders = ', '.join('?' * len(genre_artist_info.keys())) columns = columns.replace("'","") @@ -441,7 +463,7 @@ def Insert_Genre_Artist_Query(genre_artist_info): with conn.cursor() as cursor: cursor.execute(query_string,values) - def Insert_Playlist_Query(playlist_info): + def Insert_Playlist(playlist_info): columns = ', '.join(playlist_info.keys()) placeholders = ', '.join('?' * len(playlist_info.keys())) columns = columns.replace("'","") @@ -450,7 +472,7 @@ def Insert_Playlist_Query(playlist_info): with conn.cursor() as cursor: cursor.execute(query_string,values) - def Insert_Genre_Playlist_Query(genre_playlist_info): + def Insert_Genre_Playlist(genre_playlist_info): columns = ', '.join(genre_playlist_info.keys()) placeholders = ', '.join('?' * len(genre_playlist_info.keys())) columns = columns.replace("'","") @@ -459,7 +481,7 @@ def Insert_Genre_Playlist_Query(genre_playlist_info): with conn.cursor() as cursor: cursor.execute(query_string,values) - def Insert_Recently_Played_Songs_User_Query(recently_played_info): + def Insert_Recently_Played_Songs_User(recently_played_info): columns = ', '.join(recently_played_info.keys()) placeholders = ', '.join('?' * len(recently_played_info.keys())) columns = columns.replace("'","") @@ -468,7 +490,7 @@ def Insert_Recently_Played_Songs_User_Query(recently_played_info): with conn.cursor() as cursor: cursor.execute(query_string,values) - def Insert_Users_Artists_Query(user_artist_info): + def Insert_Users_Artists(user_artist_info): columns = ', '.join(user_artist_info.keys()) placeholders = ', '.join('?' * len(user_artist_info.keys())) columns = columns.replace("'","") @@ -477,11 +499,20 @@ def Insert_Users_Artists_Query(user_artist_info): with conn.cursor() as cursor: cursor.execute(query_string,values) - def Insert_Users_Playlists_Query(user_playlist_info): + def Insert_Users_Playlists(user_playlist_info): columns = ', '.join(user_playlist_info.keys()) placeholders = ', '.join('?' * len(user_playlist_info.keys())) columns = columns.replace("'","") values = [x for x in user_playlist_info.values()] query_string = f"INSERT INTO Users_Playlists ({columns}) VALUES ({placeholders});" + with conn.cursor() as cursor: + cursor.execute(query_string,values) + + def Insert_User_Genres(user_genre_info): + columns = ', '.join(user_genre_info.keys()) + placeholders = ', '.join('?' * len(user_genre_info.keys())) + columns = columns.replace("'","") + values = [x for x in user_genre_info.values()] + query_string = f"INSERT INTO Users_Genres ({columns}) VALUES ({placeholders});" with conn.cursor() as cursor: cursor.execute(query_string,values) \ No newline at end of file diff --git a/database/spotify_database.py b/database/spotify_database.py new file mode 100644 index 0000000..e013d03 --- /dev/null +++ b/database/spotify_database.py @@ -0,0 +1,358 @@ +import sys +sys.path.append("..") +from taskrunner import sun_time +from database import ml_database +from datetime import datetime +from spotify_data import genre_ml_classifier + + +def Insert_Songs_Info(songs): + ''' + Params: + Songs- A dictionary of songs which should be given from spotify get all songs table info + Return:None + Inserts All songs into the songs table and their top three genres into the song_genres table. + ''' + for song_id,song_info in songs.items(): + song_in_db = ml_database.Check_Song(song_id) + if song_in_db is None: + + song_table_info = song_info + song_table_info['Song_key'] = song_table_info['key'] # In the table we call it Song_key because you can't have key as a column name + del song_table_info['key'] + song_table_info.update({ + 'Song_ID' : song_id + }) + + ml_database.Insert_Song(song_table_info) + + genres = genre_ml_classifier.get_genres(song_info) + Insert_Genres_Info(set(genres.keys())) + Insert_Song_Genres_Info(song_id,genres) + +def Insert_All_User_Artists_Info(utc_day_id,artists,songs_artists,user_id): + ''' + Params: + utc_day_id -The id of the utc date. + Artists - List of tuples in (artist_id,artist_name) format. + Songs_artists -Dictionary in Artist_Id :List of Songs format. + User_ID - User Id. + Inserts Artists into artist table, and user_artist table with corresponding date. + Inserts all artists songs into artist_songs table. + Gets top three artist genres. + ''' + for artist_info in artists: + + Insert_Artist_Info(artist_info[0],artist_info[1]) + Insert_User_Artist_Info(artist_info[0],user_id,utc_day_id) + + Insert_Artist_Songs_Info(songs_artists[artist_info[0]],artist_info[0],utc_day_id) + + artist_genres = genre_ml_classifier.get_artist_genre(songs_artists,artist_info[0]) + Insert_Artist_Genres_Info(artist_genres,artist_info[0],utc_day_id) + +def Insert_All_User_Playlists_Info(playlists,user_id,utc_day_id,songs_playlists): + ''' + Params: + Playlists- List of tuples of playlists in (playlist_id,playlist_name) format. + Songs_playlists - dictionary in playlist id : songs format. + Inserts playlists into Playlist table, and user_playlist table with corresponding date. + Inserts all playlist songs into playlist song table. + Gets top three playlist genres. + ''' + for playlist_info in playlists: + Insert_Playlist_Info(playlist_info[0],playlist_info[1]) + Insert_User_Playlist_Info(user_id,playlist_info[0],utc_day_id) + + playlist_genres = genre_ml_classifier.get_playlist_genre(songs_playlists,playlist_info[0]) + if 'N/A' in playlist_genres: # This will only happen if the playlist is empty. + Insert_Genres_Info(['N/A']) + + Insert_Playlist_Genres_Info(utc_day_id,playlist_info[0],playlist_genres) + if playlist_info[0] in songs_playlists: # In case the playlist is empty. + Insert_Playlist_Songs_Info(playlist_info[0],songs_playlists[playlist_info[0]],utc_day_id) + + +def Insert_Time_Data_Info(zipcode,country,location_id,utc_day_id,utc_hour_id): + ''' + Inserts sun_position , time_period and the timezone into the correct tables. + ''' + latlng = sun_time.get_long_lat(zipcode,country) + curr_datetime = datetime.utcnow().strftime("%m/%d/%Y : %H:%M") + + timezone_id = ml_database.Get_TimeZone_ID_From_Location(location_id) + + time_period = sun_time.getTimePeriod(latlng['lat'], latlng['lng'], curr_datetime) + time_period_id = Insert_Time_Period_Info(time_period) + + Insert_TimeZone_Hour_Info(timezone_id,utc_hour_id,time_period_id) + + sun_position = sun_time.get_SunPosition(latlng['lat'], latlng['lng'], curr_datetime) + sun_position_id = Insert_Sun_Position_Info(sun_position) + + Insert_Location_UTC_Day_Hour_Info(location_id,utc_day_id,utc_hour_id,sun_position_id) + +def Insert_User_Recent_Songs(recent_tracks,user_id,curr_date): + ''' + Insert recently listened songs into data_hour_songs table or increments the number of times it was played. + ''' + for (track_id,track) in recent_tracks.items(): + is_new = ml_database.Check_User_Playlist_Songs(user_id,track_id) is None and ml_database.Check_User_Recent_Songs(user_id,track_id) is None + curr_hour = datetime.utcnow().strftime("%H:00") + data_hour_id = ml_database.Get_User_Hour_Data_ID(user_id,curr_hour,curr_date) + if ml_database.Check_Data_Hour_Songs(user_id,track_id,data_hour_id) is not None: + ml_database.Increment_Data_Hour_Song_Played(data_hour_id,track_id) + else: + Insert_Data_Hour_Song_Info(track_id,data_hour_id,is_new) + +def Insert_User_Genres_Info(user_id,user_genres,utc_day_id): + ''' + Inserts user's top three genres and the corresponding confidence levels. + ''' + for genre,confidence in user_genres.items(): + genre_id = ml_database.Get_Genre_ID(genre) + user_genre_table = { + 'User_ID' : user_id, + 'Genre_ID' : genre_id, + 'UTC_Day_ID' : utc_day_id, + 'Confidence' : confidence + } + ml_database.Insert_User_Genres(user_genre_table) + +def Insert_User_Recently_Played_Songs_Info(utc_day_id,user_id,recently_played_songs): + for song in recently_played_songs: + recently_played_songs_user_table = { + 'Song_ID' : song, + 'User_ID' : user_id, + 'UTC_Day_ID' : utc_day_id + } + ml_database.Insert_Recently_Played_Songs_User(recently_played_songs_user_table) + +def Insert_Genres_Info(genres): + for genre in genres: + genre_table = { + 'Genre' : genre + } + ml_database.Insert_Genre(genre_table) + +def Insert_Song_Genres_Info(song_id,genres): + for genre,confidence in genres.items(): + genre_id = ml_database.Get_Genre_ID(genre) + song_genre_table = { + 'Song_ID' : song_id, + 'Genre_ID' : genre_id, + 'Confidence' : confidence + } + ml_database.Insert_Song_Genre(song_genre_table) + +def Insert_Artist_Genres_Info(genres_artists,artist_id,utc_day_id): + for genre,confidence in genres_artists.items(): + genre_id = ml_database.Get_Genre_ID(genre) + genre_artist_table = { + 'Genre_ID' : genre_id, + 'Artist_ID' : artist_id, + 'UTC_Day_ID' : utc_day_id, + 'Confidence' : confidence + } + ml_database.Insert_Genre_Artist(genre_artist_table) + +def Insert_Artist_Songs_Info(songs_artists,artist_id,utc_day_id): + for song in songs_artists: + song_artist_table = { + 'Song_ID' : song, + 'Artist_ID' : artist_id, + 'UTC_Day_ID' : utc_day_id + } + ml_database.Insert_Song_Artist(song_artist_table) + +def Insert_Playlist_Songs_Info(playlist_id,songs_playlists,utc_day_id): + for song in songs_playlists: + song_playlist_table = { + 'Song_ID' : song, + 'Playlist_ID' : playlist_id, + 'UTC_Day_ID' : utc_day_id + } + ml_database.Insert_Songs_Playlist(song_playlist_table) + +def Insert_Playlist_Genres_Info(utc_day_id,playlist_id,genres_playlists): + for genre,confidence in genres_playlists.items(): + genre_id = ml_database.Get_Genre_ID(genre) + genre_playlist_table = { + 'Genre_ID' : genre_id, + 'Playlist_ID' : playlist_id, + 'UTC_Day_ID' : utc_day_id, + 'Confidence' : confidence + } + ml_database.Insert_Genre_Playlist(genre_playlist_table) + + +def Insert_Playlist_Info(playlist_name,playlist_id): + playlist_table = { + 'Playlist_ID' : playlist_name, + 'Playlist_name' : playlist_id + } + ml_database.Insert_Playlist(playlist_table) + +def Insert_User_Playlist_Info(user_id,playlist_id,utc_day_id): + user_playlist_table = { + 'User_ID' : user_id, + 'Playlist_ID' : playlist_id, + 'UTC_Day_ID' : utc_day_id + } + ml_database.Insert_Users_Playlists(user_playlist_table) + +def Insert_Artist_Info(artist_id,artist_name): + artist_table = { + 'Artist_ID' : artist_id, + 'Artist_name' : artist_name + } + ml_database.Insert_Artist(artist_table) + +def Insert_User_Artist_Info(artist_id,user_id,utc_day_id): + user_artist_table = { + 'User_ID' : user_id, + 'Artist_ID' : artist_id, + 'UTC_Day_ID' : utc_day_id + } + ml_database.Insert_Users_Artists(user_artist_table) + +def Insert_TimeZone_Info(timezone_name): + timezone_table = { + 'TimeZone_name' : timezone_name + } + ml_database.Insert_TimeZone(timezone_table) + timezone_id = ml_database.Get_TimeZone_ID(timezone_name) + return timezone_id + +def Insert_Location_Info(zipcode,country): + location_table = { + 'TimeZone_ID' : None, + 'Zipcode' : zipcode, + 'Country' : country + } + if zipcode != 'ZipCode N/A': + latlong = sun_time.get_long_lat(zipcode, country) + timezone_name = sun_time.getTimeZone(latlong['lat'],latlong['lng']) + location_table['TimeZone_ID'] = Insert_TimeZone_Info(timezone_name) + ml_database.Insert_Location(location_table) + location_id = ml_database.Get_Location_ID(zipcode,country) + return location_id + +def Insert_UTC_Day_Info(utc_day): + utc_day_table = { + 'UTC_Day' : utc_day + } + ml_database.Insert_UTC_Day(utc_day_table) + utc_day_id = ml_database.Get_UTC_Day_ID(utc_day) + return utc_day_id + +def Insert_UTC_Hour_Info(utc_hour): + utc_hour_table = { + 'UTC_Hour' : utc_hour + } + ml_database.Insert_UTC_Hour(utc_hour_table) + utc_hour_id = ml_database.Get_UTC_Hour_ID(utc_hour) + return utc_hour_id + +def Insert_User_Info(user_id,access_token,refresh_token,last_refresh,display_name,location_id,numSongs): + user_table = { + 'Display_name' : display_name, + 'Last_Known_Location_ID' : location_id, + 'Home_Location_ID' : location_id, + 'numRuns' : 0, + 'numSongs' : numSongs, + 'numHours' : 0, + 'access_token' : access_token, + 'refresh_token' : refresh_token, + 'last_refreshed' : last_refresh, + 'User_ID' : user_id + } + ml_database.Insert_User(user_table) + +def Insert_Weather_Info(hour_data,utc_day_id,utc_hour_id,location_id): + weather_info = { + 'UTC_Day_ID' : utc_day_id, + 'UTC_Hour_ID' : utc_hour_id, + 'Location_ID' : location_id, + 'Temperature' : hour_data['temp'], + 'Feels_like' : hour_data['feels_like'], + 'pressure' : hour_data['pressure'], + 'humidity' : 100.00 / float(hour_data['humidity']), # This is given in a percentage so we have to conver it. + 'dew_point' : hour_data['dew_point'], + 'cloudiness' : 100.00 / float(hour_data['clouds']), # Same as above. + 'visibility' : hour_data['visibility'], + 'wind_speed' : hour_data['wind_speed'], + 'wind_gust' : hour_data['wind_gust'] if 'wind_gust' in hour_data else None, + 'wind_deg' : hour_data['wind_deg'], + 'rain' : hour_data['rain']['1h'] if 'rain' in hour_data else None, + 'snow' : hour_data['snow']['1h'] if 'snow' in hour_data else None, + 'conditions_id' :hour_data['weather'][0]['id'], + 'conditions' : hour_data['weather'][0]['description'] + } + ml_database.Insert_Weather(weather_info) + weather_id = ml_database.Get_Weather_ID(location_id,utc_day_id,utc_hour_id) + return weather_id + +def Insert_Sun_Position_Info(sun_position): + sun_position_table = { + 'Sun_Position' : sun_position + } + ml_database.Insert_Sun_Position(sun_position_table) + sun_position_id = ml_database.Get_Sun_Position_ID(sun_position) + return sun_position_id + +def Insert_Time_Period_Info(time_period): + time_period_table = { + 'Time_Period' : time_period + } + ml_database.Insert_Time_Period(time_period_table) + time_period_id = ml_database.Get_Time_Period_ID(time_period) + return time_period_id + +def Insert_TimeZone_Hour_Info(timezone_id,utc_hour_id,time_period_id): + timezone_hour_table = { + 'TimeZone_ID' : timezone_id, + 'UTC_Hour_ID' : utc_hour_id, + 'Time_Period_ID' : time_period_id + } + ml_database.Insert_TimeZone_Hour(timezone_hour_table) + +def Insert_Location_UTC_Day_Hour_Info(location_id,utc_day_id,utc_hour_id,sun_position_id): + location_day_hour_table = { + 'Location_ID' : location_id, + 'UTC_Hour_ID' : utc_hour_id, + 'UTC_Day_ID' : utc_day_id, + 'Sun_Position_ID' : sun_position_id + } + ml_database.Insert_Location_UTC_Day_Hour(location_day_hour_table) + +def Insert_Data_Date_Info(utc_day_id,user_id): + data_per_date = { + 'UTC_Day_ID' : utc_day_id, + 'User_ID' : user_id + } + ml_database.Insert_Day_Data(data_per_date) + data_date_id = ml_database.Get_User_Day_Data_ID(user_id,utc_day_id) + return data_date_id + +def Insert_Data_Hour_Info(user_id,location_id,utc_hour_id,data_date_id,utc_day_id): + data_hour_table = { + 'User_ID' : user_id, + 'Location_ID' : location_id, + 'UTC_Hour_ID' : utc_hour_id, + 'UTC_Day_ID' : utc_day_id, + 'DataCollectedPerDate_ID' : data_date_id, + 'Weather_Characteristics_ID' : None + } + ml_database.Insert_Hour_Data(data_hour_table) + data_hour_id = ml_database.Get_User_Hour_Data_ID(user_id,utc_hour_id,utc_day_id) + return data_hour_id + +def Insert_Data_Hour_Song_Info(track_id,data_hour_id,is_new): + data_hour_songs_table = { + 'DataCollectedPerHour_ID' : data_hour_id, + 'Song_ID' :track_id, + 'numPlayed' : 1, + 'isNew' : int(is_new) + } + ml_database.Insert_Data_Hour_Song(data_hour_songs_table) diff --git a/spotify_data/genre_ml_classifier.py b/spotify_data/genre_ml_classifier.py index 3963127..3f2a500 100644 --- a/spotify_data/genre_ml_classifier.py +++ b/spotify_data/genre_ml_classifier.py @@ -5,36 +5,79 @@ from Genre_preprocessor import get_prediction,DropTransformer def get_genres(song_info): + ''' + Params: + Song_info - A dictionary form of the song_table in the database. + + Returns: + A dictionary of the top three genres in the form + Genre(str) : Confidence(float) + Ex: 'Anime': 0.30 + + ''' song_genre_dict = song_info song_genre_dict['key'] = song_genre_dict['Song_key'] # In the table we call it Song_key because you can't have key as a column name del song_genre_dict['Song_key'] del song_genre_dict['Song_ID'] return get_prediction(song_genre_dict) + def get_playlist_genre(songs_playlists,playlist): + ''' + Returns: + A dictionary of the top three playlist genres in Genre confidence format or N/A is empty playlist. + ''' genre_occur = dict() if playlist in songs_playlists: # This is a check if there are no songs in that playlist. for song_id in songs_playlists[playlist]: - genre_info = ml_database.Get_Song_Genre_Query(song_id) + genre_info = ml_database.Get_Song_Genre(song_id) for genre,confidence in genre_info.items(): if genre in genre_occur: genre_occur[genre]+= confidence else: genre_occur[genre] = confidence - return dict(sorted(genre_occur.items(), key=lambda x: x[1], reverse=True)[:3]) - else: + normalized_genre = {genre: confidence / len(songs_playlists[playlist]) for genre, confidence in genre_occur.items()} + return dict(sorted(normalized_genre.items(), key=lambda x: x[1], reverse=True)[:3]) # We get the top 3 genres for any playlist. + else:# If there are no songs in the playlist return N/A genre_occur = { 'N/A' : 1.00 } return genre_occur + def get_artist_genre(songs_artists,artist): + ''' + Returns: + Top three artist genres in genre: confidence format. + ''' genre_occur = dict() for song_id in songs_artists[artist]: - genre_info = ml_database.Get_Song_Genre_Query(song_id) + genre_info = ml_database.Get_Song_Genre(song_id) for genre,confidence in genre_info.items(): if genre in genre_occur: genre_occur[genre]+= confidence else: genre_occur[genre] = confidence - return dict(sorted(genre_occur.items(), key=lambda x: x[1], reverse=True)[:3]) \ No newline at end of file + normalized_genre = {genre: confidence / len(songs_artists[artist]) for genre, confidence in genre_occur.items()} + return dict(sorted(normalized_genre.items(), key=lambda x: x[1], reverse=True)[:3]) + + +def get_user_genre(songs): + ''' + Returns: + Get the the users top 3 genres in genre: confidence format. + ''' + genre_list = ['Classical', 'Comedy', 'R&B', 'Blues', 'Children’s Music', 'Rap', + 'Soul', 'Alternative', 'Soundtrack', 'Rock', 'Hip-Hop', 'Dance', + 'Electronic', 'World', 'Country', 'Anime', 'Pop', 'Reggaeton', + 'Reggae', 'Movie', 'Jazz', 'Folk', 'Opera', 'Ska'] + genre_dict = {genre: 0.0 for genre in genre_list} + for song_id in songs: + genre_info = ml_database.Get_Song_Genre(song_id) + for genre,confidence in genre_info.items(): + if genre in genre_dict: + genre_dict[genre]+= confidence + else: + genre_dict[genre] = confidence + normalized_genre = {genre: confidence / len(songs) for genre, confidence in genre_dict.items()} + return dict(sorted(normalized_genre.items(), key=lambda x: x[1], reverse=True)[:3]) \ No newline at end of file diff --git a/spotify_data/spotify.py b/spotify_data/spotify.py index 0b67c45..7e2bb6f 100644 --- a/spotify_data/spotify.py +++ b/spotify_data/spotify.py @@ -2,14 +2,29 @@ import json from datetime import datetime -# Get the Current User's Recently Played Tracks (50) -RECENT_TRACKS_ENDPOINT = 'https://api.spotify.com/v1/me/player/recently-played?limit=50' -PLAYLISTS_ENDPOINT = 'https://api.spotify.com/v1/me/playlists' +''' +All Used Endpoints: +https://api.spotify.com/v1/audio-features +https://api.spotify.com/v1/me/player/recently-played +https://api.spotify.com/v1/me/playlists +https://api.spotify.com/v1/playlists/{playlist['id']}/ +All Required Scopes: +playlist-read-private +user-read-recently-played +''' -# https://developer.spotify.com/web-api/web-api-personalization-endpoints/get-recently-played -# Access Token requires scope: user-read-recently-played + +#TODO:Unit test this method: def get_recent_tracks(access_token,after=None,limit=None,id_only=False): - url = RECENT_TRACKS_ENDPOINT + ''' + Access_token + After-A unix timestamp of the last date to get recently played tracks. + Limit-How many tracks we can have + id_only-If turned Will only return a dictionary in ID:{Name: str, Finished_at:UTC_Time} + Required Scopes:user-read-recently-played + Endpoint:https://developer.spotify.com/documentation/web-api/reference/#endpoint-get-recently-played + ''' + url = 'https://api.spotify.com/v1/me/player/recently-played?limit=50' if after is not None: url += ('&after=' + str(after)) auth_header = {"Authorization": "Bearer {}".format(access_token)} @@ -22,8 +37,14 @@ def get_recent_tracks(access_token,after=None,limit=None,id_only=False): return song_id return resp.json() +#TODO: Unit test this method def get_current_user_playlists(access_token): - url = PLAYLISTS_ENDPOINT + ''' + Access_token: + Required Scopes:playlist-read-private + Endpoint:https://developer.spotify.com/documentation/web-api/reference/#endpoint-get-a-list-of-current-users-playlists + ''' + url = 'https://api.spotify.com/v1/me/playlists' auth_header = {"Authorization": "Bearer {}".format(access_token)} resp = requests.get(url, headers=auth_header) return resp.json() @@ -46,8 +67,18 @@ def validate_month(date_text): except ValueError: return False -def get_songs_audio_features(access_token,song_id,extra_info): - song_id_req = list(chunks(list(song_id),50)) # In case they have more than 50 songs which is likely we need to split requests up into batches of 50. +#TODO: Unit test this method +def get_songs_audio_features(access_token,songs_ids,extra_info): + ''' + Extra_info a dictionary holding + popularity:int + release_year:int + is_explicit:bool + Song_name:str + Required Scopes:None + Endpoints Used:https://developer.spotify.com/documentation/web-api/reference/#endpoint-get-several-audio-features + ''' + song_id_req = list(chunks(list(songs_ids),50)) # In case they have more than 50 songs which is likely we need to split requests up into batches of 50. audio_dict = dict() for req in song_id_req: id_str = ",".join(req) @@ -67,7 +98,8 @@ def get_songs_audio_features(access_token,song_id,extra_info): temp_d[a['id']].pop('track_href', None) temp_d[a['id']]['Song_name'] = extra_info[a['id']]['name'] temp_d[a['id']]['popularity'] = extra_info[a['id']]['popularity'] - release_year = datetime.now().year + release_year = 2100 # In case we don't know the album year we'll just use this instead + # Spotify occasionally doesn't give us release_date in the traditional format so we have to check for that. if validate_full(extra_info[a['id']]['release_date']): release_year = datetime.strptime(extra_info[a['id']]['release_date'],"%Y-%m-%d").year elif validate_month(extra_info[a['id']]['release_date']): @@ -82,10 +114,25 @@ def get_songs_audio_features(access_token,song_id,extra_info): print("ERR:" + str(err)) return audio_dict +#TODO : Unit test this method +# Market Id is the country code given by spotify. def get_all_songs_table_info(access_token,market_id): + ''' + Market_ID:Country Code + Access_token + Returns: + Gets table info for all song_related tables for a user except genre related ones. + Required Scopes: + playlist-read-private + user-read-recently-played + Endpoints Used: + https://developer.spotify.com/documentation/web-api/reference/#endpoint-get-several-audio-features + https://developer.spotify.com/documentation/web-api/reference/#endpoint-get-playlists-tracks + https://developer.spotify.com/documentation/web-api/reference/#endpoint-get-a-list-of-current-users-playlists + ''' songs_playlists = dict() - songs = set() # Avoid duplicates between songs - extra_info = dict() + songs = set() + extra_info = dict() # The ML model also uses extra features such as release_year, popularity, and is_explicit to classify genres. artist_songs = dict() playlists = list() artists = set() @@ -118,6 +165,8 @@ def get_all_songs_table_info(access_token,market_id): } except requests.exceptions.HTTPError as err: print("ERR:" + str(err)) + + # Getting recently played songs info recently_played_songs = list() try: resp = get_recent_tracks(access_token) diff --git a/taskrunner/app.py b/taskrunner/app.py deleted file mode 100644 index 44783b9..0000000 --- a/taskrunner/app.py +++ /dev/null @@ -1,35 +0,0 @@ -from flask import Flask -# import Flask-APScheduler -import requests -import csv -import datetime -import json -from flask_apscheduler import APScheduler - -# set configuration values -class Config(object): - SCHEDULER_API_ENABLED = True - -# create app -app = Flask(__name__) -app.config.from_object(Config()) - -# initialize scheduler -scheduler = APScheduler() -# if you don't wanna use a config, you can set options here: -# scheduler.api_enabled = True -scheduler.init_app(app) -scheduler.start() - -# interval example -@scheduler.task('cron', id='Call_Store', minute='0-59/15', misfire_grace_time=1) -def store(): - try: - resp = requests.get("http://localhost:5000/store_tracks") - resp.raise_for_status() - except requests.exceptions.HTTPError as err: - print("ERR:" + str(err)) - - -if __name__ == '__main__': - app.run(port=8080) \ No newline at end of file diff --git a/taskrunner/sun_time.py b/taskrunner/sun_time.py index 8c23f8f..3a4a504 100644 --- a/taskrunner/sun_time.py +++ b/taskrunner/sun_time.py @@ -6,7 +6,6 @@ import os import requests from dotenv import load_dotenv -from timezonefinder import TimezoneFinder GOOGLE_MAP_KEY = '' try: @@ -15,7 +14,12 @@ load_dotenv() GOOGLE_MAP_KEY = os.getenv('GOOGLE_MAP_KEY') - +''' +Added Dependencies: +Pytz,TimeZoneFinder,astral +TODO: If we can let's attempt to see if we can find timezone and sunposition without these added dependencies. +''' +#TODO:Unit test this method def get_SunPosition(latitude, longitude, curr_time): tf = TimezoneFinder() utc = pytz.UTC @@ -41,7 +45,7 @@ def get_SunPosition(latitude, longitude, curr_time): else: return "Dusk" - +#TODO:Unit test this method def getTimePeriod(latitude,longitude, curr_time): converted_time = getConvertedTime(curr_time,getTimeZone(latitude, longitude)) if converted_time.hour == 12: @@ -57,13 +61,13 @@ def getTimePeriod(latitude,longitude, curr_time): else: return "Night" - +#TODO:Unit test this method def get_long_lat(zipcode, country): maps_query = f'https://maps.google.com/maps/api/geocode/json?components=country:{country}|postal_code:{zipcode}&sensor=false&key={GOOGLE_MAP_KEY}' try: resp = requests.get(maps_query) resp.raise_for_status() - if resp.json()['status'] == 'ZERO_RESULTS': + if resp.json()['status'] == 'ZERO_RESULTS': # Google Maps can not decode private zipcodes+ country combinations. location = dict() else: location = resp.json()['results'][0]['geometry']['location'] diff --git a/taskrunner/task_app.py b/taskrunner/task_app.py new file mode 100644 index 0000000..5e683c8 --- /dev/null +++ b/taskrunner/task_app.py @@ -0,0 +1,11 @@ +import requests + +def store(): + try: + hostname = socket.gethostname() + ip = socket.gethostbyname(hostname) + local = f"http://localhost:5000/store_tracks" + resp = requests.get(local) + resp.raise_for_status() + except requests.exceptions.HTTPError as err: + print("ERR:" + str(err)) diff --git a/weather_data/weather.py b/weather_data/weather.py index 0884650..2606b4e 100644 --- a/weather_data/weather.py +++ b/weather_data/weather.py @@ -19,6 +19,13 @@ GEO_KEY = os.getenv('GEO_API_KEY') WEATHER_KEY = os.getenv('WEATHER_API_KEY') +# Note from OpenWeather API: +# If you do not see some of the parameters in your API response it means that these weather +# phenomena are just not happened for the time of measurement for the city or location chosen. +# Only really measured or calculated data is displayed in API response. + +#NOTE: We do not collect the latitude and longitude but sun_time has a get_long_lat method from the zipcode and country. + # Return user's zip code def get_zip_code(): CLIENT_IP = request.remote_addr @@ -31,10 +38,6 @@ def get_zip_code(): return '00000' return zipcode -# Note from OpenWeather API: -# If you do not see some of the parameters in your API response it means that these weather -# phenomena are just not happened for the time of measurement for the city or location chosen. -# Only really measured or calculated data is displayed in API response. def get_current_weather(): weather_dict = {} try: @@ -54,9 +57,14 @@ def get_current_weather(): weather_dict = {"ERROR" : str(e)} return weather_dict +''' +Returns: +For information on the full response:https://openweathermap.org/api/one-call-api +''' +#TODO: Unit test this method. def get_prev_day_data(lat,lng): try: - prev_day = datetime.utcnow() - timedelta(days = 1) + prev_day = datetime.utcnow() - timedelta(days=1) wr = requests.get(f"http://api.openweathermap.org/data/2.5/onecall/timemachine?lat={lat}&lon={lng}&dt={int(prev_day.timestamp())}&appid={WEATHER_KEY}") wr.raise_for_status() return wr.json()['hourly'] @@ -64,10 +72,11 @@ def get_prev_day_data(lat,lng): print(f'ERROR:{str(e)}') return dict() -# Note from OpenWeather API: -# If you do not see some of the parameters in your API response it means that these weather -# phenomena are just not happened for the time of measurement for the city or location chosen. -# Only really measured or calculated data is displayed in API response. +''' +Returns: +For information on the full response:https://openweathermap.org/current#geo +''' +#TODO:Unit test this method def get_weather_from_zipcode(zipcode,country_code): try: units_type = 'imperial' @@ -76,7 +85,11 @@ def get_weather_from_zipcode(zipcode,country_code): except requests.exceptions.RequestException as e: # This is the correct syntax print("ERR:" + str(err)) return weatherdict - +''' +Returns: +For infomration on the full response:https://openweathermap.org/current#geo +''' +#TODO:Unit test this method def get_weather_from_latlong(latitude,longitude): try: units_type = 'imperial' @@ -86,8 +99,16 @@ def get_weather_from_latlong(latitude,longitude): print("ERR:" + str(err)) return weatherdict - -def get_historical_weather_latlong(latitude,longitude,date):#Date: Unix timestamp +''' +Params: +Date: A unix timestamp +NOTE: it must be within 5 days of the current time +Latitude,Longitude: +Returns: +For infomration on the full response:https://openweathermap.org/api/one-call-api +''' +#TODO: Unit test this method. +def get_historical_weather_latlong(latitude,longitude,date): try: units_type = 'imperial' wr = requests.get('https://api.openweathermap.org/data/2.5/onecall/timemachine?lat={lat}&lon={lon}&dt={time}&appid={key}&units={units}'.format(latitude,longitude,date,WEATHER_KEY,units_type)) @@ -172,6 +193,4 @@ def get_last_updated(data): try: return data['current']['lastupdate']['@value'] except Exception: - return -1 - -get_prev_day_data(40.178098,-74.241539) \ No newline at end of file + return -1 \ No newline at end of file