diff --git a/project_code/.ipynb_checkpoints/master_function_copy-checkpoint.ipynb b/project_code/.ipynb_checkpoints/master_function_copy-checkpoint.ipynb
deleted file mode 100644
index 278e5b8..0000000
--- a/project_code/.ipynb_checkpoints/master_function_copy-checkpoint.ipynb
+++ /dev/null
@@ -1,808 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "code",
- "execution_count": 56,
- "metadata": {},
- "outputs": [],
- "source": [
- "# importing libraries\n",
- "import pandas as pd\n",
- "import numpy as np\n",
- "from pathlib import Path\n",
- "from datetime import date, datetime, timedelta\n",
- "import os \n",
- "import pickle\n",
- "from collections import Counter"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Section 1: Get Model Input Data Function\n",
- "\n",
- "## This section takes the output from the stock selection to pull historical stock data and relevant news. With this data, we calculate daily stock returns for ML model targets, and runan NLP sentiment analysis model for ML model features. "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 57,
- "metadata": {},
- "outputs": [],
- "source": [
- "# importing libraries\n",
- "\n",
- "# APIs\n",
- "from newsapi.newsapi_client import NewsApiClient\n",
- "from iexfinance.stocks import Stock\n",
- "from iexfinance.stocks import get_historical_data\n",
- "\n",
- "# NLP\n",
- "import nltk\n",
- "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
- "analyzer = SentimentIntensityAnalyzer()\n",
- "from nltk.corpus import stopwords\n",
- "from nltk.tokenize import word_tokenize\n",
- "from nltk.stem import WordNetLemmatizer, PorterStemmer\n",
- "from string import punctuation\n",
- "import re"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 58,
- "metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "[nltk_data] Downloading package vader_lexicon to\n",
- "[nltk_data] /Users/ava/nltk_data...\n",
- "[nltk_data] Package vader_lexicon is already up-to-date!\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "True"
- ]
- },
- "execution_count": 58,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "nltk.download('vader_lexicon')"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 59,
- "metadata": {},
- "outputs": [],
- "source": [
- "# funtion to pull stock prices to calculate return prices, and pull news articles to calculate sentiment\n",
- "\n",
- "def get_model_data(company, ticker, lag=0):\n",
- " \n",
- " ## NEWS SENTIMENT\n",
- " \n",
- " # pulling in data\n",
- " sp500_csv = Path(\"Data/sp500_constituents.csv\")\n",
- " sp500_df = pd.read_csv(sp500_csv)\n",
- " sp500_df['Company'] = sp500_df['Name']\n",
- " sp500_df.drop(columns='Sector',inplace=True)\n",
- " sp500_df.set_index('Company', inplace=True)\n",
- " \n",
- " # converting DataFrame to dictionary\n",
- " new_stock_dict = sp500_df.T.to_dict('list')\n",
- " \n",
- " # setting up news API client\n",
- " from datetime import date, datetime, timedelta\n",
- " newsapi = NewsApiClient(api_key=os.environ[\"NEWS_API_KEY\"])\n",
- " pickle.dump(newsapi,open('newsapi.pickle','wb'))\n",
- " current_date = date.today()\n",
- " past_date = date.today() - timedelta(days=30)\n",
- " \n",
- " # establishing keywords for news pull\n",
- " keyword = f'{company} AND {ticker}'\n",
- " \n",
- " def get_headlines(keyword):\n",
- " \n",
- " # creating empty list for all headlines\n",
- " all_headlines = [] \n",
- " \n",
- " # creating empty list for all dates\n",
- " all_dates = [] \n",
- " \n",
- " # assigning today's date to variable\n",
- " date = current_date \n",
- " \n",
- " # establishing condition for date to be pulled as the difference between today and a past date \n",
- " while date > past_date: \n",
- " \n",
- " # pulling news articles news API based on keywords\n",
- " articles = newsapi.get_everything( \n",
- " q=keyword,\n",
- " from_param=str(date),\n",
- " to=str(date),\n",
- " language=\"en\",\n",
- " sort_by=\"relevancy\",\n",
- " page=1,\n",
- " )\n",
- " \n",
- " # creating empty list for headlines \n",
- " # **what is the point of this? why don't we just add headlines directly to all_headlines?**\n",
- " headlines = []\n",
- " \n",
- " # iterating through articles to pull content\n",
- " for i in range(0, len(articles[\"articles\"])):\n",
- "\n",
- " # adding content to headlines list\n",
- " headlines.append(articles[\"articles\"][i][\"content\"])\n",
- " \n",
- " # adding content to list\n",
- " all_headlines.append(headlines)\n",
- " \n",
- " # adding dates to list\n",
- " all_dates.append(date)\n",
- " \n",
- " # moving through the days\n",
- " date = date - timedelta(days=1) \n",
- " \n",
- " # exporting headlines and dates as function output\n",
- " return all_headlines, all_dates\n",
- " \n",
- " # running get_headlines function with the chosen company as the input \n",
- " # and assigning list variables for outputs\n",
- " headlines, dates = get_headlines(new_stock_dict[company][1])\n",
- " \n",
- " # adding headlines to DataFrame and setting dates as index\n",
- " df = pd.DataFrame(headlines)\n",
- " df.index = pd.to_datetime(dates)\n",
- " \n",
- " # creating empty list for to hold date / article dictionary\n",
- " concatenated_list = []\n",
- " \n",
- " # assigning limit variable for iterating through the headlines DataFrame\n",
- " i = 0\n",
- " \n",
- " # iterating through DataFrame to add date / article dictionary to list\n",
- " while i < len(df):\n",
- " date = df.index[i]\n",
- " daily_data = df.iloc[i,:].dropna().to_list()\n",
- " daily_data = \"\".join(daily_data)\n",
- " concatenated_list.append({\"Date\":date,\n",
- " \"articles\":daily_data\n",
- " })\n",
- " i = i + 1\n",
- " \n",
- " # converting the list of dict into a DataFrame\n",
- " df = pd.DataFrame(concatenated_list)\n",
- " \n",
- " # assigning lemmatizer function to variable\n",
- " lemmatizer = WordNetLemmatizer()\n",
- " \n",
- " # function to tokenize text\n",
- " def tokenizer(text):\n",
- " \n",
- " # cleaning text\n",
- " sw = set(stopwords.words('english'))\n",
- " regex = re.compile(\"[^a-zA-Z ]\")\n",
- " re_clean = regex.sub('', text)\n",
- " words = word_tokenize(re_clean)\n",
- " lem = [lemmatizer.lemmatize(word) for word in words]\n",
- " tokens = [word.lower() for word in lem if word.lower() not in sw]\n",
- " \n",
- " # exporting tokenized words as output\n",
- " return tokens\n",
- " \n",
- " # creating DataFrame of cleaned words from news artibels\n",
- " df[\"tokens\"] = [tokenizer(i) for i in df[\"articles\"]]\n",
- " df[\"Clean Words\"] = [\" \".join(i) for i in df[\"tokens\"]]\n",
- " \n",
- " # function to create a DataFrame of news sentiment \n",
- " def get_sentiment(dataframe):\n",
- " \n",
- " # creating list of sentiment objects observed\n",
- " df_sentiments = []\n",
- " \n",
- " # setting limit variable\n",
- " i = 0 \n",
- "\n",
- " # iterating through DataFrame to get news sentiment \n",
- " while i < len(dataframe[\"Clean Words\"]): \n",
- "\n",
- " # creating variables for text and date columns\n",
- " text = dataframe[\"Clean Words\"][i] \n",
- " date = dataframe[\"Date\"][i]\n",
- "\n",
- " # running sentiment analysis\n",
- " sentiment = analyzer.polarity_scores(text) \n",
- " \n",
- " compound = sentiment[\"compound\"]\n",
- " pos = sentiment[\"pos\"]\n",
- " neu = sentiment[\"neu\"]\n",
- " neg = sentiment[\"neg\"]\n",
- "\n",
- " # creating dictionary of articles and sentiment scores\n",
- " df_sentiments.append({\n",
- " \"text\": text,\n",
- " \"date\": date,\n",
- " \"compound\": compound,\n",
- " \"positive\": pos,\n",
- " \"negative\": neg,\n",
- " \"neutral\": neu\n",
- " })\n",
- " \n",
- " i += 1\n",
- "\n",
- " # converting dictionary to DataFrame\n",
- " final_df = pd.DataFrame(df_sentiments)\n",
- "\n",
- " # reordering DataFrame columns\n",
- " cols = [\"date\", \"text\", \"compound\", \"positive\", \"negative\", \"neutral\"]\n",
- " \n",
- " # creating list of DataFrame content\n",
- " final_list = final_df[cols]\n",
- " \n",
- " # expoering list of news text and sentiment scores as output\n",
- " return final_list\n",
- " \n",
- " # creating news sentiment DataFrame\n",
- " df = get_sentiment(df)\n",
- " df = df.set_index('date')\n",
- " df = df.sort_values(by='date',ascending=True)\n",
- "\n",
- " # function to calculate the average sentiment score leading up to every trading day\n",
- " # with logic to deal with non-trading days (weekends/holidays) by averaging scores across these days\n",
- " def cleaned_df(dataframe):\n",
- " \n",
- " # creating lists for each sentiment category\n",
- " compound = []\n",
- " positive = []\n",
- " negative = []\n",
- " neutral = []\n",
- "\n",
- " # iterating through sentiment score / article DataFrame to...\n",
- " for index, row in dataframe.iterrows():\n",
- "\n",
- " # if daily return is null value for a given day - i.e. a non-trading day,\n",
- " if pd.isnull(row['return']):\n",
- " \n",
- " # then append polarity scores to their respective lists\n",
- " compound.append(row['compound'])\n",
- " positive.append(row['positive'])\n",
- " negative.append(row['negative'])\n",
- " neutral.append(row['neutral'])\n",
- " dataframe.drop(index=index, inplace=True)\n",
- " \n",
- " # if there was a return value - i.e. it was a trading day\n",
- " elif pd.notnull(row['return']):\n",
- " \n",
- " # The list of compound polarity scores will be empty if the stock was traded\n",
- " # on the previous day; therefore, move along.\n",
- " if len(compound) == 0:\n",
- " pass\n",
- "\n",
- " # If the list is not empty, then at least one day prior was a non-trading \n",
- " # day. Append the current day's scores to the list and calculate the mean \n",
- " # for each score. Then replace the current day's polarity scores with the \n",
- " # average scores of today and previous non-trading days.\n",
- " else:\n",
- " compound.append(row['compound'])\n",
- " compound_mean = np.mean(compound)\n",
- " compound = []\n",
- "\n",
- " positive.append(row['positive'])\n",
- " positive_mean = np.mean(positive)\n",
- " positive = []\n",
- "\n",
- " negative.append(row['negative'])\n",
- " negative_mean = np.mean(negative)\n",
- " negative = []\n",
- "\n",
- " neutral.append(row['neutral'])\n",
- " neutral_mean = np.mean(neutral)\n",
- " neutral = []\n",
- "\n",
- " dataframe.at[index, 'compound'] = compound_mean\n",
- " dataframe.at[index, 'positive'] = positive_mean\n",
- " dataframe.at[index, 'negative'] = negative_mean\n",
- " dataframe.at[index, 'neutral'] = neutral_mean\n",
- "\n",
- " else:\n",
- " pass\n",
- " \n",
- " # exporting DataFrame of average sentiment scores for every trading day, sorted by date, as output\n",
- " return dataframe.sort_index(ascending=True)\n",
- " \n",
- " ## STOCK RETURNS\n",
- " \n",
- " # setting time frame - 31 days needed instead of 30 days so that we get 30 days of return calculations\n",
- " end_date_stock = datetime.now()\n",
- " start_date_stock = end_date_stock + timedelta(-31)\n",
- "\n",
- " # getting closing price data via API and adding to DataFrame\n",
- " returns_df = get_historical_data(new_stock_dict[company][0], start_date_stock, end_date_stock, output_format='pandas')\n",
- " returns_df.drop(columns=['open','high','low','volume'],inplace=True)\n",
- " \n",
- " # calculating returns and replacing closing price data\n",
- " returns_df = returns_df.pct_change() * 100\n",
- " returns_df.dropna(inplace=True)\n",
- " returns_df.isnull().sum()\n",
- " returns_df.rename(columns={'close':'return'}, inplace=True)\n",
- " \n",
- " # concatenating returns and sentiment scores DataFrames\n",
- " combined_df = pd.concat([df, returns_df], axis=1)\n",
- " \n",
- " # shifting the return column up to adjust for a lag in stock reaction to sentiments\n",
- " final_df = cleaned_df(combined_df)\n",
- " final_df['return'] = final_df['return'].shift(-lag)\n",
- " final_df.dropna(inplace=True)\n",
- " \n",
- " # exporting DataFrame of returns and sentiment scores for every trading day within the last 30 days as output\n",
- " return final_df"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Section 2: Machine Learning Model Function\n",
- "\n",
- "## This section defines a function to take in the news sentiment data as features and the stock price returns as targets, and feeds them into a machine learning model. The model will output predicted price movement, and model accuracy. "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 60,
- "metadata": {},
- "outputs": [],
- "source": [
- "# importing ML model libraries\n",
- "from sklearn.ensemble import RandomForestClassifier\n",
- "from sklearn.preprocessing import StandardScaler\n",
- "from sklearn.model_selection import train_test_split\n",
- "from sklearn.metrics import confusion_matrix, accuracy_score, classification_report\n",
- "from sklearn.metrics import balanced_accuracy_score\n",
- "from sklearn.metrics import confusion_matrix\n",
- "from sklearn import linear_model \n",
- "#from imblearn.metrics import classification_report_imbalanced"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 61,
- "metadata": {},
- "outputs": [],
- "source": [
- "# defining model to run Logit logistic regression model on the feature/target DataFrame\n",
- "# and export predicted price movement and model accuracy\n",
- "def model(df):\n",
- "\n",
- " # preparing the dataframe\n",
- " df['return_sign'] = np.sign(df['return'].values)\n",
- " df = df.drop(columns=['text'])\n",
- " \n",
- " # creating the features (X) and target (y) sets\n",
- " X = df.iloc[:, 0:4]\n",
- " y = df[\"return_sign\"]\n",
- " \n",
- " # creating training and testing data sets\n",
- " X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, shuffle=False, random_state=42) \n",
- " \n",
- " # fitting model\n",
- " M = 'Logit'\n",
- " lm = linear_model.LogisticRegression(solver = 'lbfgs')\n",
- " lm.fit(X_train, y_train)\n",
- " lm_pred = lm.predict(X_test)\n",
- " \n",
- " # calculating confusion matrix\n",
- " cm_lm = confusion_matrix(y_test, lm_pred)\n",
- " cm_lm_df = pd.DataFrame(\n",
- " cm_lm, index=[\"Actual -1\", \"Actual 1\"], columns=[\"Predicted -1\", \"Predicted 1\"]\n",
- " )\n",
- " \n",
- " # calculating the accuracy score\n",
- " acc_lm_score = balanced_accuracy_score(y_test, lm_pred)\n",
- " \n",
- " # exporting model accuracy and predicted price movement float variables as output\n",
- " return acc_lm_score, lm_pred[-1]"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Section 3: Buy/Sell Recommendation Function\n",
- "\n",
- "## This section is used to create the conditional statement function that will display the outputs of the ML model, and offer buy/sell recommendations based on them."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 62,
- "metadata": {},
- "outputs": [],
- "source": [
- "# creating conditional statement to determine buy/sell recommendations\n",
- "def conditionals(accuracy, predicted, model_df):\n",
- " \n",
- " # if model accuracy is greater than the threshold set by the user\n",
- " if (accuracy*100) > accuracy_selector.value:\n",
- " \n",
- " # then offer buy and sell recommendations\n",
- " if predicted == 1:\n",
- " output = f'{selector_widget.value}: With a composite news sentiment score of {model_df.iloc[-1][1]}, we are {accuracy*100}% confident there will be a price increase. Our recommendation: BUY.'\n",
- " else:\n",
- " output = f'{selector_widget.value}: With a composite news sentiment score of {model_df.iloc[-1][1]}, we are {accuracy*100}% confident there will be a price decrease. Our recommendation: SELL.' \n",
- " \n",
- " # if model accuracy is not greater than threshold set by the user\n",
- " else: \n",
- " \n",
- " # then inform the user the model is not accurate enough\n",
- " output = f'{selector_widget.value}: Model accuracy is only {accuracy*100}%, which does not meet your confidence threshold. We cannot provide an investment recommendation given this uncertainty.'\n",
- " \n",
- " # exporting recommendation or error text as output\n",
- " return output"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Section 4: User Interface\n",
- "\n",
- "## This section is used to create the ipywidgets that will be used to trigger running the model based on user inputs, and display the model outputs + recommendations. \n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 45,
- "metadata": {},
- "outputs": [],
- "source": [
- "# importing libraries\n",
- "import ipywidgets as widgets\n",
- "from IPython.display import display, clear_output"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 46,
- "metadata": {},
- "outputs": [],
- "source": [
- "# defining layout style\n",
- "layout = widgets.Layout(border='solid 1.5px')"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Stock Selection Widget"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 47,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- "\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " | \n",
- " Symbol | \n",
- " Name | \n",
- "
\n",
- " \n",
- " | Company | \n",
- " | \n",
- " | \n",
- "
\n",
- " \n",
- " \n",
- " \n",
- " | 3M Company | \n",
- " MMM | \n",
- " 3M Company | \n",
- "
\n",
- " \n",
- " | A.O. Smith Corp | \n",
- " AOS | \n",
- " A.O. Smith Corp | \n",
- "
\n",
- " \n",
- " | Abbott Laboratories | \n",
- " ABT | \n",
- " Abbott Laboratories | \n",
- "
\n",
- " \n",
- " | AbbVie Inc. | \n",
- " ABBV | \n",
- " AbbVie Inc. | \n",
- "
\n",
- " \n",
- " | Accenture plc | \n",
- " ACN | \n",
- " Accenture plc | \n",
- "
\n",
- " \n",
- "
\n",
- "
\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " | \n",
- " Symbol | \n",
- " Name | \n",
- "
\n",
- " \n",
- " | Company | \n",
- " | \n",
- " | \n",
- "
\n",
- " \n",
- " \n",
- " \n",
- " | 3M Company | \n",
- " MMM | \n",
- " 3M Company | \n",
- "
\n",
- " \n",
- " | A.O. Smith Corp | \n",
- " AOS | \n",
- " A.O. Smith Corp | \n",
- "
\n",
- " \n",
- " | Abbott Laboratories | \n",
- " ABT | \n",
- " Abbott Laboratories | \n",
- "
\n",
- " \n",
- " | AbbVie Inc. | \n",
- " ABBV | \n",
- " AbbVie Inc. | \n",
- "
\n",
- " \n",
- " | Accenture plc | \n",
- " ACN | \n",
- " Accenture plc | \n",
- "
\n",
- " \n",
- "
\n",
- "