From ea96f675c42988c650f3509e195d4bc24312105d Mon Sep 17 00:00:00 2001 From: Tiago Date: Wed, 22 Jan 2025 11:25:37 +0000 Subject: [PATCH] update --- .ipynb_checkpoints/Main-checkpoint.ipynb | 6 + NLP_main.ipynb | 3619 ++++++++++++++++++++++ model.pkl | Bin 0 -> 90851177 bytes news headline is real or fake news.pdf | Bin 0 -> 1118123 bytes vectorizer.pkl | Bin 0 -> 179739 bytes 5 files changed, 3625 insertions(+) create mode 100644 .ipynb_checkpoints/Main-checkpoint.ipynb create mode 100644 NLP_main.ipynb create mode 100644 model.pkl create mode 100644 news headline is real or fake news.pdf create mode 100644 vectorizer.pkl diff --git a/.ipynb_checkpoints/Main-checkpoint.ipynb b/.ipynb_checkpoints/Main-checkpoint.ipynb new file mode 100644 index 0000000..363fcab --- /dev/null +++ b/.ipynb_checkpoints/Main-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/NLP_main.ipynb b/NLP_main.ipynb new file mode 100644 index 0000000..30e184a --- /dev/null +++ b/NLP_main.ipynb @@ -0,0 +1,3619 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# %pip install pandas numpy re warnings" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "#%pip install nltk" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# \n", + "# %pip install scikit-learn" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# %pip install plotly" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "#%pip install kneed" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 3090, + "status": "ok", + "timestamp": 1737043596670, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "VRIM9b6MElD5", + "outputId": "ffe3fa16-8c09-4803-b0a4-9e71b65eacaf" + }, + "outputs": [], + "source": [ + "# 📚 Basic Libraries\n", + "\n", + "import pandas as pd\n", + "import numpy as np\n", + "import os\n", + "import re # Regular Expression\n", + "import warnings\n", + "\n", + "# 📝 Text Processing\n", + "import nltk\n", + "from nltk.stem import WordNetLemmatizer # to lemmatize the words #\n", + "from nltk.tokenize import word_tokenize\n", + "from nltk.tokenize import sent_tokenize # sentence tokenizer\n", + "from nltk.corpus import wordnet # to get the wordnet pos tags\n", + "from nltk.corpus import stopwords # to remove the stopwords\n", + "from sklearn.feature_extraction.text import CountVectorizer # to create a bag of words\n", + "\n", + "# Machine Learning\n", + "from sklearn.cluster import KMeans\n", + "import plotly.graph_objects as go\n", + "from sklearn.metrics import silhouette_score\n", + "\n", + "# Ensure kneed is installed\n", + "from kneed import KneeLocator" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OnHTm9l2ItAW" + }, + "source": [ + "***1***. **Load and Inspect the Data**" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training Data:\n", + " 0 \\\n", + "0 0 \n", + "1 0 \n", + "\n", + " donald trump sends out embarrassing new year‚s eve message; this is disturbing \n", + "0 drunk bragging trump staffer started russian c... \n", + "1 sheriff david clarke becomes an internet joke ... \n", + "\n", + "Testing Data:\n", + " 2 copycat muslim terrorist arrested with assault weapons\n", + "0 2 wow! chicago protester caught on camera admits... \n", + "1 2 germany's fdp look to fill schaeuble's big shoes \n" + ] + } + ], + "source": [ + "\n", + "# Define the folder path\n", + "folder_path = './dataset'\n", + "\n", + "# Define file paths\n", + "training_file = os.path.join(folder_path, 'training_data.csv')\n", + "testing_file = os.path.join(folder_path, 'testing_data.csv')\n", + "\n", + "# Load the CSV files\n", + "train_data = pd.read_csv(training_file, sep=\"\\t\")\n", + "test_data = pd.read_csv(testing_file, sep=\"\\t\")\n", + "\n", + "# Inspect the loaded data\n", + "print(\"Training Data:\")\n", + "print(train_data.head(2))\n", + "\n", + "print(\"\\nTesting Data:\")\n", + "print(test_data.head(2))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 1552, + "status": "ok", + "timestamp": 1737043598213, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "gF9NrzArC9ex", + "outputId": "66e7ece3-eaae-41fa-b18c-2ab9bbe4fe15" + }, + "outputs": [], + "source": [ + "#from google.colab import drive\n", + "#drive.mount('/content/drive')" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1737043598213, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "9jKl5a6rDL_x" + }, + "outputs": [], + "source": [ + "#\n", + "#folder_path = \"/content/drive/MyDrive/project-3-nlp/dataset/training_data.csv\"" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "executionInfo": { + "elapsed": 361, + "status": "ok", + "timestamp": 1737043598569, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "2E8hCRDGDq8S" + }, + "outputs": [], + "source": [ + "data = train_data\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "executionInfo": { + "elapsed": 4, + "status": "ok", + "timestamp": 1737043598569, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "TMaexyk7JAbZ" + }, + "outputs": [], + "source": [ + "# 🔧 Make your functions:\n", + "# Save this file as my_functions.py\n", + "# Import your functions in your notebook\n", + "# from my_functions import *\n", + "def clean_data(data):\n", + " \"\"\"\n", + " Standarize and returns snake_case columns\n", + " \"\"\"\n", + " data.columns = [column.lower().replace(' ', '_') for column in data.columns]\n", + "\n", + "\n", + "def map_pos_tag(word):\n", + " \"\"\"\n", + " Map POS tag to first character lemmatize() accepts.\n", + " \"\"\"\n", + " tag = nltk.pos_tag([word])[0][1][0].upper() # get the first character of the POS tag\n", + " tag_dict = { # dictionary to map POS tags\n", + " \"J\": wordnet.ADJ,\n", + " \"N\": wordnet.NOUN,\n", + " \"V\": wordnet.VERB,\n", + " \"R\": wordnet.ADV\n", + " }\n", + " return tag_dict.get(tag, wordnet.NOUN) # return the value of the key or the default value\n", + "\n", + "# ⚙️ Settings\n", + "pd.set_option('display.max_columns', None) # display all columns\n", + "warnings.filterwarnings('ignore') # ignore warnings" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "executionInfo": { + "elapsed": 3, + "status": "ok", + "timestamp": 1737043598569, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "vSe1_QV8KmU6" + }, + "outputs": [], + "source": [ + "df = data.copy()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "executionInfo": { + "elapsed": 3, + "status": "ok", + "timestamp": 1737043598569, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "tGVzSBrWbCPZ" + }, + "outputs": [], + "source": [ + "clean_data(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 206 + }, + "executionInfo": { + "elapsed": 265, + "status": "ok", + "timestamp": 1737043598831, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "NgdSn3P5E1zm", + "outputId": "327f0e58-359e-4627-dadc-b1a7ca998acb" + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
0donald_trump_sends_out_embarrassing_new_year‚s_eve_message;_this_is_disturbing
00drunk bragging trump staffer started russian c...
10sheriff david clarke becomes an internet joke ...
20trump is so obsessed he even has obama‚s name ...
30pope francis just called out donald trump duri...
40racist alabama cops brutalize black boy while ...
\n", + "
" + ], + "text/plain": [ + " 0 \\\n", + "0 0 \n", + "1 0 \n", + "2 0 \n", + "3 0 \n", + "4 0 \n", + "\n", + " donald_trump_sends_out_embarrassing_new_year‚s_eve_message;_this_is_disturbing \n", + "0 drunk bragging trump staffer started russian c... \n", + "1 sheriff david clarke becomes an internet joke ... \n", + "2 trump is so obsessed he even has obama‚s name ... \n", + "3 pope francis just called out donald trump duri... \n", + "4 racist alabama cops brutalize black boy while ... " + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 424 + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1737043598832, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "OdxZaVSLNliy", + "outputId": "3a896044-4bd1-42a7-ab7d-c6298a19607c" + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
classificationnews
00drunk bragging trump staffer started russian c...
10sheriff david clarke becomes an internet joke ...
20trump is so obsessed he even has obama‚s name ...
30pope francis just called out donald trump duri...
40racist alabama cops brutalize black boy while ...
.........
341461tears in rain as thais gather for late king's ...
341471pyongyang university needs non-u.s. teachers a...
341481philippine president duterte to visit japan ah...
341491japan's abe may have won election\\tbut many do...
341501demoralized and divided: inside catalonia's po...
\n", + "

34151 rows Ă— 2 columns

\n", + "
" + ], + "text/plain": [ + " classification news\n", + "0 0 drunk bragging trump staffer started russian c...\n", + "1 0 sheriff david clarke becomes an internet joke ...\n", + "2 0 trump is so obsessed he even has obama‚s name ...\n", + "3 0 pope francis just called out donald trump duri...\n", + "4 0 racist alabama cops brutalize black boy while ...\n", + "... ... ...\n", + "34146 1 tears in rain as thais gather for late king's ...\n", + "34147 1 pyongyang university needs non-u.s. teachers a...\n", + "34148 1 philippine president duterte to visit japan ah...\n", + "34149 1 japan's abe may have won election\\tbut many do...\n", + "34150 1 demoralized and divided: inside catalonia's po...\n", + "\n", + "[34151 rows x 2 columns]" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.columns = ['classification', 'news'] # classifictaion of the columns\n", + "df" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f5M65F-i3mN0" + }, + "source": [ + "***2***. **EDA**" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 147 + }, + "executionInfo": { + "elapsed": 11, + "status": "ok", + "timestamp": 1737043598832, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "IQ2TG9sON0fD", + "outputId": "cb472468-beef-4fc4-95c1-694da851dd9f" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "classification int64\n", + "news object\n", + "dtype: object" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.dtypes" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 178 + }, + "executionInfo": { + "elapsed": 26, + "status": "ok", + "timestamp": 1737043599096, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "x5GcGruFQ6xE", + "outputId": "34c20544-0d38-487f-fa01-49c1863f4dd9" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "classification\n", + "0 17571\n", + "1 16580\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_class = df.classification.value_counts()\n", + "df_class" + ] + }, + { + "cell_type": "code", + "execution_count": 88, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 178 + }, + "executionInfo": { + "elapsed": 25, + "status": "ok", + "timestamp": 1737043599097, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "sUK-CwvfOElY", + "outputId": "69ee3a87-378d-482a-a4d6-4d2ac5dfb4d7" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "classification\n", + "0 0.514509\n", + "1 0.485491\n", + "Name: count, dtype: float64" + ] + }, + "execution_count": 88, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "total_count_class = len(df['classification'])\n", + "\n", + "frequency_class = df_class / total_count_class\n", + "\n", + "frequency_class" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 147 + }, + "executionInfo": { + "elapsed": 24, + "status": "ok", + "timestamp": 1737043599097, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "aR6osh1jg9Nf", + "outputId": "21a3f9e0-361a-42ca-b293-01ee76c7d505" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "classification 0\n", + "news 0\n", + "dtype: int64" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.isna().sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 398 + }, + "executionInfo": { + "elapsed": 23, + "status": "ok", + "timestamp": 1737043599097, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "9rzO8uSXR19t", + "outputId": "9bad1a4b-a240-4ca3-9c9d-e5b6f6f6221d" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "7703 fox news shows real reaction from demoralized ...\n", + "23500 new u.s. u.n. envoy warns allies: back us or w...\n", + "25138 north carolina republican office painted with ...\n", + "13189 why hillary loves the idea of barack obama as ...\n", + "5280 father of purple heart-winning muslim soldier ...\n", + "4216 trump terrified of losing\n", + "33093 purge of saudi princes\\tbusinessmen widens\\ttr...\n", + "21432 the firing line: ouster of fbi's comey tests n...\n", + "162 this 2016 hannity tweet is now like a knife in...\n", + "12965 why trump supporters are laughing after wikile...\n", + "Name: news, dtype: object" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df['news'].sample(10)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 36 + }, + "executionInfo": { + "elapsed": 22, + "status": "ok", + "timestamp": 1737043599097, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "FS927o5IZq9H", + "outputId": "5dc83fef-3c02-43ca-a791-5e9c5dd5a934" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'sheriff david clarke becomes an internet joke for threatening to poke people ‚in the eye‚'" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "example1 = df['news'][1]\n", + "example1" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 36 + }, + "executionInfo": { + "elapsed": 22, + "status": "ok", + "timestamp": 1737043599098, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "MNuqSfoDZq5t", + "outputId": "857c4cb7-abf1-4363-db14-c9d071e592ae" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'trump is so obsessed he even has obama‚s name coded into his website (images)'" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "exemple2 = df['news'][2]\n", + "exemple2" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 36 + }, + "executionInfo": { + "elapsed": 21, + "status": "ok", + "timestamp": 1737043599098, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "R48Km1jFZqyc", + "outputId": "a3e1e1be-31b0-463d-8e4d-4230a6c068f0" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'racist alabama cops brutalize black boy while he is in handcuffs (graphic images)'" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "example4 = df['news'][4]\n", + "example4" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 36 + }, + "executionInfo": { + "elapsed": 369, + "status": "ok", + "timestamp": 1737043599447, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "D498WlayZqn4", + "outputId": "51c23b3a-96f9-4e4e-de5c-89f4ed0da79a" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'brand-new pro-trump ad features so much a** kissing it will make you sick'" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "example = df['news'][8]\n", + "example" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 36 + }, + "executionInfo": { + "elapsed": 25, + "status": "ok", + "timestamp": 1737043599447, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "ILaBwX-DYr_R", + "outputId": "21df434e-d9b5-4ed5-f6d6-d3a8d3443358" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'lindsey graham trashes media for portraying trump as ‚kooky‚ forgets his own words\"'" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "example = df['news'][12]\n", + "example" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 36 + }, + "executionInfo": { + "elapsed": 24, + "status": "ok", + "timestamp": 1737043599447, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "E4bIoM8Jc7V6", + "outputId": "35ad697d-6b19-4d3f-955c-6a47dede2f48" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'ky gop state rep. commits suicide over allegations he molested a teen girl (details)'" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "example11 = df['news'][20]\n", + "example11" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 36 + }, + "executionInfo": { + "elapsed": 23, + "status": "ok", + "timestamp": 1737043599447, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "UVd9r8pMZqiX", + "outputId": "65f3ae8d-eaf5-459d-8e40-317a1c1cc48a" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'snl hilariously mocks accused child molester roy moore for losing al senate race (video)'" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "example = df['news'][17]\n", + "example" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 36 + }, + "executionInfo": { + "elapsed": 22, + "status": "ok", + "timestamp": 1737043599447, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "MyZ6lTSWZq18", + "outputId": "fc930960-8919-45b2-a79f-1d546e429b65" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'meghan mccain tweets the most amazing response to doug jones‚ win in deep-red alabama'" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "example3 = df['news'][21]\n", + "example3" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 36 + }, + "executionInfo": { + "elapsed": 22, + "status": "ok", + "timestamp": 1737043599448, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "wr48DFLcZqvJ", + "outputId": "216c53a4-39b3-4c81-c44d-5352f5ff8aa3" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'cnn calls it: a democrat will represent alabama in the senate for the first time in 25 years'" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "example5 = df['news'][22]\n", + "example5" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 36 + }, + "executionInfo": { + "elapsed": 22, + "status": "ok", + "timestamp": 1737043599448, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "KC1tWjB9dTGp", + "outputId": "8bd4846a-f264-463d-8d01-5c4e8319ac20" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'tone deaf trump: congrats rep. scalise on losing weight after you almost died'" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "example4 = df['news'][14]\n", + "example4" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 36 + }, + "executionInfo": { + "elapsed": 21, + "status": "ok", + "timestamp": 1737043599448, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "Fs8N82vIZqrv", + "outputId": "562c5509-1b5e-4371-fb4c-5ada4763c583" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'mueller spokesman just f-cked up donald trump‚s christmas'" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "example = df['news'][16]\n", + "example" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 36 + }, + "executionInfo": { + "elapsed": 20, + "status": "ok", + "timestamp": 1737043599448, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "Os-TRBgRZqY4", + "outputId": "1a0e7057-3732-49b6-d97d-4dac596b6d50" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'republican senator gets dragged for going after robert mueller'" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "exemple10 = df['news'][18]\n", + "exemple10" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 36 + }, + "executionInfo": { + "elapsed": 20, + "status": "ok", + "timestamp": 1737043599448, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "AwJlGedtZqII", + "outputId": "dc3bc56d-f742-4d56-8631-3fe9016f2805" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'in a heartless rebuke to victims'" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "exemple11 = df['news'][19]\n", + "exemple11" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "p8ngkkLtmHJX" + }, + "source": [ + "Extra cleaning" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "executionInfo": { + "elapsed": 652, + "status": "ok", + "timestamp": 1737043600081, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "MwKywyOaiT36" + }, + "outputs": [], + "source": [ + "# probably i will joint to myfuntions\n", + "# listo of bad words\n", + "bad_words = [\"fcked\", \"sht\", \"fck\"]\n", + "\n", + "# Function to clean text\n", + "def clean_text(text):\n", + " text = text.lower() # Convert to lowercase\n", + " text = re.sub(r'http\\S+', '', text) # Remove URLs\n", + " text = re.sub(r'\\([^\\)]+\\)', '', text) # Remove text inside parentheses\n", + " text = re.sub(r'[^a-zA-Z\\s]', '', text) # Remove non-alphabetic characters\n", + " text = re.sub(r'\\s+', ' ', text).strip() # Remove extra whitespace\n", + " for bad_word in bad_words:\n", + " text = text.replace(bad_word, '') # Remove bad words\n", + " return text\n", + " return text\n", + "\n", + "# Apply the cleaning function\n", + "df['news'] = df['news'].apply(clean_text)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 36 + }, + "executionInfo": { + "elapsed": 12, + "status": "ok", + "timestamp": 1737043600081, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "mMcbnaAFj1hP", + "outputId": "bb728bb8-6945-4f78-af6b-53aa7859b141" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'trump sends crazytime tweet to the wrong account after losing his over world leaders remarks'" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "example21 = df['news'][50]\n", + "example21" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 36 + }, + "executionInfo": { + "elapsed": 11, + "status": "ok", + "timestamp": 1737043600081, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "rWv0rFjPkEh_", + "outputId": "6a41b62e-0720-437b-a0eb-b80a197a0b8c" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'just back from a golfing vacay'" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "example22 = df['news'][62]\n", + "example22" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 36 + }, + "executionInfo": { + "elapsed": 311, + "status": "ok", + "timestamp": 1737043600383, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "w51aeQkVkBoW", + "outputId": "a624c7fe-a8a5-48fe-a2f9-c32094479392" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'fox news bans gene simmons for life for harassing staff off camera'" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "example23 = df['news'][74]\n", + "example23" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 36 + }, + "executionInfo": { + "elapsed": 15, + "status": "ok", + "timestamp": 1737043600384, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "a99o5jGBkEfb", + "outputId": "8fc7e3c9-a6d7-4d52-8cdf-3f6879d18c1d" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'newsweek just made their best cover ever and people are freaking out'" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "example24 = df['news'][108]\n", + "example24" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 36 + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1737043600384, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "yKdbT8UAkEcD", + "outputId": "58f4f554-9700-4396-c356-8e5b5c1ae2ca" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'trump tells japanese diplomats he never knew there were so many countries in the world'" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "example25 = df['news'][126]\n", + "example25" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 677 + }, + "executionInfo": { + "elapsed": 12, + "status": "ok", + "timestamp": 1737043600384, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "vCA6-4u3mbpT", + "outputId": "006c0d7e-9950-4170-f22c-8d118f3b46d8" + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
classificationnews
111840lawyer for illegal alien rapists hysteria over...
145360epic backfire the left makes video warning fol...
239431att chief executive trump meet amid planned ti...
327771tillerson speaks to saudi counterpart about ar...
299781italy national election likely set for march p...
293261un mediator de mistura to attend syria talks o...
227641trump trade office takes aggressive view of wt...
166490obama suggests gun confiscation is solutioncit...
283631obama to make historic trip to cuba in coming ...
329011senior thai royal official dismissed in latest...
317601us sanctions chinese and north korean organiza...
176001senate leader mcconnell sees a more collegial
300121window falls from us marines helicopter onto s...
247891chinas interference in hong kong reaching alar...
179891us attorney general sessions evasive on russia...
287121us appeals court declines to block obama carbo...
238531putin says he doubts trump met with moscow pro...
110720sec of state rex tillerson steps are under way...
200611senate aims for a skinny obamacare repeal as o...
245541senior senators want to amend saudi september law
\n", + "
" + ], + "text/plain": [ + " classification news\n", + "11184 0 lawyer for illegal alien rapists hysteria over...\n", + "14536 0 epic backfire the left makes video warning fol...\n", + "23943 1 att chief executive trump meet amid planned ti...\n", + "32777 1 tillerson speaks to saudi counterpart about ar...\n", + "29978 1 italy national election likely set for march p...\n", + "29326 1 un mediator de mistura to attend syria talks o...\n", + "22764 1 trump trade office takes aggressive view of wt...\n", + "16649 0 obama suggests gun confiscation is solutioncit...\n", + "28363 1 obama to make historic trip to cuba in coming ...\n", + "32901 1 senior thai royal official dismissed in latest...\n", + "31760 1 us sanctions chinese and north korean organiza...\n", + "17600 1 senate leader mcconnell sees a more collegial\n", + "30012 1 window falls from us marines helicopter onto s...\n", + "24789 1 chinas interference in hong kong reaching alar...\n", + "17989 1 us attorney general sessions evasive on russia...\n", + "28712 1 us appeals court declines to block obama carbo...\n", + "23853 1 putin says he doubts trump met with moscow pro...\n", + "11072 0 sec of state rex tillerson steps are under way...\n", + "20061 1 senate aims for a skinny obamacare repeal as o...\n", + "24554 1 senior senators want to amend saudi september law" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.sample(20) # found some bad names at the first time, however they are incomplete" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lAms-mbn3-EJ" + }, + "source": [ + "***3***. **Text processing**" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JiND6eDIl8J-" + }, + "source": [ + "Tokenization" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 11, + "status": "ok", + "timestamp": 1737043600384, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "48-PO0AmkEJX", + "outputId": "7a52a5c0-6fb3-4f3b-ae8d-c4df2c47b194" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[nltk_data] Downloading package punkt_tab to\n", + "[nltk_data] C:\\Users\\ggsan\\AppData\\Roaming\\nltk_data...\n", + "[nltk_data] Package punkt_tab is already up-to-date!\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nltk.download('punkt_tab') #tokenizer package however in googlecolab is \"punkt_tab\" instead of \"punkt\"" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 9, + "status": "ok", + "timestamp": 1737043600385, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "9QATpKZrkEDY", + "outputId": "319cfe31-bc93-428c-eb05-8b6013649de1" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['mueller spokesman just f-cked up donald trump‚s christmas']" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "example.split(\",\")" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 8, + "status": "ok", + "timestamp": 1737043600385, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "Uhfc7nEGsu1O", + "outputId": "6e3f1059-831c-471d-c90e-1edd72d120df" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['mueller',\n", + " 'spokesman',\n", + " 'just',\n", + " 'f-cked',\n", + " 'up',\n", + " 'donald',\n", + " 'trump‚s',\n", + " 'christmas']" + ] + }, + "execution_count": 43, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokens = nltk.word_tokenize(example)\n", + "tokens[:10]" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 258, + "status": "ok", + "timestamp": 1737043600637, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "qapo_oJmsuwA", + "outputId": "af9287e8-7457-4f20-a5aa-cadff5d2747b" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "8" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(tokens)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IEvuPbne5NNi" + }, + "source": [ + "**Tokenization** of the Data" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 206 + }, + "executionInfo": { + "elapsed": 8886, + "status": "ok", + "timestamp": 1737043609521, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "COkg0OwssupK", + "outputId": "06bd3182-23e5-4316-bd07-1ff513ce86a4" + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
classificationnewstokens
00drunk bragging trump staffer started russian c...[drunk, bragging, trump, staffer, started, rus...
10sheriff david clarke becomes an internet joke ...[sheriff, david, clarke, becomes, an, internet...
20trump is so obsessed he even has obamas name c...[trump, is, so, obsessed, he, even, has, obama...
30pope francis just called out donald trump duri...[pope, francis, just, called, out, donald, tru...
40racist alabama cops brutalize black boy while ...[racist, alabama, cops, brutalize, black, boy,...
\n", + "
" + ], + "text/plain": [ + " classification news \\\n", + "0 0 drunk bragging trump staffer started russian c... \n", + "1 0 sheriff david clarke becomes an internet joke ... \n", + "2 0 trump is so obsessed he even has obamas name c... \n", + "3 0 pope francis just called out donald trump duri... \n", + "4 0 racist alabama cops brutalize black boy while ... \n", + "\n", + " tokens \n", + "0 [drunk, bragging, trump, staffer, started, rus... \n", + "1 [sheriff, david, clarke, becomes, an, internet... \n", + "2 [trump, is, so, obsessed, he, even, has, obama... \n", + "3 [pope, francis, just, called, out, donald, tru... \n", + "4 [racist, alabama, cops, brutalize, black, boy,... " + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df['tokens'] = df['news'].apply(nltk.word_tokenize)\n", + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 11, + "status": "ok", + "timestamp": 1737043609521, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "tEvDCx9Vsug2", + "outputId": "1f2010bf-2747-4960-df92-d81d494b8b31" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "Index(['classification', 'news', 'tokens'], dtype='object')" + ] + }, + "execution_count": 46, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.columns" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LsHxwDy7rirN" + }, + "source": [ + "NLTK Download packages" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 8, + "status": "ok", + "timestamp": 1737043609521, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "1mPwai4brf3P", + "outputId": "e055aa1f-b041-4757-f00b-b4efc5c1c81a" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[nltk_data] Downloading package averaged_perceptron_tagger_eng to\n", + "[nltk_data] C:\\Users\\ggsan\\AppData\\Roaming\\nltk_data...\n", + "[nltk_data] Package averaged_perceptron_tagger_eng is already up-to-\n", + "[nltk_data] date!\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 47, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import nltk\n", + "nltk.download('averaged_perceptron_tagger_eng') # This is a resource to determine the part-of-speech of a word." + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": { + "executionInfo": { + "elapsed": 41828, + "status": "ok", + "timestamp": 1737043651345, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "iXpJ48KSrzyz" + }, + "outputs": [], + "source": [ + "df['pos_tags'] = df['tokens'].apply(lambda tokens: nltk.pos_tag(tokens, lang='eng'))" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 206 + }, + "executionInfo": { + "elapsed": 8, + "status": "ok", + "timestamp": 1737043651346, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "oiDKwfLCrzu6", + "outputId": "3a8c0581-2eee-4e90-bb62-7b58245ad331" + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
classificationnewstokenspos_tags
210501vatican would see us paris deal exit as slap i...[vatican, would, see, us, paris, deal, exit, a...[(vatican, JJ), (would, MD), (see, VB), (us, P...
223401key democrat cites profound doubt about intell...[key, democrat, cites, profound, doubt, about,...[(key, JJ), (democrat, NN), (cites, NNS), (pro...
20750ivanka trump just actually became our new firs...[ivanka, trump, just, actually, became, our, n...[(ivanka, JJ), (trump, NN), (just, RB), (actua...
274081us capitol replacing flag display over confede...[us, capitol, replacing, flag, display, over, ...[(us, PRP), (capitol, VBP), (replacing, VBG), ...
305841spd leader promises to push germany to embrace...[spd, leader, promises, to, push, germany, to,...[(spd, JJ), (leader, NN), (promises, NNS), (to...
\n", + "
" + ], + "text/plain": [ + " classification news \\\n", + "21050 1 vatican would see us paris deal exit as slap i... \n", + "22340 1 key democrat cites profound doubt about intell... \n", + "2075 0 ivanka trump just actually became our new firs... \n", + "27408 1 us capitol replacing flag display over confede... \n", + "30584 1 spd leader promises to push germany to embrace... \n", + "\n", + " tokens \\\n", + "21050 [vatican, would, see, us, paris, deal, exit, a... \n", + "22340 [key, democrat, cites, profound, doubt, about,... \n", + "2075 [ivanka, trump, just, actually, became, our, n... \n", + "27408 [us, capitol, replacing, flag, display, over, ... \n", + "30584 [spd, leader, promises, to, push, germany, to,... \n", + "\n", + " pos_tags \n", + "21050 [(vatican, JJ), (would, MD), (see, VB), (us, P... \n", + "22340 [(key, JJ), (democrat, NN), (cites, NNS), (pro... \n", + "2075 [(ivanka, JJ), (trump, NN), (just, RB), (actua... \n", + "27408 [(us, PRP), (capitol, VBP), (replacing, VBG), ... \n", + "30584 [(spd, JJ), (leader, NN), (promises, NNS), (to... " + ] + }, + "execution_count": 49, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.sample(5)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7sqBWc8jtZ5E" + }, + "source": [ + "**Stemming** the Data with Porter Stemmer" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": { + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1737043651346, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "nnHbSLs7sFDr" + }, + "outputs": [], + "source": [ + "from nltk.stem import PorterStemmer" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": { + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1737043651346, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "Pom5ykR9rzq-" + }, + "outputs": [], + "source": [ + "ps = PorterStemmer()" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": { + "executionInfo": { + "elapsed": 6177, + "status": "ok", + "timestamp": 1737043657517, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "46nXeDaEso6K" + }, + "outputs": [], + "source": [ + "# Applying Stemmed to tokens\n", + "df['stemmed'] = df['tokens'].apply(lambda tokens_list: [ps.stem(word) for word in tokens_list])" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 206 + }, + "executionInfo": { + "elapsed": 15, + "status": "ok", + "timestamp": 1737043657518, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "SyrqD53ysJau", + "outputId": "47d17fed-8a06-4212-edab-f2042ea69272" + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
classificationnewstokenspos_tagsstemmed
139470obamas epa gestapo to skip hearing on co mine[obamas, epa, gestapo, to, skip, hearing, on, ...[(obamas, RB), (epa, JJ), (gestapo, NN), (to, ...[obama, epa, gestapo, to, skip, hear, on, co, ...
322421china foreign minister to visit myanmar amid r...[china, foreign, minister, to, visit, myanmar,...[(china, NNS), (foreign, JJ), (minister, NN), ...[china, foreign, minist, to, visit, myanmar, a...
98320look who the view hired to replace only conser...[look, who, the, view, hired, to, replace, onl...[(look, NN), (who, WP), (the, DT), (view, NN),...[look, who, the, view, hire, to, replac, onli,...
178561us appeals court hears arguments on trump trav...[us, appeals, court, hears, arguments, on, tru...[(us, PRP), (appeals, NNS), (court, NN), (hear...[us, appeal, court, hear, argument, on, trump,...
119130prince charles uses christmas radio broadcast ...[prince, charles, uses, christmas, radio, broa...[(prince, NN), (charles, NNS), (uses, VBZ), (c...[princ, charl, use, christma, radio, broadcast...
\n", + "
" + ], + "text/plain": [ + " classification news \\\n", + "13947 0 obamas epa gestapo to skip hearing on co mine \n", + "32242 1 china foreign minister to visit myanmar amid r... \n", + "9832 0 look who the view hired to replace only conser... \n", + "17856 1 us appeals court hears arguments on trump trav... \n", + "11913 0 prince charles uses christmas radio broadcast ... \n", + "\n", + " tokens \\\n", + "13947 [obamas, epa, gestapo, to, skip, hearing, on, ... \n", + "32242 [china, foreign, minister, to, visit, myanmar,... \n", + "9832 [look, who, the, view, hired, to, replace, onl... \n", + "17856 [us, appeals, court, hears, arguments, on, tru... \n", + "11913 [prince, charles, uses, christmas, radio, broa... \n", + "\n", + " pos_tags \\\n", + "13947 [(obamas, RB), (epa, JJ), (gestapo, NN), (to, ... \n", + "32242 [(china, NNS), (foreign, JJ), (minister, NN), ... \n", + "9832 [(look, NN), (who, WP), (the, DT), (view, NN),... \n", + "17856 [(us, PRP), (appeals, NNS), (court, NN), (hear... \n", + "11913 [(prince, NN), (charles, NNS), (uses, VBZ), (c... \n", + "\n", + " stemmed \n", + "13947 [obama, epa, gestapo, to, skip, hear, on, co, ... \n", + "32242 [china, foreign, minist, to, visit, myanmar, a... \n", + "9832 [look, who, the, view, hire, to, replac, onli,... \n", + "17856 [us, appeal, court, hear, argument, on, trump,... \n", + "11913 [princ, charl, use, christma, radio, broadcast... " + ] + }, + "execution_count": 53, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.sample(5)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ni3RROR0tsWO" + }, + "source": [ + "**Lemmatization** the Data with wordnet" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 12, + "status": "ok", + "timestamp": 1737043657518, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "oRYIdmKXsJW3", + "outputId": "3bd278cd-9986-4208-f0bd-1cf15f2bdef9" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[nltk_data] Downloading package wordnet to\n", + "[nltk_data] C:\\Users\\ggsan\\AppData\\Roaming\\nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 54, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nltk.download('wordnet')" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": { + "executionInfo": { + "elapsed": 9, + "status": "ok", + "timestamp": 1737043657518, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "4MnMhQK_sJTS" + }, + "outputs": [], + "source": [ + "from nltk.stem import WordNetLemmatizer # lemmatize\n", + "from nltk.corpus import wordnet # wordnet is a lexical database for the English language" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": { + "executionInfo": { + "elapsed": 8, + "status": "ok", + "timestamp": 1737043657518, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "J30B_OGisJQI" + }, + "outputs": [], + "source": [ + "lemmatizer = WordNetLemmatizer()" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": { + "executionInfo": { + "elapsed": 35055, + "status": "ok", + "timestamp": 1737043692565, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "KdZWRnuru802" + }, + "outputs": [], + "source": [ + "# Applying lemmatiz to Stemmed\n", + "df['lemmatized'] = df['stemmed'].apply(lambda tokens_list: [lemmatizer.lemmatize(word, pos=map_pos_tag(word)) for word in tokens_list])" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 293 + }, + "executionInfo": { + "elapsed": 23, + "status": "ok", + "timestamp": 1737043692567, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "4EF6zBxfsJF9", + "outputId": "2e30fd83-4728-46f8-85e1-3635ad5eccd8" + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
classificationnewstokenspos_tagsstemmedlemmatized
15600trump hired flynn after obama specifically war...[trump, hired, flynn, after, obama, specifical...[(trump, NN), (hired, VBD), (flynn, NN), (afte...[trump, hire, flynn, after, obama, specif, war...[trump, hire, flynn, after, obama, specif, war...
24290swedish police appalled at level of deception ...[swedish, police, appalled, at, level, of, dec...[(swedish, JJ), (police, NN), (appalled, VBD),...[swedish, polic, appal, at, level, of, decept,...[swedish, polic, appal, at, level, of, decept,...
219691russian tv says us missile strikes on syria de...[russian, tv, says, us, missile, strikes, on, ...[(russian, JJ), (tv, NN), (says, VBZ), (us, PR...[russian, tv, say, us, missil, strike, on, syr...[russian, tv, say, u, missil, strike, on, syri...
93890democrat senator al franken thought it would b...[democrat, senator, al, franken, thought, it, ...[(democrat, NNS), (senator, VBP), (al, IN), (f...[democrat, senat, al, franken, thought, it, wo...[democrat, senat, al, franken, thought, it, wo...
307751saleh was killed in rpg gun attack on his car ...[saleh, was, killed, in, rpg, gun, attack, on,...[(saleh, NN), (was, VBD), (killed, VBN), (in, ...[saleh, wa, kill, in, rpg, gun, attack, on, hi...[saleh, wa, kill, in, rpg, gun, attack, on, hi...
\n", + "
" + ], + "text/plain": [ + " classification news \\\n", + "1560 0 trump hired flynn after obama specifically war... \n", + "2429 0 swedish police appalled at level of deception ... \n", + "21969 1 russian tv says us missile strikes on syria de... \n", + "9389 0 democrat senator al franken thought it would b... \n", + "30775 1 saleh was killed in rpg gun attack on his car ... \n", + "\n", + " tokens \\\n", + "1560 [trump, hired, flynn, after, obama, specifical... \n", + "2429 [swedish, police, appalled, at, level, of, dec... \n", + "21969 [russian, tv, says, us, missile, strikes, on, ... \n", + "9389 [democrat, senator, al, franken, thought, it, ... \n", + "30775 [saleh, was, killed, in, rpg, gun, attack, on,... \n", + "\n", + " pos_tags \\\n", + "1560 [(trump, NN), (hired, VBD), (flynn, NN), (afte... \n", + "2429 [(swedish, JJ), (police, NN), (appalled, VBD),... \n", + "21969 [(russian, JJ), (tv, NN), (says, VBZ), (us, PR... \n", + "9389 [(democrat, NNS), (senator, VBP), (al, IN), (f... \n", + "30775 [(saleh, NN), (was, VBD), (killed, VBN), (in, ... \n", + "\n", + " stemmed \\\n", + "1560 [trump, hire, flynn, after, obama, specif, war... \n", + "2429 [swedish, polic, appal, at, level, of, decept,... \n", + "21969 [russian, tv, say, us, missil, strike, on, syr... \n", + "9389 [democrat, senat, al, franken, thought, it, wo... \n", + "30775 [saleh, wa, kill, in, rpg, gun, attack, on, hi... \n", + "\n", + " lemmatized \n", + "1560 [trump, hire, flynn, after, obama, specif, war... \n", + "2429 [swedish, polic, appal, at, level, of, decept,... \n", + "21969 [russian, tv, say, u, missil, strike, on, syri... \n", + "9389 [democrat, senat, al, franken, thought, it, wo... \n", + "30775 [saleh, wa, kill, in, rpg, gun, attack, on, hi... " + ] + }, + "execution_count": 58, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.sample(5)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iFMW_pxnzzXw" + }, + "source": [ + "**StopWord Removal**" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": { + "executionInfo": { + "elapsed": 21, + "status": "ok", + "timestamp": 1737043692567, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "YrDQgOensJCh" + }, + "outputs": [], + "source": [ + "from nltk.corpus import stopwords" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 20, + "status": "ok", + "timestamp": 1737043692567, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "uNWkgAV-sI_L", + "outputId": "5636c480-1423-4d4d-e242-13680fa58e15" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[nltk_data] Downloading package stopwords to\n", + "[nltk_data] C:\\Users\\ggsan\\AppData\\Roaming\\nltk_data...\n", + "[nltk_data] Package stopwords is already up-to-date!\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 60, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nltk.download('stopwords')" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": { + "executionInfo": { + "elapsed": 36891, + "status": "ok", + "timestamp": 1737043869298, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "ZvBj9gH3sI71" + }, + "outputs": [], + "source": [ + "df['news_w_out_sw'] = df['lemmatized'].apply(lambda tokens_list: [word for word in tokens_list if word not in stopwords.words('english')])" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 196 + }, + "executionInfo": { + "elapsed": 268, + "status": "ok", + "timestamp": 1737043872604, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "-6S7Ygz8sI4s", + "outputId": "b10b10b6-8e0d-4cb6-87bb-b06ecdd2b6a5" + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
classificationnewstokenspos_tagsstemmedlemmatizednews_w_out_sw
135360btch of benghazi wins democrat nomination im e...[btch, of, benghazi, wins, democrat, nominatio...[(btch, NN), (of, IN), (benghazi, NN), (wins, ...[btch, of, benghazi, win, democrat, nomin, im,...[btch, of, benghazi, win, democrat, nomin, im,...[btch, benghazi, win, democrat, nomin, im, enc...
302501mexico rightleft coalition presents bid to run...[mexico, rightleft, coalition, presents, bid, ...[(mexico, NN), (rightleft, NN), (coalition, NN...[mexico, rightleft, coalit, present, bid, to, ...[mexico, rightleft, coalit, present, bid, to, ...[mexico, rightleft, coalit, present, bid, run,...
138310trumps got em on the run rhode island supporte...[trumps, got, em, on, the, run, rhode, island,...[(trumps, NNS), (got, VBD), (em, RB), (on, IN)...[trump, got, em, on, the, run, rhode, island, ...[trump, get, em, on, the, run, rhode, island, ...[trump, get, em, run, rhode, island, support, ...
\n", + "
" + ], + "text/plain": [ + " classification news \\\n", + "13536 0 btch of benghazi wins democrat nomination im e... \n", + "30250 1 mexico rightleft coalition presents bid to run... \n", + "13831 0 trumps got em on the run rhode island supporte... \n", + "\n", + " tokens \\\n", + "13536 [btch, of, benghazi, wins, democrat, nominatio... \n", + "30250 [mexico, rightleft, coalition, presents, bid, ... \n", + "13831 [trumps, got, em, on, the, run, rhode, island,... \n", + "\n", + " pos_tags \\\n", + "13536 [(btch, NN), (of, IN), (benghazi, NN), (wins, ... \n", + "30250 [(mexico, NN), (rightleft, NN), (coalition, NN... \n", + "13831 [(trumps, NNS), (got, VBD), (em, RB), (on, IN)... \n", + "\n", + " stemmed \\\n", + "13536 [btch, of, benghazi, win, democrat, nomin, im,... \n", + "30250 [mexico, rightleft, coalit, present, bid, to, ... \n", + "13831 [trump, got, em, on, the, run, rhode, island, ... \n", + "\n", + " lemmatized \\\n", + "13536 [btch, of, benghazi, win, democrat, nomin, im,... \n", + "30250 [mexico, rightleft, coalit, present, bid, to, ... \n", + "13831 [trump, get, em, on, the, run, rhode, island, ... \n", + "\n", + " news_w_out_sw \n", + "13536 [btch, benghazi, win, democrat, nomin, im, enc... \n", + "30250 [mexico, rightleft, coalit, present, bid, run,... \n", + "13831 [trump, get, em, run, rhode, island, support, ... " + ] + }, + "execution_count": 62, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.sample(3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Spliting the data**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "df_class\n" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "classification\n", + "0 17571\n", + "1 16580\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 63, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_class" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "classification\n", + "0 0.514509\n", + "1 0.485491\n", + "Name: count, dtype: float64" + ] + }, + "execution_count": 64, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "frequency_class" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 34151 entries, 0 to 34150\n", + "Data columns (total 7 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 classification 34151 non-null int64 \n", + " 1 news 34151 non-null object\n", + " 2 tokens 34151 non-null object\n", + " 3 pos_tags 34151 non-null object\n", + " 4 stemmed 34151 non-null object\n", + " 5 lemmatized 34151 non-null object\n", + " 6 news_w_out_sw 34151 non-null object\n", + "dtypes: int64(1), object(6)\n", + "memory usage: 1.8+ MB\n" + ] + } + ], + "source": [ + "df.info()" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "classification 0\n", + "news 0\n", + "tokens 0\n", + "pos_tags 0\n", + "stemmed 0\n", + "lemmatized 0\n", + "news_w_out_sw 0\n", + "dtype: int64" + ] + }, + "execution_count": 66, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.isna().sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [], + "source": [ + "# 34151 * 0.8 # result = 27320.800000000003 *" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": {}, + "outputs": [], + "source": [ + "# train_data = df.sample(1000)\n", + "# test_data = df.sample(1000)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4PtRWPeF4d62" + }, + "source": [ + "***4*** . **Feature extraction**" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "g7KdqJai5Ru-" + }, + "source": [ + "**Bag-of words** vectorization" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": { + "executionInfo": { + "elapsed": 222, + "status": "ok", + "timestamp": 1737044176425, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "t-kScL4rsIy1" + }, + "outputs": [], + "source": [ + "from sklearn.feature_extraction.text import CountVectorizer\n", + "\n", + "# Join tokens back into strings\n", + "df['news_w_out_sw_joined'] = df['news_w_out_sw'].apply(lambda tokens: ' '.join(tokens))\n", + "\n", + "# Initialize CountVectorizer\n", + "vectorizer = CountVectorizer()\n", + "\n", + "# Fit and transform text data\n", + "X = vectorizer.fit_transform(df['news_w_out_sw_joined'])" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.feature_extraction.text import TfidfVectorizer\n", + "\n", + "# Initialize TfidfVectorizer\n", + "tfidf = TfidfVectorizer()\n", + "\n", + "# Fit and transform text data\n", + "X = tfidf.fit_transform(df['news_w_out_sw_joined'])\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***5*** . **Model**" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": { + "executionInfo": { + "elapsed": 5, + "status": "aborted", + "timestamp": 1737043692785, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "2WiqsM400ALS" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train data size: 27320\n", + "Test data size: 6831\n" + ] + } + ], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "\n", + "# train test split\n", + "train_data, test_data = train_test_split(df, test_size=0.2, random_state=42)\n", + "\n", + "# Check the sizes of the splits\n", + "print(f\"Train data size: {len(train_data)}\")\n", + "print(f\"Test data size: {len(test_data)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": { + "executionInfo": { + "elapsed": 4, + "status": "aborted", + "timestamp": 1737043692785, + "user": { + "displayName": "Tiago Santos", + "userId": "01739492997383215152" + }, + "user_tz": 0 + }, + "id": "9YGp28uQsIpe" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training data class\n" + ] + }, + { + "data": { + "text/plain": [ + "classification\n", + "0 14054\n", + "1 13266\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 72, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Ensure the column names are standardized\n", + "clean_data(train_data)\n", + "\n", + "# Rename the columns if necessary\n", + "train_data.rename(columns={'0': 'classification', '1': 'news_w_out_sw_joined'}, inplace=True)\n", + "\n", + "# Now access the 'classification' column\n", + "print(f\"Training data class\")\n", + "train_data[\"classification\"].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Test data class\n" + ] + }, + { + "data": { + "text/plain": [ + "classification\n", + "0 3517\n", + "1 3314\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 73, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Ensure the column names are standardized\n", + "clean_data(test_data)\n", + "\n", + "# Select only the relevant columns\n", + "test_data = test_data[['classification', 'news_w_out_sw_joined']]\n", + "\n", + "# Now access the 'classification' column\n", + "print(f\"Test data class\")\n", + "test_data[\"classification\"].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.feature_extraction.text import TfidfVectorizer\n", + "\n", + "# Define X_train and X_test\n", + "X_train = train_data['news_w_out_sw_joined']\n", + "X_test = test_data['news_w_out_sw_joined']\n", + "\n", + "# Initialize the vectorizer\n", + "vectorizer = TfidfVectorizer(max_features=5000)\n", + "\n", + "# Transform the text data\n", + "X_train_vectorized = vectorizer.fit_transform(X_train)\n", + "X_test_vectorized = vectorizer.transform(X_test)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 75, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.ensemble import RandomForestClassifier\n", + "from sklearn.metrics import classification_report, accuracy_score\n", + "\n", + "# Define y_train and y_test\n", + "y_train = train_data['classification']\n", + "y_test = test_data['classification']\n", + "\n", + "# Initialize and train the classifier\n", + "classifier = RandomForestClassifier()\n", + "classifier.fit(X_train_vectorized, y_train)\n", + "\n", + "# Make predictions\n", + "y_pred = classifier.predict(X_test_vectorized)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy: 0.8997218562435954\n", + "\n", + "Classification Report:\n", + " precision recall f1-score support\n", + "\n", + " 0 0.90 0.91 0.90 3517\n", + " 1 0.90 0.89 0.90 3314\n", + "\n", + " accuracy 0.90 6831\n", + " macro avg 0.90 0.90 0.90 6831\n", + "weighted avg 0.90 0.90 0.90 6831\n", + "\n" + ] + } + ], + "source": [ + "# Evaluate the model\n", + "print(\"Accuracy:\", accuracy_score(y_test, y_pred))\n", + "print(\"\\nClassification Report:\\n\", classification_report(y_test, y_pred))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 98, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgYAAAGJCAYAAADxMfswAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAARelJREFUeJzt3Qd8FFXX+PFDQoihhA4JvZcgIE1EkSJI76CgPJSXIiBB6RjpiIABBaQqKNhQEAQVBEQQULp0EJASRKUjRQKh7v9zLv9ds9kE0zfZ+X2fz7y7OzM7O7vmZc6ce+69aWw2m00AAABExMvdJwAAAFIOAgMAAOBAYAAAABwIDAAAgAOBAQAAcCAwAAAADgQGAADAgcAAAAA4EBgAAAAHAgMglo4ePSr16tWTzJkzS5o0aWTZsmWJevyTJ0+a486fPz9Rj5ua1apVyywAkg+BAVKV48ePS48ePaRIkSLyyCOPiL+/vzz11FMydepUuXnzZpJ+dqdOnWT//v3y5ptvyieffCKVK1cWT9G5c2cTlOjvGd3vqEGRbtdl0qRJcT7+6dOnZdSoUbJnz55EOmMASSVtkh0ZSGQrVqyQ5557Tnx9faVjx47y6KOPyu3bt+Xnn3+WQYMGycGDB+X9999Pks/Wi+WWLVtk6NChEhwcnCSfUbBgQfM5Pj4+4g5p06aVGzduyLfffivPP/+807bPPvvMBGIRERHxOrYGBqNHj5ZChQrJY489Fuv3ff/99/H6PADxR2CAVCEsLEzatWtnLp7r1q2TwMBAx7bevXvLsWPHTOCQVC5cuGAes2TJkmSfoXfjevF1Fw24NPvy+eefuwQGCxYskMaNG8uSJUuS5Vw0QEmfPr2kS5cuWT4PwL9oSkCqEBoaKtevX5cPPvjAKSiwK1asmLz66quO13fv3pU33nhDihYtai54eqf6+uuvy61bt5zep+ubNGlisg6PP/64uTBrM8XHH3/s2EdT4BqQKM1M6AVc32dPwdufR6bv0f0iW7NmjVSvXt0EFxkzZpSSJUuac/qvGgMNhJ5++mnJkCGDeW/z5s3l0KFD0X6eBkh6Trqf1kL83//9n7nIxtaLL74oK1eulCtXrjjW7dixwzQl6Lao/v77bxk4cKCULVvWfCdtimjYsKHs3bvXsc/69eulSpUq5rmej71Jwv49tYZAsz87d+6UGjVqmIDA/rtErTHQ5hz9bxT1+9evX1+yZs1qMhMAEobAAKmCprf1gv3kk0/Gav9u3brJiBEjpGLFijJ58mSpWbOmjB8/3mQdotKLaZs2beTZZ5+Vt99+21xg9OKqTROqVatW5hjqhRdeMPUFU6ZMidP567E0ANHAZMyYMeZzmjVrJps2bXro+3744Qdz0Tt//ry5+Pfv3182b95s7uw1kIhK7/T/+ecf8131uV58NYUfW/pd9aL91VdfOWULSpUqZX7LqE6cOGGKMPW7vfPOOyZw0joM/b3tF+nSpUub76xeeukl8/vpokGA3aVLl0xAoc0M+tvWrl072vPTWpKcOXOaAOHevXtm3XvvvWeaHKZNmyZ58uSJ9XcFEAMbkMJdvXrVpn+qzZs3j9X+e/bsMft369bNaf3AgQPN+nXr1jnWFSxY0KzbuHGjY9358+dtvr6+tgEDBjjWhYWFmf0mTpzodMxOnTqZY0Q1cuRIs7/d5MmTzesLFy7EeN72z5g3b55j3WOPPWbLlSuX7dKlS451e/futXl5edk6duzo8nldunRxOmbLli1t2bNnj/EzI3+PDBkymOdt2rSx1alTxzy/d++eLSAgwDZ69Ohof4OIiAizT9Tvob/fmDFjHOt27Njh8t3satasabbNnj072m26RLZ69Wqz/9ixY20nTpywZcyY0daiRYv//I4AYoeMAVK8a9eumcdMmTLFav/vvvvOPOrddWQDBgwwj1FrEYKCgkyq3k7vSDXNr3fDicVem/D111/L/fv3Y/WeM2fOmCp+zV5ky5bNsb5cuXImu2H/npH17NnT6bV+L70bt/+GsaFNBpr+P3v2rGnG0MfomhGUNtN4eT34Z0Tv4PWz7M0ku3btivVn6nG0mSE2tMuo9kzRLIRmOLRpQbMGABIHgQFSPG23Vpoij43ff//dXKy07iCygIAAc4HW7ZEVKFDA5RjanHD58mVJLG3btjXpf23iyJ07t2nSWLRo0UODBPt56kU2Kk3PX7x4UcLDwx/6XfR7qLh8l0aNGpkgbOHChaY3gtYHRP0t7fT8tZmlePHi5uKeI0cOE1jt27dPrl69GuvPzJs3b5wKDbXLpAZLGji9++67kitXrli/F8DDERggVQQG2nZ84MCBOL0vavFfTLy9vaNdb7PZ4v0Z9vZvOz8/P9m4caOpGejQoYO5cGqwoHf+UfdNiIR8Fzu9wOud+EcffSRLly6NMVugxo0bZzIzWi/w6aefyurVq02RZZkyZWKdGbH/PnGxe/duU3ehtKYBQOIhMECqoMVtOriRjiXwX7QHgV6UtJI+snPnzplqe3sPg8Sgd+SRK/jtomYllGYx6tSpY4r0fv31VzNQkqbqf/zxxxi/hzpy5IjLtsOHD5u7c+2pkBQ0GNCLr2ZpoivYtFu8eLEpFNTeIrqfpvnr1q3r8pvENkiLDc2SaLODNgFpMaP2WNGeEwASB4EBUoXBgwebi6Cm4vUCH5UGDVqxbk+Fq6g9B/SCrLQ/fmLR7pCaMtcMQOTaAL3TjtqtLyr7QD9Ru1DaabdM3Ufv3CNfaDVzolX49u+ZFPRir909p0+fbppgHpahiJqN+PLLL+Wvv/5yWmcPYKILouJqyJAhcurUKfO76H9T7S6qvRRi+h0BxA0DHCFV0AuwdpvT9Lu2r0ce+VC77+nFSIv0VPny5c2FQkdB1AuRdp3bvn27uZC0aNEixq5w8aF3yXqhatmypbzyyitmzIBZs2ZJiRIlnIrvtFBOmxI0KNFMgKbBZ86cKfny5TNjG8Rk4sSJphtftWrVpGvXrmZkRO2Wp2MUaPfFpKLZjWHDhsUqk6PfTe/gtSuppvW1LkG7lkb976f1HbNnzzb1CxooVK1aVQoXLhyn89IMi/5uI0eOdHSfnDdvnhnrYPjw4SZ7ACCBYtl7AUgRfvvtN1v37t1thQoVsqVLl86WKVMm21NPPWWbNm2a6Tpnd+fOHdPFrnDhwjYfHx9b/vz5bSEhIU77KO1q2Lhx4//sJhdTd0X1/fff2x599FFzPiVLlrR9+umnLt0V165da7pb5smTx+ynjy+88IL5PlE/I2qXvh9++MF8Rz8/P5u/v7+tadOmtl9//dVpH/vnRe0OqcfS9Xrs2HZXjElM3RW1W2dgYKA5Pz3PLVu2RNvN8Ouvv7YFBQXZ0qZN6/Q9db8yZcpE+5mRj3Pt2jXz36tixYrmv29k/fr1M1049bMBJEwa/T8JDS4AAIBnoMYAAAA4EBgAAAAHAgMAAOBAYAAAABwIDAAAgAOBAQAAcCAwAAAAnj3yoV+119x9CkCSO/rdaHefApDk8mX1TdLj+1UIjvd7b+6eLp7IIwMDAABiJQ2J86gIDAAA1pWIM396CkIlAIC1MwbxXeJAJ1crV66c+Pv7m0UnRlu5cqVje0REhPTu3VuyZ88uGTNmlNatW7vMJKuziupEbOnTp5dcuXLJoEGD5O7du077rF+/3kww5uvrK8WKFZP58+dLXBEYAACQxPLlyycTJkyQnTt3yi+//CLPPPOMNG/eXA4ePGi29+vXT7799lszU+yGDRvk9OnT0qpVK8f77927Z4IC+4yyOlusXvRHjBjh2CcsLMzsozPI7tmzR/r27Wumql+9enWcztUjJ1Gi+BBWQPEhrCDJiw+r9I/3e2/ueCdBn50tWzYztXqbNm0kZ86cZmp5fa4OHz5sppjfsmWLPPHEEya7oNOca8CQO3dus49OY67Tvl+4cEHSpUtnnq9YsUIOHDjgNDW8Tj+/atWqWJ8XGQMAgHUloCnh1q1bcu3aNadF1/0Xvfv/4osvJDw83DQpaBbhzp07UrduXcc+pUqVkgIFCpjAQOlj2bJlHUGBql+/vvlMe9ZB94l8DPs+9mPEFoEBAMDaxYfxXMaPHy+ZM2d2WnRdTPbv32/qB7T9v2fPnrJ06VIJCgqSs2fPmjv+LFmyOO2vQYBuU/oYOSiwb7dve9g+GjzcvHkz1j8JvRIAANaVgO6KISEh0r+/c1OEXvRjUrJkSdP2f/XqVVm8eLF06tTJ1BOkNAQGAADrSkB3RV9f34cGAlFpVkB7CqhKlSrJjh07ZOrUqdK2bVtTVKi1AJGzBtorISAgwDzXx+3btzsdz95rIfI+UXsy6GvtBeHn5xfr86QpAQAAN7h//76pSdAgwcfHR9auXevYduTIEdM9UWsQlD5qU8T58+cd+6xZs8Zc9LU5wr5P5GPY97EfI7bIGAAArCuZRj4MCQmRhg0bmoLCf/75x/RA0DEHtCuh1iZ07drVNEtoTwW92Pfp08dc0LVHgqpXr54JADp06CChoaGmnmDYsGFm7AN71kLrFqZPny6DBw+WLl26yLp162TRokWmp0JcEBgAAKwrmUY+PH/+vHTs2FHOnDljAgEd7EiDgmeffdZsnzx5snh5eZmBjTSLoL0JZs6c6Xi/t7e3LF++XHr16mUChgwZMpgahTFjxjj2KVy4sAkCdEwEbaLQsRPmzp1rjhUXjGMApFKMYwArSPJxDKoPj/d7b/78hngiMgYAAOtirgQXBAYAAOtidkUX/CIAAMCBjAEAwLrIGLggMAAAWJcXNQZRERgAAKyLjIELAgMAgHXRK8EFgQEAwLrIGLjgFwEAAA5kDAAA1kVTggsCAwCAddGU4ILAAABgXWQMXBAYAACsi4yBCwIDAIB1kTFwQagEAAAcyBgAAKyLpgQXBAYAAOuiKcEFgQEAwLrIGLggMAAAWBeBgQsCAwCAddGU4IJQCQAAOJAxAABYF00JLggMAADWRVOCCwIDAIB1kTFwQWAAALAuMgYuCAwAAJaVhsDABTkUAADgQMYAAGBZZAxcERgAAKyLuMAFgQEAwLLIGLgiMAAAWBaBgSsCAwCAZREYuKJXAgAAcCBjAACwLDIGrggMAADWRVzggsAAAGBZZAxcERgAACyLwMAVgQEAwLIIDFzRKwEAADiQMQAAWBYZgxQWGNy+fVuWLVsmW7ZskbNnz5p1AQEB8uSTT0rz5s0lXbp07jw9AICnIy5IOU0Jx44dk9KlS0unTp1k9+7dcv/+fbPo844dO0qZMmXMPgAAJGXGIL6Lp3JbxqBXr15StmxZEwj4+/s7bbt27ZoJDnr37i2rV6921ykCADycJ1/gU11gsGnTJtm+fbtLUKB03RtvvCFVq1Z1y7kBAKyBwCAFNSVkyZJFTp48GeN23ab7AACQ2o0fP16qVKkimTJlkly5ckmLFi3kyJEjTvvUqlXLpbmiZ8+eTvucOnVKGjduLOnTpzfHGTRokNy9e9dpn/Xr10vFihXF19dXihUrJvPnz08dgUG3bt1Mc8HkyZNl3759cu7cObPoc13XuXNneemll9x1egAAK0iTgCUONmzYYJrHt27dKmvWrJE7d+5IvXr1JDw83Gm/7t27y5kzZxxLaGioY9u9e/dMUKCF+5s3b5aPPvrIXPRHjBjh2CcsLMzsU7t2bdmzZ4/07dvXXG/j0iyfxmaz2cRN3nrrLZk6darpkWBP5+jpaM8E/TKDBw+O13H9qr2WyGcKpDxHvxvt7lMAkly+rL5Jevzc3b6M93vPzX0u3u+9cOGCuePXgKFGjRqOjMFjjz0mU6ZMifY9K1eulCZNmsjp06cld+7cZt3s2bNlyJAh5njak0+fr1ixQg4cOOB4X7t27eTKlSuyatWqlD/AkX4B/YLHjx+Xn3/+2Sz6XNfFNygAACA5eiXcunXLFMtHXnRdbFy9etU8ZsuWzWn9Z599Jjly5JBHH31UQkJC5MaNG45t2rVfi/btQYGqX7+++dyDBw869qlbt67TMXUfXZ+qBjgqXLiwWQAASC3Fh+PHj5fRo50zdyNHjpRRo0Y99H3aNV+z4k899ZQJAOxefPFFKViwoOTJk8c0q+vNs9YhfPXVV2a7ZtcjBwXK/to+FlBM+2jwcPPmTfHz80sdgQEAAKktMAgJCZH+/fs7rdOCv/+itQaa6tcseWSR6+o0MxAYGCh16tQxmfSiRYtKcmGuBAAA4sHX19d0r4+8/FdgEBwcLMuXL5cff/xR8uXL99B97V327YP9af2dFulHZn+t2x62j55bbLIFisAAAGBdydQrwWazmaBg6dKlsm7dulg1n2uvAqWZA1WtWjXZv3+/nD9/3rGP9nDQi35QUJBjn7Vr1zodR/fR9bFFYAAAsKzkGhK5d+/e8umnn8qCBQvMWAZaC6CLtvsrbS7Qgf127txpxvH55ptvTJd+7bFQrlw5s492b9QAoEOHDrJ3717TBXHYsGHm2PZMhY57cOLECVPAf/jwYZk5c6YsWrRI+vXrl3oCA+0+EbmdZcaMGaa7hhZhXL582a3nBgDwbMkVGMyaNcv0RNAuiZoBsC8LFy4027Wr4Q8//GAu/qVKlZIBAwZI69at5dtvv3Ucw9vb2zRD6KNmAP73v/+Z4GHMmDGOfTQTod0VNUtQvnx5efvtt2Xu3LmmZ0KqGMfAXmCh4xk0atTIpEh0ZCgt5tD2F/1x5s2bF+djMo4BrIBxDGAFST2OQf7eX8f7vX/MaC6eyO29EnSUJnvbyJIlS8zgDePGjZNdu3aZYAEAACQftzclaPrEPoCDPY1iH/RB+10CAJDaiw9TE7dnDKpXr26aDnSgB51t0d7e8ttvv/1nVw4knoEda0mLmmWkRMFccvPWHdm2/3cZOnOlHD110bFP4bzZZEKfxlKtXEHxTZdW1mz9Tfq//Y2cv3zdbH+6QhH5fmb081tU7zJddh76U4Z2rSvDujmPyqXCb96WHM/8O943kBw+mjNTPv5gttO6/AULyfyF3zit0xbXkH4vy46tm2T0W1Okes1nHNvOnT0jU0PHyp6dO8QvvZ/Ua9RMuvV6VbzTuv2fV8QCsyu6cvtf7vTp0+Xll1+WxYsXm+KMvHnzOsaEbtCggbtPzzKerlBYZi/ZKjsP/SFpvb1ldM/6snxKV6nw4jtyI+KOpH/Ex7zef+yMNOwzx7xnZPd6smRSJ6nRbab5h3Pr/t+lUOOxTscd8VI9qV25qAkK1JQFG2Xu0q1O+3w3rbtjO5DcChUpKhOnPfibVlrYFdWSLz6N9gKik9oMHdBbsmbLIe/O+VguXbwgb40ZZoICDQ6Q8hEYpMDAoECBAqbKMiqdYRHJp3k/5yLPl8Z+KX+sHC4VSuWTTXvCpFq5QlIwMKs80eld+efGg7HAu72xSM58P1JqVS4qP+44Jnfu3pNzfz/IHqi03l7S5OkgmbV4s1NmQBe7ssUCJahIbnkldGmyfE8gKm/vtJIte44Ytx/77bB8ueAjmTX/C3mu8b+ZAvXLts3ye9gJCX13jmTLnl2KlSgl//dSb5kzY4p06vay+Pj4JMM3QEIQGKTAGgMtMtTeCHZff/21maf69ddfN1NLwj38Mz5iHi9fe1D/oU0HmhW4deffeb8jbt+V+/dt8mS5QtEeQ4OC7JnTyyfLf4nxc/6vWRX57fcLsmnvyUT/DkBs/PXH7/J8kzryv1YNZdyI10zTgF1ExE15c8Rr8sqgodEGD78e2CeFixY3QYFd5SeelPDw63LyxIPR6pCyJVd3xdTE7YFBjx49TD2B0kEZdHrI9OnTy5dffskMi26if/AT+zaRzXtPyq8nHgytuf3AKQmPuCNv9m4ofr4+pmlB6w3SpvWWgByZoj1Op6aVZc223+SvC9EXkWqw0bb+Y/LRtzuS9PsAMSlVpqwMHj5Wxk+eJa8OHiZnzvwlfXt2lhvh4Wb7zCkTpUzZ8vJUjdrRvv/ypYuSNdu/QYGyv/770r/1OUBq4vamBA0KdEAjpcGAjvKkI0Nt2rTJBAkxzUttp1NcRp3m0nb/rqTxcvtXS7WmDGwuZYoESJ0esxzrLl4Jl/ZDP5N3B7WQl5970mQKFq3ZK7sO/2meR5U3p788W7WE/G/Yghg/p3nNMpIpva98+t2uJPsuwMNUffJpx/OixUtI6TJl5cUWDWT92tWSJUtW2fPLdnnv40VuPUckMc+98Y83t189NT2tU1DauyvqOAYqf/78cvHixXhNe+md9ynxyV89ic7Ys00e0EwaPVVK6vZ6z+VOf+32o1LmuYmmeeDuvfty9XqEhC0fKidP73M5TocmleXS1Ruy/KdfY/yszs2qyMpNhx29GgB3y5jJX/IVKCin//xDwo4fldN//SHNnn3KaZ/RIf2lbPmK8s6sDyVr9hxy+NcDTtsv/33JPD6sbgEphyc3CaTawKBy5coyduxYqVu3rmzYsMH0TLAPfBR1TunYTnuZ69l/h4dE3IKCZjXLSL2X35ffz8Q8HLVe8FXNSkUlV9YM0V78OzauJAtW7TIBRHS0kLFmxSLSZvDHifgNgIS5eeOGCQbqNmgiterWl0bNWjlt79a+tfR6dZBUe7qmeR30aDlZMH+OCQbsTQg7t2+VDBkySsHCyTdNLuKPwCAFBgbaVNC+fXtZtmyZDB06VIoVK2bWa/fFJ5988j/frxNHRJ3mkmaE+DUftK33mDw35GO5fuOW5M6W0ay/Gh4hEbceFBx2aFxJjpw8LxeuhEvVRwvIpH5NZdoXm5zGOlDaS6Fw3uwy75uYawe0/uDspX9k9ZYjSfzNgJjNfneSVKteS3IHBJquhvPnzBQvL295pl5DyZI1W7R3/bkCAiUwz4MxVipXfVIKFi4iE0YPlZeC+5m6gnnvTZNmbdqawduQ8hEXuHL7FVRnjYrcK8Fu4sSJ0fYnRtLo0frBlJxrZvZwWt/9jS/l0+92muclCuSUMb0aSDZ/P5NRCJ3/o7z7xb8TYNl1blpFtuw7aXobxBShd2hUST5ZsTPa+gQguVw4f17eHDFErl29IpmzZJVHy1eU6XM/NUFBbOi/UW9Omi5TQsdKn24d5BE/HeCoqfxf995Jfu5IHGQMUuAkSkmBSZRgBUyiBCtI6kmUig9aFe/3Hp3omYPwuT1joCOH6WBGOl/0qVOnXMYu+Pvvv912bgAAz0bCIAWOY6A9Ct555x1p27atmataCwlbtWolXl5eMmrUKHefHgDAgzHAUQoMDD777DOZM2eODBgwQNKmTSsvvPCCzJ07V0aMGCFbtzqPqQ8AQGLS63t8F0/l9sDg7NmzUrZsWfM8Y8aMJmugdDyDFStWuPnsAACezMsrTbwXT+X2wECnVj5z5sHY5EWLFpXvv//ePN+xY4dLN0QAABITGYMUGBi0bNlS1q5da5736dNHhg8fLsWLF5eOHTtKly5d3H16AABYitt7JUyYMMHxXAsQdRrmLVu2mOCgadOmbj03AIBn8+QiwlQbGERVrVo1swAAkNSIC1JIYPDNN9/Eet9mzZol6bkAAKyLjEEKCQxatGgR6/9gOgASAABJgcAghQQG9mmWAQBwJ+KCFNgrAQAApBxuCwzWrVsnQUFBcu3aNZdtOshRmTJlZOPGjW45NwCANTAkcgoKDKZMmSLdu3cXf39/l22ZM2eWHj16mMmVAABIKgxwlIICg71790qDBjFPWVmvXj3ZuXNnsp4TAMBayBikoHEMzp07Jz4+PjFu1wmVLly4kKznBACwFg++vqe+jEHevHnlwIEDMW7ft2+fBAYGJus5AQCshYxBCgoMGjVqZOZFiIiIcNl28+ZNGTlypJlhEQAAWKApYdiwYfLVV19JiRIlJDg4WEqWLGnWHz58WGbMmGEGNho6dKi7Tg8AYAEefOOf+gKD3Llzy+bNm6VXr14SEhIiNpvNrNf0TP369U1woPsAAJBUPLlJIFVOolSwYEH57rvv5PLly3Ls2DETHOisilmzZnXnaQEALIK4IIXOrqiBQJUqVdx9GgAAiyFjkEIDAwAA3IG4wBVzJQAAAAcyBgAAy6IpwRWBAQDAsogLXBEYAAAsi4yBKwIDAIBlERi4IjAAAFgWcYEreiUAAAAHAgMAgGUl1+yK48ePNwP5ZcqUSXLlyiUtWrSQI0eOOO2jkwr27t1bsmfPLhkzZpTWrVvLuXPnnPY5deqUNG7cWNKnT2+OM2jQILl7967TPuvXr5eKFSuKr6+vFCtWTObPnx+ncyUwAABYll7f47vExYYNG8xFf+vWrbJmzRq5c+eO1KtXT8LDwx379OvXT7799lv58ssvzf6nT5+WVq1aObbr5IIaFNy+fdvMNfTRRx+Zi/6IESMc+4SFhZl9ateuLXv27JG+fftKt27dZPXq1bE+1zQ2++xFHsSv2mvuPgUgyR39brS7TwFIcvmy+ibp8Z95d0u837vulWrxfu+FCxfMHb8GADVq1JCrV69Kzpw5ZcGCBdKmTRvHbMOlS5eWLVu2yBNPPCErV66UJk2amIDBPsng7NmzZciQIeZ46dKlM89XrFghBw4ccHxWu3bt5MqVK7Jq1apYnRsZAwCAZSUkY3Dr1i25du2a06LrYkMDAZUtWzbzuHPnTpNFqFu3rmOfUqVKSYECBUxgoPSxbNmyTjMP62zE+rkHDx507BP5GPZ97MeIDQIDAIBleaVJE+9l/PjxkjlzZqdF1/2X+/fvmxT/U089JY8++qhZd/bsWXPHnyVLFqd9NQjQbfZ9IgcF9u32bQ/bR4OHmzdvxuo3obsiAADxEBISIv3793dapwV//0VrDTTV//PPP0tKRGAAALCshIxj4OvrG6tAILLg4GBZvny5bNy4UfLly+dYHxAQYIoKtRYgctZAeyXoNvs+27dvdzqevddC5H2i9mTQ1/7+/uLn5xerc6QpAQBgWcnVXdFms5mgYOnSpbJu3TopXLiw0/ZKlSqJj4+PrF271rFOuzNq98Rq1R4UOerj/v375fz58459tIeDXvSDgoIc+0Q+hn0f+zFig4wBAMCyvJJp5MPevXubHgdff/21GcvAXhOgdQl6J6+PXbt2NU0TWpCoF/s+ffqYC7r2SFDavVEDgA4dOkhoaKg5xrBhw8yx7ZmLnj17yvTp02Xw4MHSpUsXE4QsWrTI9FSILQIDAIBlJddcCbNmzTKPtWrVclo/b9486dy5s3k+efJk8fLyMgMbae8G7U0wc+ZMx77e3t6mGaJXr14mYMiQIYN06tRJxowZ49hHMxEaBOiYCFOnTjXNFXPnzjXHii3GMQBSKcYxgBUk9TgGjd9zbrOPixU9HhdPRI0BAABwoCkBAGBZaYTpFaMiMAAAWFZyFR+mJgQGAADLSq7iw9SEwAAAYFnEBa4IDAAAlqVzHsAZvRIAAIADGQMAgGWRMHBFYAAAsCyKD10RGAAALIu4wBWBAQDAsig+dEVgAACwLMKCeAYG33zzjcRWs2bNYr0vAABIhYFBixYtYl3Ece/evYSeEwAAyYLiw3gGBvfv34/NbgAApCrMleCKGgMAgGWRMUikwCA8PFw2bNggp06dktu3bztte+WVV+JzSAAAkh1xQSIEBrt375ZGjRrJjRs3TICQLVs2uXjxoqRPn15y5cpFYAAASDXIGCTCXAn9+vWTpk2byuXLl8XPz0+2bt0qv//+u1SqVEkmTZoU18MBAIDUHBjs2bNHBgwYIF5eXuLt7S23bt2S/PnzS2hoqLz++utJc5YAACRR8WF8F08V58DAx8fHBAVKmw60zkBlzpxZ/vjjj8Q/QwAAkrApIb6Lp4pzjUGFChVkx44dUrx4calZs6aMGDHC1Bh88skn8uijjybNWQIAkAQ89/KejBmDcePGSWBgoHn+5ptvStasWaVXr15y4cIFef/99xNwKgAAJP9cCfFdPFWcMwaVK1d2PNemhFWrViX2OQEAADdhgCMAgGV58I1/8gUGhQsXfmjRxYkTJ+J/NgAAJCNPLiJMtsCgb9++Tq/v3LljBj3SJoVBgwbF+0QAAEhuxAWJEBi8+uqr0a6fMWOG/PLLL3E9HAAAbuPJRYTJ1ishJg0bNpQlS5Yk1uEAAEhyGhfEd/FUiRYYLF682MybAAAALDbAUeRiDZvNJmfPnjXjGMycOTOxzw8AgCRD8WEiBAbNmzd3+iF1eOScOXNKrVq1pFSpUpISXP5pgrtPAUhyWasEu/sUgCR3c/f01JE2t3JgMGrUqKQ5EwAAkhkZg0QIlnRGxfPnz7usv3TpktkGAEBqweyKiZAx0JqC6Oj0y+nSpYvr4QAAcBtPvsAneWDw7rvvOtIuc+fOlYwZMzq23bt3TzZu3JhiagwAAEASBwaTJ092ZAxmz57t1GygmYJChQqZ9QAApBbUGCQgMAgLCzOPtWvXlq+++spMtwwAQGpGU0Ii1Bj8+OOPcX0LAAApEgmDROiV0Lp1a3nrrbdc1oeGhspzzz0X18MBAODWuRLiu3iqOAcGWmTYqFGjaOdK0G0AAKSmi2B8F08V5+92/fr1aLsl+vj4yLVr1xLrvAAAQGoIDMqWLSsLFy50Wf/FF19IUFBQYp0XAABJjtkVE6H4cPjw4dKqVSs5fvy4PPPMM2bd2rVrZcGCBWaGRQAAUgtPrhVItoxB06ZNZdmyZXLs2DF5+eWXZcCAAfLXX3/JunXrpFixYvE+EQAAPDVjsHHjRnP9zJMnjxk7Qa+jkXXu3Nmsj7w0aNDAaZ+///5b2rdvL/7+/pIlSxbp2rWrad6PbN++ffL000/LI488Ivnz5zcdA+IqXvUTjRs3lk2bNkl4eLicOHFCnn/+eRk4cKCUL18+PocDAMCj50oIDw8318gZM2bEuI8GAmfOnHEsn3/+udN2DQoOHjwoa9askeXLl5tg46WXXnJs1zq/evXqScGCBWXnzp0yceJEM/Hh+++/n7RNCXZ6Qh988IEsWbLEREDavPCwLwwAgFWbEho2bGiWh/H19ZWAgIBotx06dEhWrVolO3bskMqVK5t106ZNM70EJ02aZK7Dn332mdy+fVs+/PBD00mgTJkysmfPHnnnnXecAohEzRicPXtWJkyYIMWLFzdjFmg6QydP0pSIrq9SpUpcDgcAQKp169Ytc5ceedF18bV+/XrJlSuXlCxZUnr16mVmLbbbsmWLaT6wBwWqbt264uXlJdu2bXPsU6NGDaeeg/Xr15cjR47I5cuXEz8w0LYRPVltv5gyZYqcPn3aRCsAAFixxmD8+PGSOXNmp0XXxYc2I3z88cemmF8HEdywYYPJMOgkhfYbcw0aIkubNq1ky5bNbLPvkzt3bqd97K/t+yRqU8LKlSvllVdeMVGMZgwAALDyXAmDQ0Kkf//+Ls0B8dGuXTunYQHKlSsnRYsWNVmEOnXqSHKKdcbg559/ln/++UcqVaokVatWlenTp8vFixeT9uwAAEhCaRLwP19fX9OkHnmJb2AQVZEiRSRHjhymB6DS2oPz58877XP37l3TU8Fel6CP586dc9rH/jqm2oUEBQZPPPGEzJkzx1RK9ujRwwxopMUO9+/fNxWSGjQAAJCaJFevhLj6888/TY1BYGCgeV2tWjW5cuWK6W1gp8ME6DVYb9bt+2jHgDt37jj20euzlgHEZUbkOHdXzJAhg3Tp0sVkEPbv32/GMdDCQ237aNasWVwPBwCAxwcG169fNz0EdFFhYWHm+alTp8y2QYMGydatW+XkyZOmzqB58+ZmbCAtHlSlS5c2dQjdu3eX7du3myEDgoODTROE3qSrF1980RQe6vgG2q1RRymeOnWqS3PHf/4mkgAahejgCRrZRO1vCQAAHvjll1+kQoUKZlF6sdbnI0aMEG9vb1PYrzfXJUqUMBd2bbb/6aefnJomtDtiqVKlTM2BdlOsXr260xgFWvz4/fffm6BD36837nr8uHRVVGlsNptNPEzEXXefAZD0slYJdvcpAEnu5u7pSXr8ietPxPu9g2oVEU8U7wGOAABI7ZK6ViA1IjAAAFgWcyi5IjAAAFgWsyu6IjAAAFgWTQmJ3CsBAAB4FjIGAADLoiXBFYEBAMCyvITIICoCAwCAZZExcEVgAACwLIoPXREYAAAsi+6KruiVAAAAHMgYAAAsi4SBKwIDAIBl0ZTgisAAAGBZxAWuCAwAAJZFoZ0rAgMAgGWlIWXggmAJAAA4kDEAAFgW+QJXBAYAAMuiV4IrAgMAgGURFrgiMAAAWBYJA1cEBgAAy6JXgit6JQAAAAcyBgAAy+Lu2BWBAQDAsmhKcEVgAACwLMICVwQGAADLImPgisAAAGBZ1Bi44jcBAAAOZAwAAJZFU0IqyhicO3dOxowZ4+7TAAB4sDQJWDxVig0Mzp49K6NHj3b3aQAAPJgmDOK7eCq3NSXs27fvoduPHDmSbOcCALAmL4++909lgcFjjz1m2nZsNpvLNvt62n4AAEmJy0wKCgyyZcsmoaGhUqdOnWi3Hzx4UJo2bZrs5wUAgJW5LTCoVKmSnD59WgoWLBjt9itXrkSbTQAAILGkoSkh5QQGPXv2lPDw8Bi3FyhQQObNm5es5wQAsBaaElJQYNCyZcuHbs+aNat06tQp2c4HAGA9FB+6YoAjAIBlkTFwRWAAALAsAoNUNMARAABIfmQMAACWRa8EVwQGAADL8iIuSHlNCatWrZKff/7Z8XrGjBlmVMQXX3xRLl++7NZzAwB4fsYgvv/zVG4PDAYNGiTXrl0zz/fv3y8DBgyQRo0aSVhYmPTv39/dpwcA8GDJNYnSxo0bzWi+efLkMcP9L1u2zGm7Dug3YsQICQwMFD8/P6lbt64cPXrUaZ+///5b2rdvL/7+/pIlSxbp2rWrXL9+3WUeoqeffloeeeQRyZ8/vxlhONUFBhoABAUFmedLliyRJk2ayLhx40zmYOXKle4+PQAAEkwH9Ctfvry5tkVHL+DvvvuuzJ49W7Zt2yYZMmSQ+vXrS0REhGMfDQp0uoA1a9bI8uXLTbDx0ksvObbrTXa9evXMiMI7d+6UiRMnyqhRo+T9999PXTUG6dKlkxs3bpjnP/zwg3Ts2NExl4I9kwAAQFJIriaBhg0bmiU6mi2YMmWKDBs2TJo3b27Wffzxx5I7d26TWWjXrp0cOnTINL3v2LFDKleubPaZNm2aybBPmjTJZCI+++wzuX37tnz44Yfm2lqmTBnZs2ePvPPOO04BRIrPGFSvXt00Gbzxxhuyfft2ady4sVn/22+/Sb58+dx9epb1wZz35MXnW0u1KhWk1tPVpG+fl+Vk2IkY/6hf7tFNypcpKevW/uC07cD+fdK9Syep/kRlqV6tivTs3lWOHD6cTN8CcNb9ueqyfWGInPtpolnWfzRA6j31IGOpfNOllcmvPS9//viWXNj0tnw+qZvkypbJ5Tj/a1rVHOfy1sny+9rx5j1R9e1QR/YtGyFXtk2W46vHyuCu9ZP8+yF+xYfxXW7dumVuYCMvui4+mfOzZ8+a5gO7zJkzS9WqVWXLli3mtT5q84E9KFC6v5eXl8kw2PepUaOGCQrsNOtw5MiRONXsuT0wmD59uqRNm1YWL14ss2bNkrx585r12ozQoEEDd5+eZf2yY7u0faG9fPL5Inlvzjy5e/euuajbszuRffrxR9FOkX0jPFxe7tFdAgLzyKefL5L5nyww6bFeL3WVO3fuJNM3Af7117krMnza1/Jk+1B5qv1EWb/9N/ly8ktSukiA2R46sLU0rvGotB/8gdTrNkUCc2aWL97u5nSMV/73jIwObipvz1sjFdu8KY17TpMfthxy2uftwW2kc8tqEjJ5qZRvOVba9H1Pfjnwe7J+VyR98eH48ePNBTzyouviSoMCpRmCyPS1fZs+5sqVy2m7Xjs1ux55n+iOEfkzUkVTgk6WpG0lUU2ePNkt54MHZr3/gdPrMW9OkNpPV5NDvx6USpWrONYfPnRIPv7oQ/l84RKpU6u603vCwk7I1atXpHfwKxIQGGjW9Xy5t7Rp2UzOnD4tBWKYWRNIKt9tPOD0etSMb00W4fFyheWv81ekc4tq0vn1+bJhx29m+0sjP5W9S4fL42ULyfb9JyVLJj8Z+XITad13tgkq7A4cPe14XrJwbune5mmp9NybcvT382bd76cvJdt3RPKNfBgSEuJSJO/r6yupndszBrt27TK9Eey+/vpradGihbz++uumrQQpw/V//jGP/pkzO9bdvHlTQgYPkNeHjZAcOXO6vKdQ4cIm9bX0q8Vy5/ZtU0SzdMliKVKkqOT5/5khwF28vNLIc/UrSQa/dLJtX5hUKF1A0vmklXVbjzj2+e3kOTl15m+pWq6weV3niVLmfXlyZZHdS4bJsVVvyKdvdZF8ubM43tO4RlkJ++uiNKrxqBxaPkoOrxgtM0e8KFn907vle+Lh0iRg8fX1NT0EIi/xCQwCAh5krM6dO+e0Xl/bt+nj+fMPAk07zeRqT4XI+0R3jMifkSoCgx49eph6AnXixAlTZJE+fXr58ssvZfDgwe4+PYjI/fv3JfStcfJYhYpSvHgJx/qJb42X8hUqSO1n/m0XiyxDhowyd/4nsuLbb+TxSuVNvcKmTT/JjPfmmBQY4A5liuUx9QNXt02Rd4e2lbYD5sjhE2clILu/3Lp9R65ev+m0//lL1yR3dn/zvHC+HCYwGNylngyatEReHPSBZM2cXpbPChaftN5mn0L5ckiBwGzSqm4F6Tb8E+k+4lOpUDq/LJjY1S3fFylf4cKFzYV77dq1jnVar6C1A9WqVTOv9fHKlSumt4HdunXrzL/PWotg30d7KkRuqtUeDCVLljQzFqeawECDAh3QSGkwoIUTCxYskPnz55vui/8lsYo/ELNxY0fL8aNHJXTSv80769etlR3btsrgIa/H+D7NEIwaPtQEFJ8sWCgfffq5FCtWQoJ79XDqggMkJ80CVG03Xmp0nCRzvvxZ5ozpIKX+f43Bf9FaGs0qDAhdbOoKtHmhU8h8KVYgl9Ss8iBo9kqTRh7x9ZGuwz+RTbuPy087j0qv0Z9JrcdLSvGCzm3EcD/97xXfJS50vAHtIaCLveBQn586dcr8XfXt21fGjh0r33zzjcmiaw897WmgGXRVunRpU3fXvXt3U6i/adMmCQ4ONjfTup/SgQG18FDHN9BujQsXLpSpU6fGeUwgtwcGWtGuEY+9u6J2vVA6MMPFixf/8/3RFX/onSwSx7ixY2TjhvUyZ95HkjtSKmr7tq3yxx+nTE+DiuWCzKIG9O0jXTt3MM+/W/GtnD79l4x5c7w8WraclCv/mEwInSR//fWn/Lju38gYSE537t6TE39clN2H/pAR076R/b/9Jb1fqCVnL10T33Q+kjmjn9P+ubL7y7lLD7pOn7344FEzDHYXL1+Xi1euS/6AB3dkZy9elTt37smxU/+mfQ+HPUjn5g/IlizfEcnTlBAXv/zyi1SoUMEsSi/W+lwHNVKaIe/Tp4/pVlilShUTSGj3RB2oyE67I5YqVUrq1KljrpXaqy/yGAV6/fv+++9N0FGpUiUzYKAePy5dFZXb87na9UKjJO12sWHDBtMzQekXi1pdGdviD5t36i/+SAkB2/g335B1a9fIB/M/kXz58jtt79LtJWnZ5jmndW1aNJWBQ0KkZq3a5rVmBbzSeDn1WEjj5WWqeW3/PxgE3E3v/LSb4u5Dp+T2nbtSu2pJWbb2wV2d3uFrs4DWIKgtex502S1eKJcpVlRaO5AjS0ZTi2Dfx8fH2zQ7hP354ObGnimw74MUJJlGNq5Vq5b5dzXG00iTRsaMGWOWmGgPBM2oP0y5cuXkp59+StC5uj0w0EEddDQnHcRh6NChUqxYMbNeuy8++eST//l+LfSIWuwRcTfJTtcyxr0xWlZ+t1ymTJspGdJnkIsXLpj1GTNlMhGsFhtGV3AYGJjHEURUq/akTJ4Uao71QvsOct92Xz6c+76kTestVf5/mxiQnMb0aSarNx2UP85clkwZHpG2DStLjcrFpenLM+Xa9QiZv2yLvDWglfx9NVz+CY+Qd4Y8J1v3njBNBkqzAN/+uFcmDWojwWM/N+/RYx45eU42/PKgVmrdtiOy69dT8t6o9jJo4hJTkzDltedN00PkLAJSBk+e8yC+0tgeFsK4kd5tent7i4+PT9zfS2CQYDpYUXTGjB0vzVu2ivE9k9+dIc/U+bcYccvmTTJ75nQ5fuyopEnjJaVKl5Y+r/YzzQpImKxVgt19CqnOrJEvSu3HS0pADn+5ej1CDhz9S96e94Os2/Zg0C3NHEzo30qeb1DJPP9h8yF5dfxCOXfpQa8cpQFF6MBW0vyZx+T+fZv8vPOoDJy4WP489yCDoHT8Aw0qtBdD+M3b8v2mX+W1d76Sy9dcxwHBw93cPT1Jj7/9xNV4v/fxIv/20vIkKTYwSAgCA1gBgQGsgMAg+bm9KeHevXtmMKNFixaZ6syoYxdoH00AAJICDQkpsFfC6NGjzQQPbdu2latXr5pCwlatWpnxn3VWKAAAUn23hFTE7YGBdr+YM2eO6Vahg9688MILMnfuXNPFYuvWre4+PQCAB0vIXAmeyu2BgU7sULZsWfM8Y8aMJmugmjRpIitWrHDz2QEAPJn2po7v4qncHhjo1Mpnzpwxz4sWLWoGZ1A657QnTEYBAEi5aElIgYFBy5YtHeND66hPw4cPl+LFi5vhILt06eLu0wMAwFLc3ithwoQJjudagKjTMG/ZssUEB02bNnXruQEAPJwn3/qn1sAgKp0dyj6bFAAAScmTiwhTVWCgs0fFVrNmzZL0XAAA1uXJRYSpKjCwTyP5X3RSCR0ACQCApEBckEICA/s0ywAAuBWRQcrrlQAAAFIOtwUG69atk6CgILl27ZrLNh3kqEyZMrJx40a3nBsAwBoY+TAFBQZTpkyR7t27i7+/v8u2zJkzS48ePczkSgAAJBVGPkxBgcHevXulQYMGMW6vV6+e7Ny5M1nPCQBgLYx8mILGMTh37pz4+PjEuF0nVLpw4UKynhMAwGI8+Qqf2jIGefPmlQMHDsS4fd++fRIYGJis5wQAsBZqDFJQYNCoUSMzL0JERITLtps3b8rIkSPNDIsAACD5pLHZbDZxU1NCxYoVxdvbW4KDg6VkyZJm/eHDh2XGjBlmYKNdu3ZJ7ty543zsiLtJcMJACpO1SrC7TwFIcjd3T0/S4/96Ojze7w3Kk0E8kdtqDPSCv3nzZunVq5eEhISIPT7R0Q7r169vgoP4BAUAAMSW5zYIpNJJlAoWLCjfffedXL58WY4dO2aCA51VMWvWrO48LQCAVRAZpMzZFTUQqFKlirtPAwBgMZ5cRJiqAwMAANzBkwcqii/mSgAAAA5kDAAAlkXCwBWBAQDAuogMXBAYAAAsi+JDVwQGAADLovjQFYEBAMCyiAtc0SsBAAA4kDEAAFgXKQMXBAYAAMui+NAVgQEAwLIoPnRFYAAAsCziAlcEBgAA6yIycEGvBAAA4EDGAABgWRQfuiIwAABYFsWHrggMAACWRVzgisAAAGBZZAxcUXwIALCwNAlYYm/UqFGSJk0ap6VUqVKO7REREdK7d2/Jnj27ZMyYUVq3bi3nzp1zOsapU6ekcePGkj59esmVK5cMGjRI7t69K4mNjAEAAMmgTJky8sMPPzhep0377yW4X79+smLFCvnyyy8lc+bMEhwcLK1atZJNmzaZ7ffu3TNBQUBAgGzevFnOnDkjHTt2FB8fHxk3blyinieBAQDAspKzKSFt2rTmwh7V1atX5YMPPpAFCxbIM888Y9bNmzdPSpcuLVu3bpUnnnhCvv/+e/n1119NYJE7d2557LHH5I033pAhQ4aYbES6dOkS7TxpSgAAWFZCGhJu3bol165dc1p0XUyOHj0qefLkkSJFikj79u1N04DauXOn3LlzR+rWrevYV5sZChQoIFu2bDGv9bFs2bImKLCrX7+++cyDBw8m6m9CYAAAsHTGIL7L+PHjTdo/8qLrolO1alWZP3++rFq1SmbNmiVhYWHy9NNPyz///CNnz541d/xZsmRxeo8GAbpN6WPkoMC+3b4tMdGUAACwrIQMcBQSEiL9+/d3Wufr6xvtvg0bNnQ8L1eunAkUChYsKIsWLRI/Pz9JScgYAACsKwFtCb6+vuLv7++0xBQYRKXZgRIlSsixY8dM3cHt27flypUrTvtorwR7TYI+Ru2lYH8dXd1CQhAYAACQzK5fvy7Hjx+XwMBAqVSpkuldsHbtWsf2I0eOmBqEatWqmdf6uH//fjl//rxjnzVr1phgJCgoKFHPjaYEAIBlJVenhIEDB0rTpk1N88Hp06dl5MiR4u3tLS+88IKpTejatatplsiWLZu52Pfp08cEA9ojQdWrV88EAB06dJDQ0FBTVzBs2DAz9kFssxSxRWAAALCs5Oqu+Oeff5og4NKlS5IzZ06pXr266Yqoz9XkyZPFy8vLDGykPRu0x8HMmTMd79cgYvny5dKrVy8TMGTIkEE6deokY8aMSfRzTWOz2WziYSISfyAoIMXJWiXY3acAJLmbu6cn6fEv/BP/C0bOTJ55b+2Z3woAgNhgrgQXBAYAAMsiLnBFrwQAAOBAxgAAYFlMu+yKwAAAYFkJGfnQUxEYAAAsi4yBK2oMAACAAxkDAIBlkTFwRcYAAAA4kDEAAFgWxYeuCAwAAJZFU4IrAgMAgGURF7giMAAAWBeRgQuKDwEAgAMZAwCAZVF86IrAAABgWRQfuiIwAABYFnGBKwIDAIB1ERm4IDAAAFgWNQau6JUAAAAcyBgAACyL4kNXaWw2my2a9UCs3bp1S8aPHy8hISHi6+vr7tMBkgR/57AKAgMk2LVr1yRz5sxy9epV8ff3d/fpAEmCv3NYBTUGAADAgcAAAAA4EBgAAAAHAgMkmBZijRw5koIseDT+zmEVFB8CAAAHMgYAAMCBwAAAADgQGAAAAAcCAzhJkyaNLFu2zN2nASQp/s6BmBEYWMjZs2elT58+UqRIEVNZnT9/fmnatKmsXbtWUgKtgx0xYoQEBgaKn5+f1K1bV44ePeru00Iqk9L/zr/66iupV6+eZM+e3QQoe/bscfcpAU4IDCzi5MmTUqlSJVm3bp1MnDhR9u/fL6tWrZLatWtL7969JSUIDQ2Vd999V2bPni3btm2TDBkySP369SUiIsLdp4ZUIjX8nYeHh0v16tXlrbfecvepANHT7orwfA0bNrTlzZvXdv36dZdtly9fdjzXP4mlS5c6Xg8ePNhWvHhxm5+fn61w4cK2YcOG2W7fvu3YvmfPHlutWrVsGTNmtGXKlMlWsWJF244dO8y2kydP2po0aWLLkiWLLX369LagoCDbihUroj2/+/fv2wICAmwTJ050rLty5YrN19fX9vnnnyfa7wDPltL/ziMLCwsz57F79+5E+OZA4mHaZQv4+++/zV3Tm2++ae7Co8qSJUuM782UKZPMnz9f8uTJY+6+unfvbtYNHjzYbG/fvr1UqFBBZs2aJd7e3iYt6uPjY7bpHdrt27dl48aN5nN//fVXyZgxY7SfExYWZlLA2nxgpxPWVK1aVbZs2SLt2rVLhF8Cniw1/J0DqQGBgQUcO3bMtN+XKlUqzu8dNmyY43mhQoVk4MCB8sUXXzj+wTx16pQMGjTIcezixYs79tdtrVu3lrJly5rX2uYbEw0KVO7cuZ3W62v7NiC1/50DqQE1BhaQkMEtFy5cKE899ZQEBASYuyD9B1T/IbTr37+/dOvWzdzpT5gwQY4fP+7Y9sorr8jYsWPN+3Uo2X379iX4uwAx4e8cSBwEBhagdzda/Xz48OE4vU9T+JpCbdSokSxfvlx2794tQ4cONWlTu1GjRsnBgwelcePGpuArKChIli5darbpP6QnTpyQDh06mPRs5cqVZdq0adF+lv6DrM6dO+e0Xl/btwGp/e8cSBUSsV4BKViDBg3iXJQ1adIkW5EiRZz27dq1qy1z5swxfk67du1sTZs2jXbba6+9ZitbtuxDiw/1M+2uXr1K8SE86u88MooPkVKRMbCIGTNmyL179+Txxx+XJUuWmPEBDh06ZLoHVqtWLcY7ME2nalurpk51X/tdkrp586YEBwfL+vXr5ffff5dNmzbJjh07pHTp0mZ73759ZfXq1aawcNeuXfLjjz86tkWld3q6v6Zkv/nmG3Pn1bFjR1MM1qJFiyT6VeBpUvrfub1IUosXtUhRHTlyxLymlgYphrsjEySf06dP23r37m0rWLCgLV26dObOqlmzZrYff/wxxm5cgwYNsmXPnt1002rbtq1t8uTJjjupW7dumTun/Pnzm+PlyZPHFhwcbLt586bZrs+LFi1q7vpz5sxp69Chg+3ixYsxnp9mDYYPH27LnTu3eU+dOnVsR44cSdLfBJ4npf+dz5s3z3x+1GXkyJFJ+rsAscW0ywAAwIGmBAAA4EBgAAAAHAgMAACAA4EBAABwIDAAAAAOBAYAAMCBwAAAADgQGAAAAAcCAyAV6Ny5s9PQ0LVq1TJD8SY3HRZYh6++cuVKsn82gORBYAAk8IKtF0pd0qVLJ8WKFZMxY8bI3bt3k/Rzv/rqK3njjTditS8XcwBxkTZOewNw0aBBA5k3b57cunVLvvvuO+ndu7f4+PhISEiI0346ja8GD4khW7ZsiXIcAIiKjAGQQL6+vhIQECAFCxaUXr16Sd26dc0Mkfb0/5tvvmlmiSxZsqTZ/48//pDnn39esmTJYi7wzZs3l5MnTzqOp7MD9u/f32zPnj27DB48WCc7c/rMqE0JGpQMGTJE8ufPb85HMxcffPCBOW7t2rXNPlmzZjWZAz0vdf/+fRk/frwULlxY/Pz8pHz58rJ48WKnz9FAp0SJEma7HifyeQLwTAQGQCLTi6hmB9TatWvNtLpr1qyR5cuXy507d6R+/fqSKVMm+emnn8wUvhkzZjRZB/t73n77bZk/f758+OGH8vPPP5tpeiNPAxwdnaL6888/N1MG6zTD7733njmuBgo6/bDS8zhz5oxMnTrVvNag4OOPP5bZs2fLwYMHpV+/fvK///1PNmzY4AhgWrVqJU2bNjXTAnfr1k1ee+21JP71ALhdrOdhBOCiU6dOtubNmzumjV6zZo2ZfnfgwIFmm04hrdP22n3yySe2kiVLmn3tdLufn59t9erV5nVgYKAtNDTUsf3OnTu2fPnyOT5H1axZ0/bqq6+a5zo1tf6/sn52dHS6Yd1++fJlx7qIiAhb+vTpbZs3b3bat2vXrrYXXnjBPA8JCbEFBQU5bR8yZIjLsQB4FmoMgATSTIDenWs2QNPzL774oowaNcrUGpQtW9aprmDv3r1y7NgxkzGILCIiQo4fPy5Xr141d/VVq1Z1bEubNq1UrlzZpTnBTu/mvb29pWbNmrE+Zz2HGzduyLPPPuu0XrMWFSpUMM818xD5PFS1atVi/RkAUicCAyCBtO191qxZJgDQWgK9kNtlyJDBad/r169LpUqV5LPPPnM5Ts6cOePddBFXeh5qxYoVkjdvXqdtWqMAwLoIDIAE0ou/FvvFRsWKFWXhwoWSK1cu8ff3j3afwMBA2bZtm9SoUcO81q6PO3fuNO+NjmYlNFOhtQFa+BiVPWOhRY12QUFBJgA4depUjJmG0qVLmyLKyLZu3Rqr7wkg9aL4EEhG7du3lxw5cpieCFp8GBYWZsYZeOWVV+TPP/80+7z66qsyYcIEWbZsmRw+fFhefvnlh45BUKhQIenUqZN06dLFvMd+zEWLFpnt2ltCeyNok8eFCxdMtkCbMgYOHGgKDj/66CPTjLFr1y6ZNm2aea169uwpR48elUGDBpnCxQULFpiiSACejcAASEbp06eXjRs3SoECBUzFv96Vd+3a1dQY2DMIAwYMkA4dOpiLvbbp60W8ZcuWDz2uNmW0adPGBBGlSpWS7t27S3h4uNmmTQWjR482PQpy584twcHBZr0OkDR8+HDTO0HPQ3tGaNOCdl9Ueo7ao0GDDe3KqL0Xxo0bl+S/EQD3SqMViG4+BwAAkEKQMQAAAA4EBgAAwIHAAAAAOBAYAAAABwIDAADgQGAAAAAcCAwAAIADgQEAAHAgMAAAAA4EBgAAwIHAAAAAiN3/A1cGhDQgN6DDAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from sklearn.metrics import confusion_matrix\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "\n", + "# Define class names\n", + "class_names = ['Class 0', 'Class 1'] # Update this list based on your actual class names\n", + "\n", + "# Confusion matrix\n", + "conf_matrix = confusion_matrix(y_test, y_pred)\n", + "\n", + "# Plot the confusion matrix\n", + "plt.figure(figsize=(6, 4))\n", + "sns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues', xticklabels=class_names, yticklabels=class_names)\n", + "plt.xlabel('Predicted')\n", + "plt.ylabel('Actual')\n", + "plt.title('Confusion Matrix')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***6*** . **try other Models**" + ] + }, + { + "cell_type": "code", + "execution_count": 77, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the model\n", + "rf = RandomForestClassifier(random_state=42)\n", + "\n", + "# Train the model\n", + "rf.fit(X_train_vectorized, y_train)\n", + "\n", + "# Predict on the test set\n", + "y_pred = rf.predict(X_test_vectorized)" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy: 0.8954765041721563\n" + ] + } + ], + "source": [ + "rf = RandomForestClassifier(\n", + " n_estimators=200,\n", + " max_depth=200,\n", + " min_samples_split=5,\n", + " min_samples_leaf=1,\n", + " random_state=42\n", + ")\n", + "rf.fit(X_train_vectorized, y_train)\n", + "y_pred = rf.predict(X_test_vectorized)\n", + "print(f\"Accuracy: {accuracy_score(y_test, y_pred)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "metadata": {}, + "outputs": [], + "source": [ + "#from sklearn.model_selection import GridSearchCV\n", + "\n", + "# param_grid = {\n", + "# 'n_estimators': [200, 500, 1000],\n", + "# 'max_depth': [50, 100, 200],\n", + "# 'min_samples_split': [2, 5, 10],\n", + "# 'min_samples_leaf': [1, 2, 5]\n", + "#}\n", + "\n", + "#grid_search = GridSearchCV(\n", + "# estimator=RandomForestClassifier(random_state=42),\n", + "# param_grid=param_grid,\n", + "# cv=5,\n", + "# scoring='accuracy',\n", + "# n_jobs=-1\n", + "#)\n", + "\n", + "#grid_search.fit(X_train_vectorized, y_train)\n", + "\n", + "#print(f\"Best Parameters: {grid_search.best_params_}\")\n", + "#print(f\"Best Accuracy: {grid_search.best_score_}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "XGBoost Accuracy: 0.8846435368174499\n" + ] + } + ], + "source": [ + "# Ensure xgboost is installed\n", + "\n", + "from xgboost import XGBClassifier\n", + "\n", + "xgb = XGBClassifier(random_state=42)\n", + "xgb.fit(X_train_vectorized, y_train)\n", + "y_pred = xgb.predict(X_test_vectorized)\n", + "\n", + "print(f\"XGBoost Accuracy: {accuracy_score(y_test, y_pred)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 82, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Gradient Boosting Accuracy: 0.8230127360562143\n", + "\n", + "Classification Report:\n", + " precision recall f1-score support\n", + "\n", + " 0 0.91 0.73 0.81 3517\n", + " 1 0.76 0.93 0.84 3314\n", + "\n", + " accuracy 0.82 6831\n", + " macro avg 0.84 0.83 0.82 6831\n", + "weighted avg 0.84 0.82 0.82 6831\n", + "\n" + ] + } + ], + "source": [ + "from sklearn.ensemble import GradientBoostingClassifier\n", + "from sklearn.metrics import classification_report, accuracy_score\n", + "\n", + "# Initialize and train the Gradient Boosting classifier\n", + "gb = GradientBoostingClassifier(random_state=42)\n", + "gb.fit(X_train_vectorized, y_train)\n", + "\n", + "# Make predictions\n", + "y_pred_gb = gb.predict(X_test_vectorized)\n", + "\n", + "# Evaluate the model\n", + "print(\"Gradient Boosting Accuracy:\", accuracy_score(y_test, y_pred_gb))\n", + "print(\"\\nClassification Report:\\n\", classification_report(y_test, y_pred_gb))" + ] + }, + { + "cell_type": "code", + "execution_count": 84, + "metadata": {}, + "outputs": [], + "source": [ + "#pip install streamlit" + ] + }, + { + "cell_type": "code", + "execution_count": 85, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['vectorizer.pkl']" + ] + }, + "execution_count": 85, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import joblib\n", + "\n", + "# Save the model and vectorizer\n", + "joblib.dump(classifier, 'model.pkl')\n", + "joblib.dump(vectorizer, 'vectorizer.pkl')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 86, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-01-21 16:49:33.907 WARNING streamlit.runtime.scriptrunner_utils.script_run_context: Thread 'MainThread': missing ScriptRunContext! This warning can be ignored when running in bare mode.\n", + "2025-01-21 16:49:34.022 \n", + " \u001b[33m\u001b[1mWarning:\u001b[0m to view this Streamlit app on a browser, run it with the following\n", + " command:\n", + "\n", + " streamlit run c:\\Users\\ggsan\\miniconda3\\envs\\IronHack\\Lib\\site-packages\\ipykernel_launcher.py [ARGUMENTS]\n", + "2025-01-21 16:49:34.023 Thread 'MainThread': missing ScriptRunContext! This warning can be ignored when running in bare mode.\n", + "2025-01-21 16:49:34.023 Thread 'MainThread': missing ScriptRunContext! This warning can be ignored when running in bare mode.\n", + "2025-01-21 16:49:34.024 Thread 'MainThread': missing ScriptRunContext! This warning can be ignored when running in bare mode.\n", + "2025-01-21 16:49:34.024 Thread 'MainThread': missing ScriptRunContext! This warning can be ignored when running in bare mode.\n", + "2025-01-21 16:49:34.025 Thread 'MainThread': missing ScriptRunContext! This warning can be ignored when running in bare mode.\n", + "2025-01-21 16:49:34.025 Session state does not function when running a script without `streamlit run`\n", + "2025-01-21 16:49:34.027 Thread 'MainThread': missing ScriptRunContext! This warning can be ignored when running in bare mode.\n", + "2025-01-21 16:49:34.027 Thread 'MainThread': missing ScriptRunContext! This warning can be ignored when running in bare mode.\n", + "2025-01-21 16:49:34.028 Thread 'MainThread': missing ScriptRunContext! This warning can be ignored when running in bare mode.\n", + "2025-01-21 16:49:34.028 Thread 'MainThread': missing ScriptRunContext! This warning can be ignored when running in bare mode.\n", + "2025-01-21 16:49:34.028 Thread 'MainThread': missing ScriptRunContext! This warning can be ignored when running in bare mode.\n", + "2025-01-21 16:49:34.029 Thread 'MainThread': missing ScriptRunContext! This warning can be ignored when running in bare mode.\n", + "2025-01-21 16:49:34.029 Thread 'MainThread': missing ScriptRunContext! This warning can be ignored when running in bare mode.\n" + ] + }, + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 86, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import streamlit as st\n", + "import joblib\n", + "\n", + "# Load the model and vectorizer\n", + "model = joblib.load('model.pkl')\n", + "vectorizer = joblib.load('vectorizer.pkl')\n", + "\n", + "# App title\n", + "st.title(\"Fake News Detection App\")\n", + "\n", + "# Input text box\n", + "user_input = st.text_area(\"Enter the news headline:\")\n", + "\n", + "# Predict button\n", + "if st.button(\"Predict\"):\n", + " if user_input.strip():\n", + " # Preprocess and predict\n", + " user_input_vectorized = vectorizer.transform([user_input])\n", + " prediction = model.predict(user_input_vectorized)\n", + " prediction_label = \"Real News\" if prediction[0] == 1 else \"Fake News\"\n", + " st.success(f\"Prediction: {prediction_label}\")\n", + " else:\n", + " st.error(\"Please enter a news headline.\")\n", + "0" + ] + } + ], + "metadata": { + "colab": { + "authorship_tag": "ABX9TyPYIwndgNfqsBlnv9Y5j0aO", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.1" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/model.pkl b/model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..1deb63f89e70800b9c4c8d6883681e36c13fb500 GIT binary patch literal 90851177 zcmZ_XbzBtBA29G03lS9?`)q6!0~P(8jiQJhUzL;4LlF>AvCl44RO~vt6$uGZmTsk` z4$j8LMn!!F{Oo<^_j;bkKU{8R-Z#6mQ#-e}r>b^vYgM80pSQ8cVrR$s?k+tYT|69@ z*f~4)v|Zrp?&#s^>)mAXd>04TC8H}}j&Ppu;o-Ev$~vlPDG)fPIrIQe?( zd$>6}d9pwjVpq8!{@vcrw+?KIDU&(gM-raG&hodj|mX(K(88cAJ-rmi2zO$Qh zK9{J@5?2>jPgfTwds}-}+}7SkwpfO?%N^Y-ip zZzJ7HTetb{^Otz|nkZ{4>nV+t^_2~j4V8_QO_Yt5#>!^wOxChQnTf$rdy4ZW**EM-dB$XeNif}2?@jdicQ>$>QQ+?Exl z?}Y6Qy$x)2r`2~t$Rck87hBh5o^H!LJ!~t^r9RdvTVL;nY@eN-?0UK^TjI9T*3HSD z)nQ-n`qmXMhq=4YUpa#vsvBQb^*xr^ReDyYmCDz2hUhr)L&J8#}~_$JRP*J$tR72OuASq=WT8>@2d5h3)Y&7f;^_gS)XBS=rJ2?|ZWd-J1{et~$Mc zU%u9@@wGPbHgI-a;Av~GbaHm^^{(w+*~{Upn%JSSw(qHA{hHFXB8|S(Y57((z{