user_analysis_GetCounts.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2c720bb8",
   "metadata": {},
   "source": [
    "This notebook gives the count comparison of submissions and/or comments for INDIVIDUAL users (in r/stopsmoking and r/stopdrinking) BEFORE and DURING COVID-19 pandemic. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "832f4c37",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.dates as md\n",
    "import seaborn as sns\n",
    "import datetime as dt\n",
    "import time\n",
    "import datetime\n",
    "from textblob import TextBlob\n",
    "import nltk\n",
    "from nltk import ngrams\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "# nltk.download('punkt')\n",
    "import operator\n",
    "import re\n",
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "c535e820",
   "metadata": {},
   "outputs": [],
   "source": [
    "def getStatsDict(all_files):\n",
    "    stats_dict = dict() # this will keep record of username as the key and total of number of comments, number comments\n",
    "    #before Covid and number of comments during Covid\n",
    "\n",
    "    # Will do a for loop that goes through each file once and get the stats on number of comments/posts\n",
    "    for filename in all_files:\n",
    "        username = filename.split('/')[-1].split('.')[0]\n",
    "        extension = filename.split('/')[-1].split('.')[1]\n",
    "        if extension == 'tsv':\n",
    "            df_temp = pd.read_csv(filename, sep='\\t')\n",
    "        else:\n",
    "            df_temp = pd.read_csv(filename, sep=',') # for csv files\n",
    "        total_coms = df_temp.shape[0]\n",
    "        # spliting based on Covid official pandemic date\n",
    "        try:\n",
    "            b_df_temp = df_temp.loc[df_temp['created_utc'] < datetime.datetime(2020, 3, 11, 0, 0).timestamp()]\n",
    "            a_df_temp = df_temp.loc[df_temp['created_utc'] > datetime.datetime(2020, 3, 10, 0, 0).timestamp()]\n",
    "        except TypeError:    \n",
    "            utc_lst = df_temp['created_utc'].tolist()\n",
    "            ind_lst = [] # list of indices in the data frame that is causing problems\n",
    "            for i in range(df_temp.shape[0]):\n",
    "                try:\n",
    "                    temp = float(utc_lst[0])\n",
    "                except ValueError:\n",
    "                    ind_lst.append(i)\n",
    "            df_temp.drop(df_temp.index[ind_lst], inplace=True)\n",
    "            b_df_temp = df_temp.loc[df_temp['created_utc'] < datetime.datetime(2020, 3, 11, 0, 0).timestamp()]\n",
    "            a_df_temp = df_temp.loc[df_temp['created_utc'] > datetime.datetime(2020, 3, 10, 0, 0).timestamp()]\n",
    "        b_coms = b_df_temp.shape[0]\n",
    "        a_coms = a_df_temp.shape[0]\n",
    "        stats_dict[username] = [total_coms, b_coms, a_coms]\n",
    "    return stats_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6704af4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This notebook will contain the code for the analysis of reddit posts by individual reddit users.\n",
    "\n",
    "# Working on Smoking data set (comments)\n",
    "\n",
    "path_smokeC = \"/Users/nayza/Downloads/comments_threads/smokingComments_threads/\"\n",
    "smokeC_files = glob.glob(path_smokeC + \"*.tsv\") # glob allows one to import all files with the same file extension at once\n",
    "statsDict_smokeC = getStatsDict(smokeC_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c8f89423",
   "metadata": {},
   "outputs": [],
   "source": [
    "# continuation of the previous cell; for Smoking (comments)\n",
    "df_smokeC = pd.DataFrame.from_dict(statsDict_smokeC, orient='index', columns=['Total comments', 'Before Covid',\n",
    "                                                                      'During Covid'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "bf7ba1b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "dfSmokeC = df_smokeC.loc[(df_smokeC['Before Covid'] > 0) & (df_smokeC['During Covid'] > 0)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "466e7fe3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5129"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfSmokeC.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "ba82d79e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now, will do the same thing for Smoking (sumbmissions)\n",
    "\n",
    "path_smokeS = \"/Users/nayza/Downloads/Submissions_threads/smoking_threads/\"\n",
    "smokeS_files = glob.glob(path_smokeS + \"*.tsv\")\n",
    "statsDict_smokeS = getStatsDict(smokeS_files)\n",
    "\n",
    "df_smokeS = pd.DataFrame.from_dict(statsDict_smokeS, orient='index', columns=['Total submissions', 'Before Covid',\n",
    "                                                                      'During Covid'])\n",
    "\n",
    "dfSmokeS = df_smokeS.loc[(df_smokeS['Before Covid'] > 0) & (df_smokeS['During Covid'] > 0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "4a3c8da0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "821"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfSmokeS.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "808c3f10",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lastly, Drinking data set.\n",
    "# Note that both submissions and comments files of the drinking data set are stored in one folder (one path). Will \n",
    "# separate them out in the next cell.\n",
    "\n",
    "path_drinkCS = \"/Users/nayza/Downloads/stopdrinking_authors/\"\n",
    "drinkCS_files = glob.glob(path_drinkCS + \"*.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "eeec1a1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separating out drinkCS_files into comments and submissions.\n",
    "\n",
    "#drinkCS_files[0].split('/')[-1]\n",
    "drinkC_files, drinkS_files = [], []\n",
    "notCorS_files = [] # if the filename has neither 'comments' or 'submissions' in it, filepath will be appended to here.\n",
    "for filepath in drinkCS_files:\n",
    "    filename = filepath.split('/')[-1]\n",
    "    if re.search('comments', filename):\n",
    "        drinkC_files.append(filepath)\n",
    "    elif re.search('submissions', filename):\n",
    "        drinkS_files.append(filepath)\n",
    "    else:\n",
    "        notCorS_files.append(filepath)\n",
    "# notCorS_files is empty!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "e2434b0e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Now, we repeat what we did with smokeC and smokeS here with drinkC and drinkS.\n",
    "\n",
    "# first drinkC\n",
    "statsDict_drinkC = getStatsDict(drinkC_files)\n",
    "df_drinkC = pd.DataFrame.from_dict(statsDict_drinkC, orient='index', columns=['Total comments', 'Before Covid',\n",
    "                                                                      'During Covid'])\n",
    "dfDrinkC = df_drinkC.loc[(df_drinkC['Before Covid'] > 0) & (df_drinkC['During Covid'] > 0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "d76048e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lastly, drinkS.\n",
    "statsDict_drinkS = getStatsDict(drinkS_files)\n",
    "df_drinkS = pd.DataFrame.from_dict(statsDict_drinkS, orient='index', columns=['Total submissions', 'Before Covid',\n",
    "                                                                      'During Covid'])\n",
    "dfDrinkS = df_drinkS.loc[(df_drinkS['Before Covid'] > 0) & (df_drinkS['During Covid'] > 0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "265c561e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now, exporting the results as CSV files\n",
    "\n",
    "#dfSmokeC.to_csv(\"/Users/nayza/Desktop/YTproject/smokeComments_results.csv\")\n",
    "#dfSmokeS.to_csv(\"/Users/nayza/Desktop/YTproject/smokeSubmissions_results.csv\")\n",
    "#dfDrinkC.to_csv(\"/Users/nayza/Desktop/YTproject/drinkComments_results.csv\")\n",
    "#dfDrinkS.to_csv(\"/Users/nayza/Desktop/YTproject/drinkSubmissions_results.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "c52248f0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}