Reddit_Pull.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from pandas.io.json import json_normalize\n",
    "import os\n",
    "import pandas as pd\n",
    "import time\n",
    "import math\n",
    "from datetime import datetime\n",
    "import numpy as np\n",
    "import sqlalchemy \n",
    "import psycopg2\n",
    "import smtplib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datetime.datetime(2020, 10, 1, 0, 0)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "company = 'GAME_STOP'\n",
    "keyword = '\"GME\"'\n",
    "date_str = '2020-10-01'\n",
    "date = datetime.strptime(date_str, '%Y-%m-%d')\n",
    "date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PushShiftPull():\n",
    "    def _init_(keyword,last_comment,last_submmision,company):\n",
    "        self.company = company\n",
    "        self.keyword = keyword\n",
    "        self.last_comment = last_comment\n",
    "        self.last_submission = last_submission\n",
    "    def create_directories(self):\n",
    "        def check_create(directory):\n",
    "            if not os.path.exists(directory):\n",
    "                os.makedirs(directory)\n",
    "        #Topline\n",
    "        self.directory = os.path.join(os.getcwd(),'Reddit_Content',self.company,self.keyword)\n",
    "        check_create(self.directory)\n",
    "        #Comments\n",
    "        self.directory_comments = os.path.join(self.directory,'comments')\n",
    "        check_create(self.directory_comments)\n",
    "        #Posts\n",
    "        self.directory_posts = os.path.join(self.directory,'posts')\n",
    "        check_create(self.directory_posts)\n",
    "        print(self.directory)\n",
    "    def downloadFromUrl_comment(self):\n",
    "        last_comment_timestamp = int(self.last_comment.timestamp())\n",
    "        #print(last_comment_timestamp)\n",
    "        #url = \"https://api.pushshift.io/reddit/comment/search?q={}&after={}&limit=1000&sort=desc&before=\".format(self.keyword,self.comment_max)\n",
    "        url = \"https://api.pushshift.io/reddit/comment/search?q={}&limit=1000&subreddit=wallstreetbets&sort=desc&after={}&before=\".format(self.keyword,last_comment_timestamp )\n",
    "        print(url)\n",
    "        count = 0\n",
    "        previous_epoch = int(datetime.utcnow().timestamp())\n",
    "        dfs = []\n",
    "        while True:\n",
    "            #new_url = url+str(previous_epoch)\n",
    "            new_url = url.format(self.keyword,last_comment_timestamp)+str(previous_epoch)\n",
    "            print(new_url)\n",
    "            json = requests.get(new_url, headers={'User-Agent': \"Post downloader by /u/Watchful1\"})\n",
    "            try:\n",
    "                json_data = json.json()\n",
    "            except:\n",
    "                print('failed!')\n",
    "                break\n",
    "            if 'data' not in json_data:\n",
    "                break\n",
    "            objects = json_data['data']\n",
    "            if len(objects) == 0:\n",
    "                break\n",
    "            print('no break')\n",
    "            for object in objects:\n",
    "                previous_epoch = object['created_utc'] - 1\n",
    "                count += 1\n",
    "                try:\n",
    "                    if count == 1:\n",
    "                        self.comment_df = json_normalize(object)\n",
    "                    else:\n",
    "                        df = json_normalize(object)\n",
    "                        self.comment_df = pd.concat([self.comment_df,df],axis = 0,ignore_index = True,sort=False)\n",
    "                except Exception as err:\n",
    "                    print(f\"Couldn't print comment: https://www.reddit.com{object['permalink']}\")\n",
    "                    print(traceback.format_exc())\n",
    "            print(\"Saved {} comments through {}\".format(count, datetime.fromtimestamp(previous_epoch).strftime(\"%Y-%m-%d\")))\n",
    "        try:\n",
    "            #self.comment_df = pd.concat(dfs,axis = 0,ignore_index = True,sort=False)\n",
    "            self.comment_df['created_utc'] = pd.to_datetime(self.comment_df['created_utc'],unit='s')\n",
    "            self.comment_df['type'] = 'comment'\n",
    "            columns = ['author','author_fullname','body','created_utc','id','link_id','parent_id','permalink','score','subreddit','subreddit_id','subreddit_name_prefixed','subreddit_type','total_awards_received','type']\n",
    "            self.comment_df = self.comment_df.reindex(columns = columns)\n",
    "            #self.comment_df.to_csv(self.filename_comments,index = False)\n",
    "        except:\n",
    "            print('Not comments for ' + str(self.keyword))\n",
    "    def downloadFromUrl_submission(self):\n",
    "        #url = \"https://api.pushshift.io/reddit/submission/search?q={}&after={}&limit=1000&sort=desc&author=&before=\".format(self.keyword,self.submission_max)\n",
    "        #print(url)\n",
    "        last_sumbission = int(self.last_submission.timestamp())\n",
    "        url = \"https://api.pushshift.io/reddit/submission/search?q={}&limit=1000&subreddit=wallstreetbets&sort=desc&author=&after={}&before=\".format(self.keyword,last_sumbission)\n",
    "        count = 0\n",
    "        previous_epoch = int(datetime.utcnow().timestamp())\n",
    "        dfs = []\n",
    "        while True:\n",
    "            new_url = url.format(self.keyword,last_sumbission)+str(previous_epoch)\n",
    "            json = requests.get(new_url, headers={'User-Agent': \"Post downloader by /u/Watchful1\"})\n",
    "            print('got json')\n",
    "            try:\n",
    "                json_data = json.json()\n",
    "            except:\n",
    "                print('failed!')\n",
    "                break\n",
    "            if 'data' not in json_data:\n",
    "                break\n",
    "            objects = json_data['data']\n",
    "            if len(objects) == 0:\n",
    "                break\n",
    "            for object in objects:\n",
    "                previous_epoch = object['created_utc'] - 1\n",
    "                count += 1\n",
    "                #if object['is_self']:\n",
    "                #    if 'selftext' not in object:\n",
    "                #        continue\n",
    "                try:\n",
    "                    if count == 1:\n",
    "                        self.submission_df = json_normalize(object)\n",
    "                    else:\n",
    "                        df = json_normalize(object)\n",
    "                        self.submission_df = pd.concat([self.submission_df,df],axis = 0,ignore_index = True,sort=False)\n",
    "                except Exception as err:\n",
    "                    print(f\"Couldn't print post: {object['url']}\")\n",
    "                    print(traceback.format_exc())\n",
    "            print(\"Saved {} submissions through {}\".format(count, datetime.fromtimestamp(previous_epoch).strftime(\"%Y-%m-%d\")))\n",
    "        try:\n",
    "            self.submission_df['created_utc'] = pd.to_datetime(self.submission_df['created_utc'],unit='s')\n",
    "            self.submission_df['type'] = 'submission'\n",
    "            columns = ['author','author_fullname','created_utc','full_link','id','is_meta','is_original_content','is_reddit_media_domain','is_robot_indexable','is_self','is_video','media_only','num_comments','num_crossposts','permalink','score','selftext','subreddit','subreddit_id','subreddit_subscribers','subreddit_type','thumbnail','title','url','crosspost_parent','promoted','media.type']\n",
    "            self.submission_df = self.submission_df.reindex(columns = columns)\n",
    "        except:\n",
    "            print('No DF for Submissions!')\n",
    "        #self.submission_df.to_csv(self.filename_submission,index = False)\n",
    "    def create_files(self):\n",
    "        #comments\n",
    "        print('Saving Comments')\n",
    "        cdate_min = str(np.min(self.comment_df['created_utc']))\n",
    "        cdate_max = str(np.max(self.comment_df['created_utc']))\n",
    "        comment_stuff = [self.company,self.keyword,'comments',cdate_min,cdate_max]\n",
    "        comments_files = '-'.join(comment_stuff )\n",
    "        self.comment_df.to_csv(os.path.join(self.directory_comments,comments_files),index = False)\n",
    "        print('Saved Comments')\n",
    "        #Posts\n",
    "        pdate_min = str(np.min(self.submission_df['created_utc']))\n",
    "        pdate_max = str(np.max(self.submission_df['created_utc']))\n",
    "        submission_stuff = [self.company,self.keyword,'posts',pdate_min,pdate_max]\n",
    "        submission_file = '-'.join(submission_stuff)\n",
    "        self.submission_df.to_csv(os.path.join(self.directory_posts,submission_file),index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Run_PushShift(keyword,company,last_comment,last_submission):\n",
    "    print(keyword)\n",
    "    print(last_comment)\n",
    "    print(last_submission)\n",
    "    #Associate Variables\n",
    "    Test = PushShiftPull()\n",
    "    Test.company = company\n",
    "    Test.keyword = keyword\n",
    "    Test.last_comment = last_comment\n",
    "    Test.last_submission = last_submission\n",
    "    #Run Code\n",
    "    Test.create_directories()\n",
    "    Test.downloadFromUrl_comment()\n",
    "    Test.downloadFromUrl_submission()\n",
    "    Test.create_files()\n",
    "    return(Test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"GME\"\n",
      "2020-10-01 00:00:00\n",
      "2020-10-01 00:00:00\n",
      "/Users/Avery/Desktop/WSB/Reddit_Content/GAME_STOP/\"GME\"\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611781603\n",
      "no break\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:51: FutureWarning: pandas.io.json.json_normalize is deprecated, use pandas.json_normalize instead\n",
      "/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:53: FutureWarning: pandas.io.json.json_normalize is deprecated, use pandas.json_normalize instead\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved 100 comments through 2021-01-24\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611485732\n",
      "no break\n",
      "Saved 200 comments through 2021-01-24\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611481724\n",
      "no break\n",
      "Saved 300 comments through 2021-01-24\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611479351\n",
      "no break\n",
      "Saved 400 comments through 2021-01-24\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611476435\n",
      "no break\n",
      "Saved 500 comments through 2021-01-24\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611474352\n",
      "no break\n",
      "Saved 600 comments through 2021-01-24\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611472250\n",
      "no break\n",
      "Saved 700 comments through 2021-01-24\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611470212\n",
      "no break\n",
      "Saved 800 comments through 2021-01-24\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611468469\n",
      "no break\n",
      "Saved 900 comments through 2021-01-24\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611466616\n",
      "no break\n",
      "Saved 1000 comments through 2021-01-24\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611464938\n",
      "no break\n",
      "Saved 1100 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611463576\n",
      "no break\n",
      "Saved 1200 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611462489\n",
      "no break\n",
      "Saved 1300 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611461378\n",
      "no break\n",
      "Saved 1400 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611460436\n",
      "no break\n",
      "Saved 1500 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611459223\n",
      "no break\n",
      "Saved 1600 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611458039\n",
      "no break\n",
      "Saved 1700 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611456832\n",
      "no break\n",
      "Saved 1800 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611455817\n",
      "no break\n",
      "Saved 1900 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611454819\n",
      "no break\n",
      "Saved 2000 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611453772\n",
      "no break\n",
      "Saved 2100 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611452716\n",
      "no break\n",
      "Saved 2200 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611451782\n",
      "no break\n",
      "Saved 2300 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611450695\n",
      "no break\n",
      "Saved 2400 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611449706\n",
      "no break\n",
      "Saved 2500 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611448491\n",
      "no break\n",
      "Saved 2600 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611447624\n",
      "no break\n",
      "Saved 2700 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611446715\n",
      "no break\n",
      "Saved 2800 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611445995\n",
      "no break\n",
      "Saved 2900 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611445339\n",
      "no break\n",
      "Saved 3000 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611444502\n",
      "no break\n",
      "Saved 3100 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611443780\n",
      "no break\n",
      "Saved 3200 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611442849\n",
      "no break\n",
      "Saved 3300 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611441973\n",
      "no break\n",
      "Saved 3400 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611441204\n",
      "no break\n",
      "Saved 3500 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611440420\n",
      "no break\n",
      "Saved 3600 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611439656\n",
      "no break\n",
      "Saved 3700 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611438848\n",
      "no break\n",
      "Saved 3800 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611438132\n",
      "no break\n",
      "Saved 3900 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611437284\n",
      "no break\n",
      "Saved 4000 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611436467\n",
      "no break\n",
      "Saved 4100 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611435654\n",
      "no break\n",
      "Saved 4200 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611434845\n",
      "no break\n",
      "Saved 4300 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611434030\n",
      "no break\n",
      "Saved 4400 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611433314\n",
      "no break\n",
      "Saved 4500 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611432636\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no break\n",
      "Saved 4600 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611431824\n",
      "no break\n",
      "Saved 4700 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611430959\n",
      "no break\n",
      "Saved 4800 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611430179\n",
      "no break\n",
      "Saved 4900 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611429356\n",
      "no break\n",
      "Saved 5000 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611428757\n",
      "no break\n",
      "Saved 5100 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611428206\n",
      "no break\n",
      "Saved 5200 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611427421\n",
      "no break\n",
      "Saved 5300 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611426692\n",
      "no break\n",
      "Saved 5400 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611425852\n",
      "no break\n",
      "Saved 5500 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611424931\n",
      "no break\n",
      "Saved 5600 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611424107\n",
      "no break\n",
      "Saved 5700 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611423309\n",
      "no break\n",
      "Saved 5800 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611422346\n",
      "no break\n",
      "Saved 5900 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611421556\n",
      "no break\n",
      "Saved 6000 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611420725\n",
      "no break\n",
      "Saved 6100 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611419828\n",
      "no break\n",
      "Saved 6200 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611419134\n",
      "no break\n",
      "Saved 6300 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611418245\n",
      "no break\n",
      "Saved 6400 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611417298\n",
      "no break\n",
      "Saved 6500 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611416410\n",
      "no break\n",
      "Saved 6600 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611415377\n",
      "no break\n",
      "Saved 6700 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611414516\n",
      "no break\n",
      "Saved 6800 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611413448\n",
      "no break\n",
      "Saved 6900 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611412325\n",
      "no break\n",
      "Saved 7000 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611411380\n",
      "no break\n",
      "Saved 7100 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611410427\n",
      "no break\n",
      "Saved 7200 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611409111\n",
      "no break\n",
      "Saved 7300 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611407706\n",
      "no break\n",
      "Saved 7400 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611405659\n",
      "no break\n",
      "Saved 7500 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611404004\n",
      "no break\n",
      "Saved 7600 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611402073\n",
      "no break\n",
      "Saved 7700 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611400557\n",
      "no break\n",
      "Saved 7800 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611398747\n",
      "no break\n",
      "Saved 7900 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611397096\n",
      "no break\n",
      "Saved 8000 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611395293\n",
      "no break\n",
      "Saved 8100 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611393171\n",
      "no break\n",
      "Saved 8200 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611390911\n",
      "no break\n",
      "Saved 8300 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611388895\n",
      "no break\n",
      "Saved 8400 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611387251\n",
      "no break\n",
      "Saved 8500 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611385905\n",
      "no break\n",
      "Saved 8600 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611384668\n",
      "no break\n",
      "Saved 8700 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611383386\n",
      "no break\n",
      "Saved 8800 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611382287\n",
      "no break\n",
      "Saved 8900 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611381456\n",
      "no break\n",
      "Saved 9000 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611380593\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no break\n",
      "Saved 9100 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611379849\n",
      "no break\n",
      "Saved 9200 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611379197\n",
      "no break\n",
      "Saved 9300 comments through 2021-01-23\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611378486\n",
      "no break\n",
      "Saved 9400 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611377506\n",
      "no break\n",
      "Saved 9500 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611376715\n",
      "no break\n",
      "Saved 9600 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611376054\n",
      "no break\n",
      "Saved 9700 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611375260\n",
      "no break\n",
      "Saved 9800 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611374487\n",
      "no break\n",
      "Saved 9900 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611373885\n",
      "no break\n",
      "Saved 10000 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611373024\n",
      "no break\n",
      "Saved 10100 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611372209\n",
      "no break\n",
      "Saved 10200 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611371301\n",
      "no break\n",
      "Saved 10300 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611370495\n",
      "no break\n",
      "Saved 10400 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611369841\n",
      "no break\n",
      "Saved 10500 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611369151\n",
      "no break\n",
      "Saved 10600 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611368306\n",
      "no break\n",
      "Saved 10700 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611367701\n",
      "no break\n",
      "Saved 10800 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611366951\n",
      "no break\n",
      "Saved 10900 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611366276\n",
      "no break\n",
      "Saved 11000 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611365596\n",
      "no break\n",
      "Saved 11100 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611364893\n",
      "no break\n",
      "Saved 11200 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611364303\n",
      "no break\n",
      "Saved 11300 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611363737\n",
      "no break\n",
      "Saved 11400 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611363058\n",
      "no break\n",
      "Saved 11500 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611362318\n",
      "no break\n",
      "Saved 11600 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611361743\n",
      "no break\n",
      "Saved 11700 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611361142\n",
      "no break\n",
      "Saved 11800 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611360543\n",
      "no break\n",
      "Saved 11900 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611359825\n",
      "no break\n",
      "Saved 12000 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611359351\n",
      "no break\n",
      "Saved 12100 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611358792\n",
      "no break\n",
      "Saved 12200 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611358305\n",
      "no break\n",
      "Saved 12300 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611357790\n",
      "no break\n",
      "Saved 12400 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611357261\n",
      "no break\n",
      "Saved 12500 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611356764\n",
      "no break\n",
      "Saved 12600 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611356275\n",
      "no break\n",
      "Saved 12700 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611355889\n",
      "no break\n",
      "Saved 12800 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611355440\n",
      "no break\n",
      "Saved 12900 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611354996\n",
      "no break\n",
      "Saved 13000 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611354598\n",
      "no break\n",
      "Saved 13100 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611354217\n",
      "no break\n",
      "Saved 13200 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611353867\n",
      "no break\n",
      "Saved 13300 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611353452\n",
      "no break\n",
      "Saved 13400 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611353044\n",
      "no break\n",
      "Saved 13500 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611352706\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no break\n",
      "Saved 13600 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611352409\n",
      "no break\n",
      "Saved 13700 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611352036\n",
      "no break\n",
      "Saved 13800 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611351702\n",
      "no break\n",
      "Saved 13900 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611351392\n",
      "no break\n",
      "Saved 14000 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611351062\n",
      "no break\n",
      "Saved 14100 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611350778\n",
      "no break\n",
      "Saved 14200 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611350505\n",
      "no break\n",
      "Saved 14300 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611350158\n",
      "no break\n",
      "Saved 14400 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611349841\n",
      "no break\n",
      "Saved 14500 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611349582\n",
      "no break\n",
      "Saved 14600 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611349321\n",
      "no break\n",
      "Saved 14700 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611348962\n",
      "no break\n",
      "Saved 14800 comments through 2021-01-22\n",
      "https://api.pushshift.io/reddit/comment/search?q=\"GME\"&limit=1000&subreddit=wallstreetbets&sort=desc&after=1601524800&before=1611348651\n",
      "no break\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-ede44d5b8e91>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mAvery\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRun_PushShift\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkeyword\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcompany\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdate\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-8-6266db77e850>\u001b[0m in \u001b[0;36mRun_PushShift\u001b[0;34m(keyword, company, last_comment, last_submission)\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0;31m#Run Code\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mTest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_directories\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m     \u001b[0mTest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdownloadFromUrl_comment\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m     \u001b[0mTest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdownloadFromUrl_submission\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0mTest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_files\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-7-ee1a711056dc>\u001b[0m in \u001b[0;36mdownloadFromUrl_comment\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     52\u001b[0m                     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m                         \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson_normalize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobject\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 54\u001b[0;31m                         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcomment_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcomment_df\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0maxis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mignore_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0msort\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     55\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m                     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Couldn't print comment: https://www.reddit.com{object['permalink']}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/pandas/core/reshape/concat.py\u001b[0m in \u001b[0;36mconcat\u001b[0;34m(objs, axis, join, ignore_index, keys, levels, names, verify_integrity, sort, copy)\u001b[0m\n\u001b[1;32m    285\u001b[0m     )\n\u001b[1;32m    286\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 287\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    288\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    289\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/pandas/core/reshape/concat.py\u001b[0m in \u001b[0;36mget_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    501\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    502\u001b[0m             new_data = concatenate_block_managers(\n\u001b[0;32m--> 503\u001b[0;31m                 \u001b[0mmgrs_indexers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnew_axes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconcat_axis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbm_axis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    504\u001b[0m             )\n\u001b[1;32m    505\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/pandas/core/internals/concat.py\u001b[0m in \u001b[0;36mconcatenate_block_managers\u001b[0;34m(mgrs_indexers, axes, concat_axis, copy)\u001b[0m\n\u001b[1;32m     52\u001b[0m     \u001b[0mblocks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 54\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mplacement\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjoin_units\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mconcat_plan\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     55\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjoin_units\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mjoin_units\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindexers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/pandas/core/internals/concat.py\u001b[0m in \u001b[0;36m_combine_concat_plans\u001b[0;34m(plans, concat_axis)\u001b[0m\n\u001b[1;32m    573\u001b[0m                         \u001b[0;31m# _trim_join_unit updates unit in place, so only\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    574\u001b[0m                         \u001b[0;31m# placement needs to be sliced to skip min_len.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 575\u001b[0;31m                         \u001b[0mnext_items\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mplc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmin_len\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_trim_join_unit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0munit\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmin_len\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    576\u001b[0m                     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    577\u001b[0m                         \u001b[0myielded_placement\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mplc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/pandas/core/internals/concat.py\u001b[0m in \u001b[0;36m_trim_join_unit\u001b[0;34m(join_unit, length)\u001b[0m\n\u001b[1;32m    509\u001b[0m         \u001b[0mextra_block\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjoin_unit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mblock\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    510\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 511\u001b[0;31m         \u001b[0mextra_indexers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjoin_unit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindexers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    512\u001b[0m         \u001b[0mextra_indexers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mextra_indexers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlength\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    513\u001b[0m         \u001b[0mjoin_unit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindexers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjoin_unit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindexers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlength\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/copy.py\u001b[0m in \u001b[0;36mcopy\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m     70\u001b[0m     \"\"\"\n\u001b[1;32m     71\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 72\u001b[0;31m     \u001b[0mcls\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     73\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     74\u001b[0m     \u001b[0mcopier\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_copy_dispatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "Avery = Run_PushShift(keyword,company,date,date)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}