listen_stream (1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import twitter\n",
    "import json\n",
    "import datetime\n",
    "from datetime import *\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn import svm, ensemble\n",
    "from sklearn.metrics import *\n",
    "from sklearn.cross_validation import train_test_split, KFold\n",
    "from autorizador import *\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "auth = get_credents()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_credents():\n",
    "\tcreds = get_creds('secrets-manu.txt')\n",
    "\tauth=twitter.OAuth(creds.AccessToken,creds.AccessTokenSecret,creds.ConsumerKey, creds.ConsumerKeySecret)\n",
    "\treturn auth\n",
    "\n",
    "def save_tweet(tweet, f):\n",
    "    # Replace HTML entities; function extracted from Borja Sotomayor Twitter Harvester \n",
    "    tweet['text'] = tweet['text'].replace(\"&gt;\", \">\").replace(\"&lt;\", \"<\").replace(\"&amp;\", \"&\")\n",
    "    json.dump(tweet, f)\n",
    "    f.write('\\n')\n",
    "\n",
    "def save_user_tweets(user, n, auth):\n",
    "    t = twitter.Twitter(auth=auth)\n",
    "    print(\"Fetching %i tweets from @%s\" % (n, user))\n",
    "    tweets = t.statuses.user_timeline(screen_name=user, count=n)\n",
    "    print(\"  (actually fetched %i)\" % len(tweets))\n",
    "    for tweet in tweets:\n",
    "        save_tweet(tweet, outfile)\n",
    "        print(tweet['text'])\n",
    "\n",
    "# read the filters file to track\n",
    "# infile = open('filters_file.txt', 'r')\n",
    "# track = \",\".join(infile.read().strip().split(\"\\n\"))\n",
    "\n",
    "def get_tweets(num_tweets, auth, filters_file, filter_words, filename_):\n",
    "    '''\n",
    "    Person Responsible: Manu Aragones\n",
    "\n",
    "    + filter_file: list of words / phrases to be included in query IN FILE\n",
    "    + filter_words: if filter_file not specified -> you can input filter manually\n",
    "                    format: comma-separated list of phrases which will be used to\n",
    "                            determine what Tweets will be delivered on the stream\n",
    "    + num_tweets: the maximum number of tweets before break\n",
    "    builds query for twitter API from user input\n",
    "\n",
    "    Simplified fuction from Borja Sotomayor Twitter Harvester\n",
    "\n",
    "    '''\n",
    "    outf = open(filename_, \"w\")\n",
    "    # Connect to the stream\n",
    "    twitter_stream = twitter.TwitterStream(auth=auth)\n",
    "    if filter_words is None and filters_file is None:\n",
    "        stream = twitter_stream.statuses.sample()\n",
    "    else:\n",
    "        if filter_words is not None:\n",
    "            track = \",\".join(filter_words)\n",
    "        elif filters_file is not None:\n",
    "            infile = open(filters_file, 'r')\n",
    "            track = \",\".join(infile.read().strip().split(\"\\n\"))\n",
    "        stream = twitter_stream.statuses.filter(track=track)\n",
    "    # Fetch the tweets\n",
    "    fetched = 0\n",
    "\n",
    "    if num_tweets > 0:\n",
    "        if outf != sys.stdout: print(\"Fetching %i tweets... \" % num_tweets)\n",
    "    else:\n",
    "        signal.signal(signal.SIGINT, signal_handler)\n",
    "        now = datetime.now().isoformat(sep=\" \")\n",
    "        msg = \"[{}] Fetching tweets. Press Ctrl+C to stop.\".format(now)\n",
    "        if outf != sys.stdout: print(msg)\n",
    "\n",
    "    for tweet in stream:\n",
    "        # The public stream includes tweets, but also other messages, such\n",
    "        # as deletion notices. We are only interested in the tweets.\n",
    "        # See: https://dev.twitter.com/streaming/overview/messages-types\n",
    "        if tweet.get('text'):\n",
    "            # We also only want English tweets\n",
    "            if tweet['lang'] == \"en\" or tweet['lang'] == \"es\":\n",
    "                save_tweet(tweet, outf)\n",
    "                fetched += 1\n",
    "                if fetched % num_tweets == 0:\n",
    "                    now = datetime.now().isoformat(sep=\" \")\n",
    "                    msg = \"[{}] Fetched {:,} tweets.\".format(now, fetched)\n",
    "                    if outf != sys.stdout: \n",
    "                        print(msg)\n",
    "                        stream.close()\n",
    "                if num_tweets > 0 and fetched >= num_tweets:\n",
    "                    stream.close()\n",
    "                    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fetching 20 tweets... \n",
      "[2016-06-01 13:14:43.137111] Fetched 20 tweets.\n"
     ]
    }
   ],
   "source": [
    "get_tweets(20, auth, None, ['Trump','Hillary'], 'Trumpd2_test.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}