3_NLTK Experiments.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Competition - Spooky Author Identification\n",
    "**Identify horror authors from their writings**\n",
    "\n",
    "**File descriptions**\n",
    "\n",
    "    train.csv - the training set\n",
    "    test.csv - the test set\n",
    "    sample_submission.csv - a sample submission file in the correct format\n",
    "\n",
    "**Data fields**\n",
    "\n",
    "    id - a unique identifier for each sentence\n",
    "    text - some text written by one of the authors\n",
    "    author - the author of the sentence (EAP: Edgar Allan Poe, HPL: HP Lovecraft; MWS: Mary Wollstonecraft Shelley)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLTK Experiments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## This notebook looks at experiments to include the results of the Grammer: Parts of Speech analysis from the Exploratory Data Analysis notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 398,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import itertools\n",
    "import nltk\n",
    "from nltk import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import wordnet\n",
    "from nltk import pos_tag\n",
    "from nltk.stem.snowball import EnglishStemmer\n",
    "from sklearn.feature_extraction.text import CountVectorizer, HashingVectorizer\n",
    "from sklearn.feature_extraction.text import  TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "# Set graph parameters\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 399,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot confusion matrix\n",
    "# http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html#sphx-glr-auto-examples-model-selection-plot-confusion-matrix-py\n",
    "def plot_conf_matrix(cm, classes, title='Confusion Matrix', cmap=plt.cm.Blues):\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "    fmt = 'd'\n",
    "    thresh = cm.max()/2\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, format(cm[i, j], fmt),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True Label')\n",
    "    plt.xlabel('Predicted Label')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 400,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the path of the spooky author dataset\n",
    "SPOOKY_PATH = 'spooky'\n",
    "\n",
    "\n",
    "def load_spooky_dataset(dataset_type):\n",
    "    filepath = os.path.join(SPOOKY_PATH, dataset_type, dataset_type + \".csv\")\n",
    "    print(filepath)\n",
    "    return pd.read_csv(filepath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 401,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spooky\\train\\train.csv\n",
      "spooky\\test\\test.csv\n"
     ]
    }
   ],
   "source": [
    "train = load_spooky_dataset(\"train\")\n",
    "test = load_spooky_dataset(\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 402,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(train['text'], train['author'])\n",
    "#X_train, y_train = train['text'], train['author']\n",
    "#X_test = test['text']\n",
    "#X_ID = test['id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = 'How is your studies? You should be studying.'\n",
    "#text = train.text[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_text = nltk.word_tokenize(text)\n",
    "tagged_text = [nltk.pos_tag(word) for word in tokenized_text]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "wnl = WordNetLemmatizer()\n",
    "# Reference : http://coling.epfl.ch/TP/TP-tagging.html\n",
    "def get_wordnet_pos(universal_tag):\n",
    "    if universal_tag == 'VERB':\n",
    "        return wordnet.VERB\n",
    "    elif universal_tag == 'ADJ':\n",
    "        return wordnet.ADJ\n",
    "    elif universal_tag == 'ADV':\n",
    "        return wordnet.ADV\n",
    "    else:\n",
    "        return wordnet.NOUN\n",
    "\n",
    "def lemmaToken(text):\n",
    "    tokenized_text = nltk.word_tokenize(text)\n",
    "    tagged_text = nltk.pos_tag(tokenized_text)\n",
    "    print(tagged_text)\n",
    "    lemma_text = []\n",
    "    for item in tagged_text:\n",
    "        #print(item[0], item[1])\n",
    "        lemma_text.append(wnl.lemmatize(item[0], pos=get_wordnet_pos(item[1])))\n",
    "    return lemma_text\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['How', 'is', 'your', 'studies', '?', 'You', 'should', 'be', 'studying', '.']\n",
      "Stem: ['how', 'is', 'your', 'studi', '?', 'you', 'should', 'be', 'studi', '.']\n",
      "Lemmatize: ['How', 'is', 'your', 'study', '?', 'You', 'should', 'be', 'studying', '.']\n",
      "[('How', 'WRB'), ('is', 'VBZ'), ('your', 'PRP$'), ('studies', 'NNS'), ('?', '.'), ('You', 'PRP'), ('should', 'MD'), ('be', 'VB'), ('studying', 'VBG'), ('.', '.')]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['How', 'is', 'your', 'study', '?', 'You', 'should', 'be', 'studying', '.']"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(tokenized_text)\n",
    "print('Stem:',[EnglishStemmer().stem(word) for word in tokenized_text])\n",
    "print('Lemmatize:',[WordNetLemmatizer().lemmatize(word) for word in tokenized_text])\n",
    "lemmaToken(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 602,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['The 1 quick brown fox jumps over the 2 lazy dog']\n"
     ]
    }
   ],
   "source": [
    "#nltk.download()\n",
    "sen = nltk.sent_tokenize(text)\n",
    "print(sen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 606,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['The', '1', 'quick', 'brown', 'fox', 'jumps', 'over', 'the', '2', 'lazy', 'dog']]\n"
     ]
    }
   ],
   "source": [
    "sentences = [nltk.word_tokenize(sent) for sent in sen]\n",
    "print(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 604,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[('The', 'DT'), ('1', 'CD'), ('quick', 'JJ'), ('brown', 'NN'), ('fox', 'NN'), ('jumps', 'VBZ'), ('over', 'IN'), ('the', 'DT'), ('2', 'CD'), ('lazy', 'NN'), ('dog', 'NN')]]\n"
     ]
    }
   ],
   "source": [
    "sentences = [nltk.pos_tag(sent) for sent in sentences]\n",
    "print(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LemmaTokenizer(object):\n",
    "    def __init__(self):\n",
    "        self.wnl = WordNetLemmatizer()\n",
    "    def __call__(self, doc):\n",
    "        return [self.wnl.lemmatize(t) for t in word_tokenize(doc)]\n",
    "        \n",
    "class LemmaTokenizer_c(object):\n",
    "    def __init__(self):\n",
    "        self.wnl = WordNetLemmatizer()\n",
    "    def __call__(self, doc):\n",
    "        # TODO check this\n",
    "        #tokenized_text = [nltk.word_tokenize(w) for w in text]\n",
    "        #tagged_text = [nltk.pos_tag(word_tokenize(text))]\n",
    "        #print(tagged_text)\n",
    "        #lemma_text = []\n",
    "        #for item in tagged_text:\n",
    "            #print(item)\n",
    "            #for w in item:\n",
    "                #print(w[0], w[1])\n",
    "            #lemma_text.append(wnl.lemmatize(w[0], pos=get_wordnet_pos(w[1])) for w in item)\n",
    "        #return lemma_text\n",
    "        return [self.wnl.lemmatize(w[0], pos=get_wordnet_pos(w[1])) for w in pos_tag(word_tokenize(doc))]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "metadata": {},
   "outputs": [],
   "source": [
    "vec1 = CountVectorizer(tokenizer=LemmaTokenizer())\n",
    "vec2 = CountVectorizer(tokenizer=LemmaTokenizer_c())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = ['How is your studies? You should be studying.']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1x10 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 10 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 317,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec1.fit_transform(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1x10 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 10 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 318,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec2.fit_transform(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'.': 0,\n",
       " '?': 1,\n",
       " 'be': 2,\n",
       " 'how': 3,\n",
       " 'is': 4,\n",
       " 'should': 5,\n",
       " 'study': 6,\n",
       " 'studying': 7,\n",
       " 'you': 8,\n",
       " 'your': 9}"
      ]
     },
     "execution_count": 319,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec1.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'.': 0,\n",
       " '?': 1,\n",
       " 'be': 2,\n",
       " 'how': 3,\n",
       " 'is': 4,\n",
       " 'should': 5,\n",
       " 'study': 6,\n",
       " 'studying': 7,\n",
       " 'you': 8,\n",
       " 'your': 9}"
      ]
     },
     "execution_count": 320,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec2.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Mr. Charles Green']\n"
     ]
    }
   ],
   "source": [
    "sample = 'For its introduction into common use for purposes of aerostation, we are indebted to Mr. Charles Green.'\n",
    "\n",
    "def extract_entity_names(t):\n",
    "    entity_names = []\n",
    "\n",
    "    if hasattr(t, 'label') and t.label:\n",
    "        if t.label() == 'NE':\n",
    "            entity_names.append(' '.join([child[0] for child in t]))\n",
    "        else:\n",
    "            for child in t:\n",
    "                entity_names.extend(extract_entity_names(child))\n",
    "\n",
    "    return entity_names\n",
    "\n",
    "sentences = nltk.sent_tokenize(sample)\n",
    "tokenized_sentences = [nltk.word_tokenize(sentence) for sentence in sentences]\n",
    "tagged_sentences = [nltk.pos_tag(sentence) for sentence in tokenized_sentences]\n",
    "chunked_sentences = nltk.ne_chunk_sents(tagged_sentences, binary=True)\n",
    "\n",
    "entities = []\n",
    "for tree in chunked_sentences:\n",
    "    entities.extend(extract_entity_names(tree))\n",
    "\n",
    "print(entities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 391,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = ['Ok, tell me, was it really, worth the effort, please']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 392,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1x27 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 27 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 392,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#ngram_range=(1, 2),\n",
    "cvec = CountVectorizer(token_pattern='[,]*\\w+[,]*', ngram_range=(1,3))\n",
    "cvec.fit_transform(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 393,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'effort,': 0,\n",
       " 'effort, please': 1,\n",
       " 'it': 2,\n",
       " 'it really,': 3,\n",
       " 'it really, worth': 4,\n",
       " 'me,': 5,\n",
       " 'me, was': 6,\n",
       " 'me, was it': 7,\n",
       " 'ok,': 8,\n",
       " 'ok, tell': 9,\n",
       " 'ok, tell me,': 10,\n",
       " 'please': 11,\n",
       " 'really,': 12,\n",
       " 'really, worth': 13,\n",
       " 'really, worth the': 14,\n",
       " 'tell': 15,\n",
       " 'tell me,': 16,\n",
       " 'tell me, was': 17,\n",
       " 'the': 18,\n",
       " 'the effort,': 19,\n",
       " 'the effort, please': 20,\n",
       " 'was': 21,\n",
       " 'was it': 22,\n",
       " 'was it really,': 23,\n",
       " 'worth': 24,\n",
       " 'worth the': 25,\n",
       " 'worth the effort,': 26}"
      ]
     },
     "execution_count": 393,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cvec.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 3), preprocessor=None, stop_words=None,\n",
       "        strip_accents=None, token_pattern='[,]*\\\\w+', tokenizer=None,\n",
       "        vocabulary=None)"
      ]
     },
     "execution_count": 352,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cvec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "#[,. ]*\n",
    "#[,]*\\w{1,}[,]*\n",
    "\n",
    "# max_features=None, max_df=0.5, ngram_range=(1,2) norm='l2', use_idf=False alpha=0.01 0.852093973442 0.3763402815\n",
    "\n",
    "# token_pattern='[,]*\\w+', binary=True, ngram_range=(1,3) use_idf=False alpha=0.01 0.850663942799 0.385644965161\n",
    "# token_pattern='[,]*\\w+', binary=True, ngram_range=(1,2) use_idf=False alpha=0.01 0.857201225741 0.367171610854\n",
    "\n",
    "#token_pattern='[,]*\\w+', binary=True, max_df=0.5, ngram_range=(1,2) norm='l2', use_idf=False alpha=0.01\n",
    "# 0.857405515832 0.367096405468\n",
    "\n",
    "mnb_clf = Pipeline([\n",
    "    ('vect', CountVectorizer(max_features=None, max_df=0.5, ngram_range=(1,2))),\n",
    "    ('tfidf', TfidfTransformer(norm='l2', use_idf=False)),\n",
    "    ('clf', MultinomialNB(alpha=0.01)),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 552,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('vect', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=0.5, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 2), preprocessor=None, stop_words=None,\n",
       "        strip...lse,\n",
       "         use_idf=False)), ('clf', MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True))])"
      ]
     },
     "execution_count": 552,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnb_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 553,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score: 0.852093973442\n",
      "Log Loss: 0.3763402815\n"
     ]
    }
   ],
   "source": [
    "y_pred = mnb_clf.predict(X_test)\n",
    "y_pred_prob = mnb_clf.predict_proba(X_test)\n",
    "print(\"Accuracy score:\", accuracy_score(y_test, y_pred))\n",
    "print(\"Log Loss:\", log_loss(y_test, y_pred_prob))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 554,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV4AAAEqCAYAAABdmDjNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XeYVEXWx/Hvb0AyKAqoIEFFAUFw\nzTknjCjmiK6yxl3T+uqKiDm7iiAqKohhzaKIWQkGUEEERRRFQAGRjMCQOe8fdRuadma6menp7uk5\nH5/7zHTVvberG+d09bl1q2RmOOecy5yCbDfAOecqGw+8zjmXYR54nXMuwzzwOudchnngdc65DPPA\n65xzGeaB1znnMswDr8t5kmpKGiRpoaSXy3CeMyW9n862ZYOkdySdm+12uNLzwJthkqZIWiGpQUL5\nN5JMUovocf/o8e5x+7SUZHGPh0paJmlxFJSGS9oxxXZsJ+kFSbMl/SnpJ0kPS9oqbp//SJocnX+a\npBeTnPMMSaOi/X+PAsS+qb0zJToJ2BzYzMxOLu1JzOw5Mzs8De1Zj6QDo3+r1xLKO0TlQ1M8Tw9J\nzybbz8w6mtnTpWyuywEeeLNjMnB67EEULGsWsd884LYk57rMzOoAmwFDgWeSPbmklsAXwAzgb2ZW\nD9gHmATsG+1zLnA2cGh0/l2Bj0o451XAg8AdhCDZDHgEOD5Ze1LQHJhoZqvScK7yMhvYW9JmcWXn\nAhPT9QQK/G82H5iZbxncgClAN+CruLL7gBsAA1pEZf2BB4CZwAFRWcvwT7b2uKHABXGPdwBWpNCG\nZ4FBSfbpBTyY4mvaGFgMnFzCPtUJgXlGtD0IVI/qDgSmAVcDs4DfgfOiupuBFcDK6Dn+DvQAno07\nd4vovasaPe4C/AIsInzInRlX/mnccXsDXwELo597J7y3twKfRed5H2hQzGuLtf9R4NKorEpU1h0Y\nGrfvQ8BvwJ/AaGC/qPzIhNc5Nq4dt0ftWBr9P7D23x3oA7wSd/67CR+Qyvb/674Vv/mnZ3aMBOpJ\naiOpCnAqIRgmKiT0IG9PdkJJ1YAzo3MncyjwagptPEfSvyXtGrWzOHsBNYDXS9jnBmBPYCegA7A7\n4QMoZgtCAG9CCK69JdU3s5sI78GLZlbHzJ4sqdGSagM9gY5mVpcQXL8pYr9NgcHRvpsRPuQGJ/RY\nzwDOAxoB1YBrSnpuYABwTvT7EcB4wodMvK8I78GmwPPAy5JqmNm7Ca+zQ9wxZwNdgbrA1ITzXQ20\nl9RF0n6E9+5ci6Kwy00eeLPnGcIf6WHAD8D0YvZ7DGgmqWMx9T0lLSD0ki4j9BCTaUDoSQMg6TJJ\nC6LcbF8AM3sWuJwQQIYBsyRdV8z5NgPmWMmpgDOBW8xslpnNjtp5dlz9yqh+pZm9Hb2eVim8lqKs\nAdpJqmlmv5vZ+CL2ORr4ycyeMbNVZvY/wr/DsXH79DOziWa2FHiJEDCLZWafA5tKakX4tx1QxD7P\nmtnc6DnvJ3wTSPY6+5vZ+OiYlQnnKwTOInxwPAtcbmbTkpzPZZkH3ux5htCj6kIRf6AxZrac8JX3\nVkBF7PJPM9uE0OM8BnhFUvskzz0X2DLuOXpF53gQ2Ciu/DkzOxTYBLgIuEXSEcWcr4GkqiU8Z2PW\n761NjcrWniMhcBcCdZK8jr8wsyWEbxAXAb9LGiypdQrtibWpSdzjmXG/p9qeZwgfgAdRxDcASVdL\nmhBdDF1A6OU3SNwvwW8lVZrZl4TUiggfEC7HeeDNEjObSsg/HgW8lmT3foQ/0BNKON8aM/sE+BlI\nduX+I+DEDWjrSjN7GRgHtCtilxHAMqBTCaeZQbhIFtOMv34NT9USoFbc4y3iK83sPTM7jPDh8gPQ\nN4X2xNpU3DePVD0DXAK8HfVG14pSAf8HnALUjz7sFrLuA7W49ECJaQNJlxJ6zjOAa0vfdJcpHniz\n6+/AwVEvrVhRT7AH4Y+2WJL2IlxgK+qrdbwewH6SHpDUJDq2AdAm7lxdJB0tqa6kgijV0ZYwGiKx\nfQsJF5F6S+okqZakjSR1lHRPtNv/gG6SGkbP1Z2i89qp+AbYX1IzSRsD18e1e3NJx0W53uWElMXq\nIs7xNrB9NASuqqRTCe/dW6VsEwBmNhk4gJDTTlQXWEUYAVFVUnegXlz9H0CLDRm5IGl7wsiXswip\nm2sllZgScdnngTeLzGySmY1Kcff/Ea72J+oV5WYXE3pb3czsnSTPO5FwoWsrYKykRYSr5jOAG6Pd\n/gT+A/wKLADuAS42s0+LOecDwFWEC2azCV+PLwMGRrvcBowi9Jq/Bb4m+VC54tr/AfBidK7RrB8s\nCwgXnGYQhuMdQOiBJp5jLiE1czUhVXItcIyZzSlNmxLO/amZFdWbfw94hzDEbCrhW0J8GiF2c8hc\nSV8ne54otfMscLeZjTWznwj/Zs9Iql6W1+DKl/zip3POZZb3eJ1zLsM88Oap6HbdxUVs/8l225yr\n7DzV4JxzGVbSuMu8pqo1TdXqZrsZOWmnNs2y3YSctXqNd1SKM+3XqcydO6eosealVqVec7NVS5Pu\nZ0tnv2dmR6bzuctT5Q281epSvdUp2W5GTvpkxMPZbkLOWrwsl+fpya7DD9gz7ee0VUtT+jtd9k3v\nZDeh5JRKG3idcxWBIA8nZPPA65zLXQIKSpqfqWLywOucy21Ka9o4J3jgdc7lME81OOdc5nmP1znn\nMkh4j9c55zJLfnHNOecyLg9TDfnXh3fO5ZHo4lqyLZUzhSWuRklaLql/Ql0tSY9ImhOtDjI8rk6S\n7pY0N9rukdZ9GkjaSdJoSYXRz6TzIXvgdc7lLhF6vMm21MwgzAH9VBF1jxMWIG0T/bwyrq4rYXWV\nDkB7wjzO/4C1i8y+QZgXuT7wNPBGVF4sD7zOudyWph6vmb1mZgMJE9+vO31YnPQ4oKuZzTaz1WY2\nOm6Xc4H7zWyamU0H7ieslQhwICFl+6CZLTeznoSPi4NLaosHXudcDhNUqZJ8K5s9CCuC3BylGr6V\n1Dmuvi0wNu7x2KgsVjfO1p/mcVxcfZE88DrncldsOFnyHm+DKH8b27puwLNsRVjEdSFh9enLgKcl\nxdYgrBPVxSwE6kR53sS6WH2JUx/6qAbnXG5LLYc7x8x2LeUzLAVWArdFC8sOkzSEsFr3BMKCqfGL\nktYDFpuZRWsd1ks4Xz1gUUlP6D1e51wOS9+ohhKMS1I/nnBhLaYD61byHg+0jx/lQLgAV+JK3x54\nnXO5LU2jGiRVlVQDqAJUkVQjWql5OGE17eujffYhXDR7Lzp0AHCVpCaSGhNWpu4f1Q0FVgP/lFRd\n0mVR+ccltcUDr3Mudym6cy3ZlppuhLTCdcBZ0e/dzGwlcDxwFCE/2xc4x8x+iI57DBgEfAt8BwyO\nyjCzFYShZucAC4DzgU5RebE8x+ucy21pmqvBzHoAPYqpGw/sVUydAddGW1H1Y4BdNqQtHnidc7kt\nD28Z9sDrnMthPh+vc85lnvd4nXMugyQoyL8wlX+vyDmXX7zH65xzGeY5XuecyzDv8TrnXAbJRzU4\n51zGqcADr0vRozedScf92zF73iJ2PfkOAJ656zy2a7E5AJvUrcmCRUvZ87S7ALjm/MPpcvxerF6z\nhqvveYUPR0wo9jz5rNdD/6V/vyeRRNt2O/Jo36eoUaMGAFdfcTnPDujPH/NKnPgpb1xx6YV88O7b\nNGjYkGEjvwGga5czmPTzRAAWLlzIxhtvzEefjuLVl57nkZ4PrD32++++5YPhX9CufdJVaHJaWIAi\n/1IN+fdRkiOeGTSS4y/tvV7Z2df1Y8/T7mLP0+5i4Eff8MbH4Y+p9TZbcPIRO7PzSbdz3KWP8ND1\np1BQoGLPk69mTJ9On94P88mIr/hqzLesXr2aV156AYCvR49i4cLEaU/z26lnnMP/Xn1rvbLH+z/P\nR5+O4qNPR3H0cSdw1LGdAOh8yhlry3s91o+mzVpU+KALRJE3ha2C8cBbTj77ehLzFhYWW9/5sJ15\n6d2wusgxB7bn5fe+ZsXKVUydMZdJv81ht3YtUjpPvlm1ehVLly5l1apVLC0sZMstG7N69WpuuP5a\nbrvj7mw3L6P22mc/Nqlfv8g6M2PQ669wwkmn/qXu9Vde5ISTTinv5mWIkJJvFY0H3izYZ+dt+WPe\nIib9OhuAJg03ZtrM+Wvrp8+aT+NGG2ereVnTuEkT/nnF1bRp2Zxtmzem3sYbc8hhh/PoI704+uhj\n2WLLLbPdxJwx8vNPadCwEdtsu91f6t547RU6FRGQKyoPvOVE0hRJSyUtjtt6xdUfKMkkXZtwXIuo\nPHbMFEnXZf4VbJhTjtyVl98dta6giP9x1lvBqZKYP38+g996k+9+/IWfp0yncMkSnn92AANfe4WL\nLr08283LKaFX+9fg+vWoL6lZqyZtdmiXhVaVj4KCgqRbRZNLF9eONbMPi6k7F5gX/byniPpNzGyV\npL2AjyR9Y2bvlldDy6JKlQKOP7gD+5yx7mVMn7WArbZY95WySaP6/D67cuUzAYZ8/CEtWrSgYcOG\nABzX6QRuv6UHS5ctpf0OoWdXWFhI+zbbMW7CT9lsalatWrWKtwcN5P1hI/9SN/DVlzihc/70ditq\nDjeZnP+okFQLOAm4FNhOUrHrKpnZCMKSGzn7cX/wHq2YOOUPps9asLZs8NBxnHzEzlTbqCrNG29G\ny2YN+eq7KdlrZJY0bdqML7/4gsLCQsyMoUM+5rJ/Xckvv/7O9xMn8/3EydSqVatSB12A4UM/ouX2\nrWjcZKv1ytesWcOgga/SqXO+5HdBnuPNms6ExeZeJizFcU5ROynYh7Cs8phi9ukaW4XUVi0tr/YC\n8PSdXRj69NVs33xzfn73Vs7tFOZYPvmIXdZeVIuZ8MtMXn1/DGNevYE3e1/CFXe9xJo1VuJ58tFu\nu+9BpxM7s88eu7D7zu1Zs2YN51+wIYvF5peLzj+LYw7bn0k/TeRvbbbm+QH9gOJ7tSM++4QtGzeh\n+dbbZLqp5SpdgVfSZdHf/3JJ/YvZ56YofXloXFl1SU9J+lPSTElXJRxziKQfJBVKGiKpedK2WA4k\nEyVNARoAq+KK/21mfSV9CHxnZldIOh3oCTQ2s5WSWgCTCct1GDAT6GNmPZM9Z0GtRla9Vf70DNJp\nzhcPZ7sJOWvxslXJd6qkDj9gT8aOGZ3W7mfVzbaxekfdlnS/+c+eOTrZKsOSTgTWAEcANc2sS0L9\ntsBAYDPC0j8fRuV3AvsCxwFbAEOALmb2rqQGwCTgAsLyQLcC+5nZniW1JZd6vJ3MbJO4ra+kpsBB\nwHPRPm8ANYCjE45tYGb1zaxNKkHXOVdBCFSgpFsqzOw1MxsIzC1ml17A/wGJ66WdA9xqZvPNbAJh\nTbYuUd2JwHgze9nMlhGWFuogqXVJbcmlwFuUswltHCRpJvALIfAWmW5wzuWfFFMNDWJpxGjboByV\npJOBFWb2dkJ5faAxMDaueCwhpUn0c22dmS0h9IDbUoJcGtVQlHOAm4FH48p2B16WtFl2muScy5TY\nxbUUzEmWaij2OaQ6wB3A4UVU14l+xg8zWgjUjaufnXBMfH2RcinwDpK0Ou7xV0ALoLeZxb+wNyX9\nDJwOrH8/pXMu72Rg1MLNwDNmNrmIusXRz3rAsrjfF8XV10s4Jr6+SDkReM2sxQbuH9+Nr3hjSZxz\nqSv/v/BDgK0kXRI9bgi8JOluM7tb0u9AB+CDqL4DYdgq0c9z1zZVqg1sG1dfpJwIvM45VySRtjvT\nJFUlxLwqQBVJNQgjqQ4BNorb9SvgKuCd6PEAoJukUcDmwIXAeVHd68C9kjoDg4HuwDgz+6GktuT6\nxTXnXCWXxhsougFLgeuAs6Lfu5nZXDObGduA1cB8M4ulGW4iXDCbCgwD7o3dGRulQTsDtwPzgT2A\n05I1xHu8zrmctQEX15Iysx6E4V7J9muR8Hg5cH60FbX/h0CJw8cSeeB1zuW2PLyK44HXOZe7lJ8r\nUHjgdc7ltIo47WMyHnidc7kt/zq8Hnidc7nNUw3OOZdBFXW+3WQ88DrncpoHXuecy7BUp32sSDzw\nOudymvd4nXMuk3wcr3POZZaAPIy7Hnidc7nMRzU451zGFfjFNeecyyB5qsE55zJKeI/XOecyLh97\nvPk37Y9zLq+kawUKSZdFS78vl9Q/rnxPSR9ImidptqSXJW0ZVy9Jd0uaG233KO5JJe0kabSkwujn\nTsna4oHXOZezpJBqSLalaAZwG/BUQnl94HHCqubNCSsE94ur7wp0Iixy2R44BvhHaJ+qAW8Az0bn\neRp4Iyovlgde51wOS97bTbXHa2avmdlAYG5C+Ttm9rKZ/WlmhUAvYJ+4Xc4F7jezaWY2Hbgf6BLV\nHUhI2T5oZsvNrCchNX1wSW3xwOucy2lS8g1oEKURYlvXMjzl/qy/PHtbYGzc47FRWaxunJlZXP24\nuPoi+cU151xOS7FHO8fMdk3Dc7UnLNF+fFxxHWBh3OOFQJ0oz5tYF6uvW9LzeOB1zuWuDI7jldQS\neAf4l5l9Ele1GKgX97gesNjMTFJiXax+UUnP5akG51zOio3jTdPFteKfR2oOfAjcambPJFSPJ1xY\ni+nAulTEeKC91u+Wt2f9VMVfeOB1zuW0NA4nqyqpBlAFqCKpRlTWBPgY6G1mjxZx6ADgKklNJDUG\nrgb6R3VDgdXAPyVVl3RZVP5xSW3xVINzLqelMdXQDbgp7vFZwM2AAdsAN0laW29mdaJfH4vqv40e\nPxGVYWYrJHWKyu4CJgCdzGxFSQ2ptIF3x1ZNeW/oA9luRk466L5h2W5Czhp6zQHZbkLOqlIet/am\ncT5eM+sB9Cim+uYSjjPg2mgrqn4MsMuGtKXYwCupVqonica+OedcWlXG+XgXE7rgqaiShrY451yC\n9Fw8yzUlBd6OGWuFc84Vo1JNhG5m72WyIc459xd5Oh9vysPJJG0Wze7zX0mbRWW7SWpafs1zzlVm\nIcebnuFkuSSlUQ2SOgAfAbOBbYGHCRNNHEeY0efscmqfc66Sq4iBNZlUe7z3A0+YWRtgeVz5O8B+\naW+Vc85FMnHnWqalOo53N6L5JxNMB7ZIX3Occy5OnuZ4Uw28y/nrRBAA2wNz0tcc55xbR3m6vHuq\nqYZBQDdJsUBt0f3NdwGvl0vLnHOOlOfjrVBSDbzXAFsBM4GahAkgfgFWAP8pn6Y55xwUSEm3iial\nVIOZzZe0F3AksDMhYH8NvG1ma8qxfc65Siy25lq+SXmSnCjAvh1tzjmXEXkYdzfoBoojJb0vaVq0\nfSDJbyt2zpWrfLyBIqXAK+ly4E3gD+COaJsJDJT0z/JrnnOussvHi2upphquA640s95xZY9IGkGY\nXLhn2lvmnKv0RBhSlm9STTVsDBQ1ac77FD2+1znnyk6iSkHyLbVT6bJo6fflkvon1B0i6QdJhZKG\nRGuwxeqqS3pK0p+SZkq6KtVji7Mh43iPL6L8ePxim3OuHKUx1TADuA14av3zqwHwGnAjsCkwCngx\nbpcewHZAc+Ag4FpJR6Z4bJFKWoHikriH3xJuoNgPGBmV7QkcANyT7Emcc640BGkbp2tmrwFI2pVw\nX0LMicB4M3s5qu8BzJHU2sx+AM4BzjOz+cB8SX2BLsC7KRxbpJJyvDcmPF4G7BFt8WWXA3eW9IKd\nc660Uoy7DSSNinv8uJk9nuJTtAXGxh6Y2RJJk4C2kv4AGsfXR793SnYssOGB18y2TLHRzjlXblIc\nLjbHzHYt5VPUIUx5G28hUDeqiz1OrEt2bLEq7SrDzrncJ5XT6sXrW8xfBwnUAxZFdbHHyxLqkh1b\nrJQDr6QWhHxGM6BafJ2ZXVLEIc45V2YZGEw2Hjh37fNJtQkLPoyPpkv4HegAfBDt0iE6psRjS3rC\nVG+gOAyYQFhp4hLC/LynAmcALVM5h3POlUa67lyTVFVSDcKq6FUk1YhmXHwdaCepc1TfHRgXd3Fs\nAGFwQX1JrYELgf5RXbJji5TqcLI7gLvM7G+EuXlPJfR8hxHuaHPOubQLoxqSbynqBiwl3BB2VvR7\nNzObDXQGbgfmEwYQnBZ33E3AJGAqIebda2bvAqRwbJFSTTW0jjvZKqBmdPWuOyHw9krxPM45l7o0\nzsVgZj0IY3KLqvuQEOeKqlsOnB9tG3RscVLt8S5hXV73d0IOA8CAzTbkCZ1zbkNU5jXXvgT2JuR5\n3wXuldSG0MX+spza5pyr5GKphnyTauD9N+vGs90E1Af+Dkwk3EDhSnDlpV354L23adCwIUNHjAFg\n/Lfj+L+rLmPJksU0bdqc3n2fpm69egwb8iG39+jGypUr2GijanS/5U72PeCgLL+C9Op2dCv2bbkZ\n8wtXcnrfrwA4pHVDLtyvBS0a1OK8fl8zYea60TgtG9bm+o7bU7t6VdaY0aXf16xYvYbWW9Sh+zGt\nqV61Cp9Pmsv9H/ycrZdU7ib++CPnnLUudThl8i90634z8+bN5a1Bb1JQUEDDho14/Il+bNm4cRZb\nmn4VcdrHZFJKNZjZj2Y2Ovp9kZmdZ2bbAyeQkdEeFdspZ5zN868MWq/s6n9exH9uuo0hn39Nx2OO\n55GeDwCw6aYNGPDCawz5/Gt69nmSyy8qMq1UoQ0eN5N/vTBuvbJJs5dw7avfMebXheuVV5G4+fg2\n3PXuRE7r+xUXP/cNq9aERU/+78jtufOdiXR+9AuablqTvbbZNGOvIdO2b9WKkV+NYeRXY/hs5Chq\n1qrFccefwBVX/ZsvR49l5Fdj6HjU0dx5+y3ZbmraKYWtokl5IvRi7AD8lI6G5LO99tmP+vXrr1c2\n6eeJ7LXPfgDsf9AhDB4U1gzdscNObLFl6LG0arMDy5ctY/ny5ZltcDkb89tC/ly2ar2yKXML+XXe\n0r/su8c29fl51hJ+mrUEgIVLV7HGYLPa1ahdvSrfTv8TgLe//YMDWjUo/8bngCEff8Q222xLs+bN\nqVdv3dj9JYVL8q53KFXiNddc+rVu05b33h7EkUcfx6CBrzJj+rS/7DP4zddp174D1atXz0ILc0Oz\nTWthZvQ8rT2b1NqID76fxTMjf6NR3erM+nPdB9KsRctpVKdyvE+vvPwCJ5+yLu3Qo/sNPP/cM9Sr\ntzHvvP9xFltWPirixbNkytrjTQtJUyQdmlDWRdKncfVLJS2W9IekfpLqRHVDJV2QjXaXxQO9HqPf\nE49y+AF7smTxYqpttN7NgPw44Xtuu+k/3PNg72LOUDlUKRA7Nd2YG9+YwIUDxnDg9g3YrcUmRX6/\nNCzzDcywFStW8PZbgzih88lry3rccjsTJ/3KqaefwWN98m9kZz6uQJETgTdFx5pZHcIqx7sRBkNX\nWNtt35oXX3+b94eNpNNJp9B8623W1s2YPo3zzzqZno8+RYutty3hLPlv1qLlfP3rQhYuXcnyVWv4\nbNI8Wm1el1l/LqdRvXU93EZ1qzN78YostjQz3n/3HTrstDObb775X+pOPfUMBr7+WhZaVX5E8jRD\nRUw1lBh4Je1Q0sa68bwZY2bTgXeAdpl+7nSaM3sWAGvWrOHBe+/inPMuBGDhggWcfUonru9+G7vv\nuXc2m5gTRv4yj5aNalO9agFVJHZutgmT5yxh7pIVFC5fRbvGIcd51I6bM3zinCy3tvy9/NILnHzq\nujTDzz+tu8Qy+K03adVqg8bx574UersVMO4mzfF+ByV+f1OS+rST1BQ4ijDr+4Ye2xXoCtCkabM0\nt6x4F//9bD7/dDjz5s5h5x224ZrrbmTJksX0f+JRAI46thOnnRXm2Xiqbx8mT57Eg/fewYP33gHA\nC68PpkHDRhlrb3m79fg27NJ8EzapuRGDLtuLvp9M5s+lq7j68O2oX2sjHjh1R376YzH/fGEci5at\n4vkvpvH0ebtgwOc/z+WzSfMAuPvdn+h+bGuqVy3g80nz+Dwqz1eFhYV8/NEH9Oz96Nqy7t2uZ+LE\nHykoKKBZs+b07NUniy0sH/l2wRBAZsXHTUmtUjmJmf1YpkZIU4AGhNuRY6oBX5vZvgn1C4HBwNVm\ntlTSUOBZM3tiQ56zw992sfeGjihLs/PWcb0/z3YTctbQaw7IdhNy1r577cbXo0elNUpu3rKdnXrf\nK0n3e/iENqPLMB9vxpXY4y1rQN1AnaJ7noFwcQ24oLh651zlkIeDGnw4mXMut3ngzV2xeTZjVpvZ\nyqy1xjmXFuHiWf5F3oo0nKwkfQhza8a2ftltjnMuXdI4H2/OyIker5m1KKKsP9Es70XVx+13YPm0\nyjmXbSIja65l3Ab1eCXVkdRB0kbl1SDnnItXkMKWCkktJL0tab6kmZJ6RUv/IGknSaMlFUY/d4o7\nTpLuljQ32u5RGfMfqa65VlvSAOBPYDTQNCrvJemGsjTAOedKksYbKB4BZgFbAjsBBwCXSKoGvAE8\nS5jy9mngjagcwtj/ToRFLtsDxwD/KMtrSvXD4k7C0hZ7s26JY4D3gZOLPMI558pIKdwuvAG3DG8N\nvGRmy8xsJmFRh7bAgYS064NmttzMehKyHAdHx50L3G9m06I7Z+8HupTldaUaeI8HLjezkax/p9r3\nwDZFH+Kcc2WXYo+3gaRRcVvXIk71EHCapFqSmgAdWRd8x9n6d5ONi8qJfo6NqxsbV1cqqV5ca0jo\noieqXZYnd865kgiomtrFtTkp3Lk2jLA0+5+EJd6fBgYSJtxamLDvQqBu9HudhPqFQB1JspJu/S1B\nqj3e0YT5EWJiT3Y+4PfdOufKTTpyvJIKgPcIc7zUJkxBUB+4G1gM1Es4pB4QW38qsb4esLi0QRdS\nD7w3AHdLepjQS75U0juEpPONpX1y55wrUQpjeFMcbbYpYVBAryiPO5cw3v8oYDzQPmGkQvuonOhn\nh7i6DnF1pZLqmmvDCVcAGwHTgRMJS77vY2a+yrBzrtwohf+SMbM5wGTgYklVJW1CuGg2FhgKrAb+\nKam6pMuiw2LLeQwArpLURFJj4GqiewxKK+UbKKLFLk8ty5M559yGSPPy7icCDwL/Rwi0Q4ArzWyF\npE7AE8BdwATCpFyxmfUfIwwi+DZ6/ERUVmopBV5JtUqqN7PCsjTCOeeKk64718zsG8LQsaLqxgC7\nFFNnwLXRlhap9ngXU/KE51WVIxVYAAAa2ElEQVTS0BbnnFtPmnu8OSPVwNsx4fFGwN8I8+X6xTXn\nXPmooEv7JJNS4DWz94oofkvSROAsQvLZOefSriIuZplMWaeFHMW62+qccy6tYqkGnxYyEk0gcSlh\neJlzzpUDUSUPe7ypjmqYzfoX1wRsAqwAzimHdjnnHKIS53gJ9zLHWwPMBj43s6LmcHDOubKroKmE\nZJIG3mii4JXA29FUas45lzGV8uKama0CegHVy785zjm3TizVkKaJ0HNGqqmGLwkTQ0wtx7Y459xf\n5OOaa6kG3l7A/dEEEaMJE+SsZWbfp7thzjkn8mcp9HipBt6Xop+PRD9jIxwU/e63DDvn0k9h+Z98\nk2rgbVOurXDOuWLkX9hNEnglPQX8y8x+zFB7nHNurXDnWv6F3mTpk3OBmploiHPOFaUy3jJcAV+S\ncy5/KC9zvKlcMCz1gm7OOVcWsVENybaUzyedJmmCpCWSJknaLyo/RNIPkgolDZHUPO6Y6pKekvSn\npJmSrirr60qlzTMlrS5pK2sjnHOuOJKSbime5zDCqsLnEZZu3x/4RVIDwurDNxIWxRwFvBh3aA9g\nO6A5cBBwraQjy/KaUhnV0BVYUJYncc650kpjouFm4BYzGxk9ng4gqSsw3sxejh73AOZIam1mPxAm\nAjvPzOYD8yX1BboA75a2IakE3kH5OBGOBNWq5uPQ7LIbfu2B2W5Czmp11aBsNyFnzZy2MO3nlEh1\nWsgGkkbFPX7czB5fdx5VAXYF3pT0M1ADGAj8G2hLWG0YADNbImkS0FbSH0Dj+Pro906lfElA8sDr\n+V3nXFalmEqYY2a7llC/OWHJspOA/QgTf71BmHmxDmG2xXgLCemIOnGPE+tKLVmXL/8uJzrnKhSl\nsKVgafTzYTP73czmAA8ARxEW862XsH89YFFUR0J9rK7USgy8ZlaQj2kG51zFkY7ZyaL87DSK/hY/\nnjAJWPR8qg1sS8j7zgd+j6+Pfh9f+leUn/NPOOfyRBhOpqRbivoBl0tqJKk+cAXwFvA60E5SZ0k1\ngO7AuOjCGoTFfLtJqi+pNXAh0L8sr8sDr3Muh4kCJd9SdCvwFTARmACMAW43s9lAZ+B2YD6wB3Ba\n3HE3AZMI0+IOA+41s1KPaIAyLHbpnHOZkK4b18xsJXBJtCXWfQi0Lua45cD50ZYWHnidczkrlmrI\nNx54nXO5q4Iu7ZOMB17nXE7zwOuccxkkUr5zrULxwOucy2nyHK9zzmVWHnZ4PfA653Kb93idcy6D\nwppr2W5F+nngdc7lrg27M63C8MDrnMtp+Rd2PfA653JYvi7v7oHXOZfT8i/seuB1zuW6PIy8Hnid\ncznNUw3OOZdh+Rd2PfA653JdHkZeX4HCOZezwmKWyf/boHNK20laJunZuLIzJE2VtETSQEmbxtVt\nKun1qG6qpDPK+ro88DrnclcKC12WIgXcm7AEUHgKqS3wGHA2YRn4QuCRhP1XRHVnAn2iY0rNUw3O\nuZyWzmtrkk4DFgCfAy2j4jOBQWY2PNrnRmCCpLrAGsJ6bO3MbDHwqaQ3CUH6utK2w3u8zrkclkqi\nQQANJI2K27r+5UxSPeAW4OqEqrbA2NgDM5tE6OFuH22rzWxi3P5jo2NKzXu8zrmclmKPd46Z7Zpk\nn1uBJ83sN61/0jrAwoR9FwJ1gdUl1JWaB94MuPziC3j/nbdp0LARn331DQB33HIT7wx+k4KCAho0\nbESvx55kyy0bs2D+fC6/+EKmTJ5E9Ro1ePiRvrRp2y7LryBzWrVsQd06dalSpQpVq1blsy9GMW7s\nWC6/9CKWLF5M8xYt6DfgOerVq5ftppaLe8/owMFtN2fuouUcftcwAK4+qhWH7bgFa8yYu3gFVz87\nhll/LmfbRnW478wOtG26Mfe99QOPf/zL2vMc0KYhN53YjioF4oURv9Lnw5+z9ZLKRKRnUIOknYBD\ngb8VUb0YSPwfqh6wiJBqKK6u1DzVkAGnn3kuLw18a72yy664mk++GMOwEaM5/MijuO/O2wD47313\nsWP7DnzyxRgeebwf1197VTaanFXvfjiEL0Z/w2dfjALg4n9cwG133MWob77luONP4L/335vlFpaf\nl7/4jXP7fLFe2WMfT+LIu4dx1D3D+ei7P/jXkdsDsKBwBTe9+h19P/plvf0LBLeevCPnPvoFh94x\nhON2acx2W9TJ2GtIO6WwJXcg0AL4VdJM4Bqgs6SvgfFAh7VPJ20DVAcmRltVSdvFnatDdEypeeDN\ngL333Y/69Tddryy+x1ZYWLj2+9SPP0xg/wMPAmD7Vq357depzPrjj8w1Ngf9NPFH9t1vfwAOPvQw\nBr7+apZbVH6+nDSPBYUr1itbvGzV2t9rVa+CRb/PXbyCcb8uZOWaNevtv1Pz+kyZvYTf5haycrUx\n6OsZHLbjFuXd9HJTEE0NWdKWgseBbYGdou1RYDBwBPAccKyk/STVJuSBXzOzRWa2BHgNuEVSbUn7\nAMcDz5TpNZXlYFc2t/W4kR1bbc0rL/6P67v1AKDtju15682BAIwe9SW//TqVGTOmZbGVmSWJYzse\nzt6778KTfR8HYIe27Xhr0JsAvPbKy0z77bdsNjEr/n10a0bcfCiddmnCA2//WOK+W2xSg98XLF37\n+PcFy9hi4xrl3cRyk44Or5kVmtnM2EZILywzs9lmNh64iBCAZxHyt5fEHX4JUDOq+x9wcXRMqWUs\n8EqaImmFpAYJ5d9IMkl7SfpTUpW4ur7FlD0a/d5W0vuS5ktaIGm0pKMy9ZrKqluPW/n2x8mcdOrp\nPPFYGDb4r6uuZcGC+Ryw1y70fbQ3O3bYiapVK08q/uNhnzHiq68Z+NY7PNanN59+MpzH+j7FY316\ns/fuu7B48SKqVauW7WZm3L2Df2Cvmz5k4OjpnLtfiw0+3iz5PjkplahbiiSwmfUws7PiHj9vZs3M\nrLaZHW9m8+Lq5plZp6iumZk9X6bXROZ7vJOB02MPJO1I+CQBmANUAXaO238/YEZC2f7A8Oj3QcAH\nhIHNjYB/An+WR8PL00mnnMagN14HQgqi16NPMmzEaPr07c/cOXNo1nzrLLcwcxo3bgxAo0aNOK7T\nCXz11Ze0at2at955n8+/HM0pp57O1ttsm+VWZs8bo6bTscOWJe4zc8Eyttyk5trHW25Sgz/+XFbe\nTSs36b5zLRdkOvA+A5wT9/hcYED0+0pgJCGwIqkRUA14MaFse2B41HPeGuhrZiui7TMz+zQjr6SM\nJv3809rf3xk8iO22bwXAwgULWLEi5Pie6f8ke+2zb95ewU+0ZMkSFi1atPb3Dz94n7Zt2zFr1iwA\n1qxZw1133MaFXS/KZjMzrkXD2mt/P2zHzZk0a3GJ+4/9dQFbN6xN001rslEVcezOjfng25nl3cxy\nIcrlzrWsy/R32JHA2ZLaEK4WngrsC9wW1Q8nBNn7o5+fRttlcWWTzWyawkC8n4FnJT0BjDCzEq9C\nRYOquwJs1bRZml9a8S7schaffTKMuXPn0G77Flx3Q3c+eO9dfv5pIgUFommz5tz3UG8AJv44gUu6\nnk9BQRVatW5Dz0cez1g7s23WH39w6kknALBq9SpOPe0MDj/iSHr1fIjHHg3vz/GdTuScLudls5nl\nque5O7NXy82oX6caI285lP++/SMH7bA52zSqzRqD6fML+c+L3wLQsG51Bv17P+rUqMqaNXD+gdtw\n6B1DWbxsFd1f+Y4Bl+xJlQLx0sjf+GlmycE6l1XEwJqMLEPJH0lTgAuAPYHawDDCHSQdCb3drQnD\nPV4FGgAPAt8RktmTgC2isnpmdl50zq0It+11jI7/FPi7ma3rThZjp513sY8/+SLZbpVSreqVJ6e8\noVpdNSjbTchZM1+8iuV//JTWMNmuw872yrvJv8S2aVx7dAo3UOSMbIxqeAY4A+jCujRDzEjCXSTt\nCL3bT6L7o3+LK4vldzGzaWZ2mZltCzQHlhRxTudcBZaPqYaMB14zm0q4yHYUYXxcfN0ywqxBxwBb\nmtkPUdUnUVl74gJvwrG/EWYRqjy3eTlXCZTDoIasy9Y43r8DB0eDkxMNB64gzB4U82lUNjOawAJJ\n9SXdLKmlpILoYtv5hF6zcy5f5GHkzUrgNbNJZjaqmOphhKFh8YmdT6Oy+N7uCkJO+EPCELLvgOWE\nFIZzLg9IabtzLadk7CqKmbUopnwVcZ9ZZvYeCZ9h0WiFxLIlhOFozrk8VvHCanJ++do5l9vyMPJ6\n4HXO5bCKeWdaMh54nXM5rQKmcJPywOucy1mxW4bzjQde51xO81SDc85lmPd4nXMuw/Iw7voKFM65\nHKawKkmyLelppOqSnpQ0VdIiSWMkdYyrP0TSD5IKJQ2R1Dzh2KeiRRlmSirzQogeeJ1zOSuN8/FW\nJUy2dQCwMXAj8JKkFtF0A69FZZsCowjzgMf0ALYjTMR1EHCtpCPL8ro81eCcy2npSDVEd7r2iCt6\nS9JkYBdgM2C8mb0MIKkHMEdS62iirnOA88xsPjBfUl/C1ATvlrY93uN1zuW0FHu8DSSNitu6lnxO\nbU5YzWY80BYYG6uLgvQkoK2k+kDj+Pro97ZleU3e43XO5bQUh5PNSXUidEkbEVYUftrMfpBUB5id\nsNtCwmrDdeIeJ9aVmgde51xOS+dwMkkFhMUYVhCWFIOw1Hviwob1gEVRXezxsoS6UvNUg3MuZ6WS\nZkg1MEfrND5JWJW8s5mtjKrGAx3i9qsNbEvI+84Hfo+vj34fX5bX5YHXOZfT0ri8ex+gDXCsmS2N\nK38daCeps6QaQHdgXNwKOAOAbtHiC62BC4H+ZXlNHnidc7ktDStQRONy/wHsBMyUtDjazjSz2UBn\n4HZgPrAHcFrc4TcRLrZNJSzUcK+ZlXpEA3iO1zmX49I0nGxqSacysw+B1sXULScsK3Z+GpoCeOB1\nzuW0irm0TzIeeJ1zOStfp4X0HK9zzmWY93idczktH3u8HnidcznNJ0J3zrkMkqAg/+KuB17nXI7z\nwOucc5nlqQbnnMswv7jmnHMZlodx1wOvcy63pbKmWkXjgdc5l7Py9c41mVm225AVkmYTZhvKBQ2A\nOdluRI7y96ZkufT+NDezhuk8oaR3Ca8xmTlmVqYFKDOp0gbeXCJpVKrLllQ2/t6UzN+fisnnanDO\nuQzzwOuccxnmgTc3PJ7tBuQwf29K5u9PBeQ5XuecyzDv8TrnXIZ54HXOuQzzwOuccxnmgTcHKR/v\nkXTOreWBNweZX/F0pSTpWElFLlPucofP1ZBDJF0IbAF8AYwys3lZbpKrQCS9CDQEbpc02cyWZ7tN\nrmg+nCxHSBoINAEWA42Ah8zscUnyHrBLJgq6jYGjgaVmtjKhvsDM1mSlce4vPNWQAyQNAmqa2W5m\ndhAwDPiHpKqVPehKOklS22y3I5dJ2hVoBhxpZn+a2UpJDSUdIOkwADNbI8n/3nOEpxqyTNJBhF7K\ngXHFbxB6v7WBhVloVk6Q1A64FXhT0pNmNjHbbcpRBtQ1syWSGgE7AwOA+UB9ScPN7CTv8eYO/wTM\nMjMbAtwEvCdpJ0kbAX2Bj82s0gZdADP7DjgH2BP4u6TtY3WSqsT9vnEWmpd1kq6UdL6ZjQZqSBoH\n9AP+CzwMHACcAewe6/m63OA93iyK5d3M7Nboa+AoYCnwHzN7ONqn0uV4Jd0BbAvMBh4E/kUIJCap\nn5n9aGaro33/BXSQdJGZrchaozNMUm1gD0Ln6SlCL/c2YCww2cw+jvYrJMw7vShLTXVF8ItrWSBp\nF2CimS1KKL8ceAhoZ2bfS9oo8SJJvpP0BmHi64+AnQiBpROh93YTMAJ4ysx+lHQNcA+wi5mNyVKT\ns0bStsBo4F4zu72Yff4BXAEcambTM9k+Vzzv8WaYpNuA/wCfSrodGGdmvwOY2cOSGgLfSDrCzIZU\npqvRkl4BmsQm9pbUCbgL2NLMPpJkQA9gQZTLvAjYrTIFXUkXAU8ABWY2SdLFQA9JI2K93Gi/fYF9\ngH8Dh3vQzS0eeDPvO2AyMA/oD4yR9DHQC1huZt0lrQY+krS/mX2avaZmjqRNgSOAhyTVMrNC4Hug\nEKgTpVw+jm7q60m4+LhvlN+sFCSdCDxCyHl/KGk48DLhfesoabSZLYzSEB2j/Q4ys2+z1mhXJE81\nZIikGma2TNI2wBDgGuBr4CygCyGf+Q7ha+NiSZcBH5nZhGy1OVMkdSekEKYBbxI+kN4BBgK9zOye\nhP13JayxNSWzLc0uSU0J78kSQsDtSOjRbkX44O5iZp9F+9YkDFH0m3BykI9qyABJfYEzAczsF+A+\nQm5yjZndTAjCuwAnAb9LugroU0mCbn2gJnA7sClwPHA+8DnwXCzoxo9BNbNRlS3oApjZb8DlQBXg\nF+AlYBBQDZgB9JZUN9p3qQfd3OWBt5xFF4v+ZmZPxhUPAf4AmknanzD85xJgR0L+d2Dsqn2+M7P5\nQG/C2OUHgPrA4cB0Qi5302i/SpHnTiTpRkkjJLWMUjCfA28BO5hZf+Ay4FDCSsPtCf8fuRznqYZy\nJOl1oKGZ7RtXVs3MVkh6BDibcIvwDWb2VLbamQ3RXXmr4h5vBNwBHAOcB/xJ6M39j3D79OysNDSL\nJP0f4S7G24GNgA+BPoT5PO4CrjSziZKaAdsRhpOdZ2Y/ZKnJLkUeeMuJpCcIFz1axI053YZwYegy\nYDnwATDczC6pTON1Jd1JuMj4CrDCzExSY0LO+zugLnA1oRc3gtATvr0y9XqjHH9P4H2gK3AscAjQ\nBjiNcGPJ3sABsfHLlWkETEXnqYby8wzhivzhAJK2AoYD30T5yQXAGELvpbJNBTkE6A4cGwXdpoQZ\n2R4ifFgNJOTAtyTcJPC/ShhQBgOfALsTUguDCcPnBhOC8TxCWuai2PzNlfA9qrA88JYTMxtG6Kk8\nEI29HAE8Ymbdot7tUkKwOUjSJpVp8nMzex+4ELg1em8+Bx41szujbwfPAx8D3YDfzOzn7LU286Lb\noWcQguy5hA/o4UBtM7uG8I2pDbA9cCJQI0tNdaXkqYZyFl08ext4y8xOiyu/kvB18TQz+zFb7cum\not6b2NflKPWw3MzmZrWRGSRpL+DLuNTUecCVhF7vY4SLaPua2WRJTQjjdL+rrP//VGQeeDNA0j6E\nu41uMLPXol7enYTbOCvNDQBFid6bvsANwHvRjROVjqT+hA/iJwgjXm6KPoDuAn43s4cU5mxuT7gT\nrVJ9C8g3HngzRNIBhIleviFMA3mYmX2d3Vblhui9eYRwVf7VyjTZTYykvQm52wHA1oCA9wh57mpm\ndkW03weE6wJ/ix8V4ioWD7wZFM29+zxhwuqx2W5PLpF0KOFbwMGJkwdVFlHv/1HgKsLE5nUJee5N\ngSPM7INov63MbFrWGurKzANvhsVuHc52O3JR3BwNlVb04dwT6G5mr0vaCWhJuIV6qY9cyA8eeJ3L\nMZIOJOS9rzezV7LcHFcOfHYy53KMmQ2VdAHQJ7qjr1LmvfOZ93idy1Ge985fHnidy2Ge985PHnid\ncy7D/JZh55zLMA+8zjmXYR54nXMuwzzwOudchnngdc65DPPA60pN0neSesQ9niLpmiy0Y1dJJqlF\nOT5HD0nfpeE8WXmPXG7xwJtHJPWPApBJWinpF0n3SaqdoSbsRphlLClJXSQtLuf2xD/fUEm9MvV8\nzpXEbxnOPx8SFtHcCNiPML9rbeDionaWtJGZrUzHE1fGBSmdKw3v8eaf5WY208x+M7PngeeAThAm\nX4l6w0dJ+lLSCsIaZ0g6VtJoScskTZZ0u6RqsZNKaiTpDUlLJU2VdH7iEyd+jZZUT1IfSb9H550g\n6dRoEph+QO24HnqP6Jhqku6WNE3SEklfSToi4XmOlPRDdM5PCEvglImkuyT9GL2+KZLukfSXJXUk\nXSDp12i/gZIaJNSfJ+n7qG0TJV0pyf/O3Hq8x5v/lhJ6v/HuJqzi+zOwKApszwH/Iqzt1YwwL2x1\nIBZI+wPNCcvPFAL/BVoU96TRGnLvEBZkPA+YCLQirA/2OXAFYTn3baNDYmmHflHZGcA04ChgkKTd\nzGxstDDmQMLsXb0JKzI8kPrbUawlwPnAdGAHwutfDtwYt08L4CzgeKAW8DjwFHBc9JovBG4BLgdG\nA+2idq4EPM3h1jEz3/JkIwTHt+Ie705YIv3F6PGBgAGdE44bDtyYUNaJEAxF6FEasE9cfXNgNdAj\nrmwKcE30+2HAGqBNMW3tAixOKNs2OqZZQvlAwkKhEIL1RKLb3aOyblH7WpTw3gwFem3Ae3kR8HPc\n4x7R620WV7Zv9LzbRY9/Bc5OOM8VwPdFvUe+Vd7Ne7z558joolVVQk/3DUIPLN6ohMe7ALtL+r+4\nsgKgJmGZmTaEgPhlrNLMpkqaUUI7/kZYK2zCBrR9Z0Kg/z5h0eXqhFWHidoy0sziJxkZsQHPUSRJ\nJxGCZEugDlAl2uJNN7Nf4x5/QfThImkB0BR4TFKfuH2qEl6Tc2t54M0/wwnLyq8EZljRF86WJDwu\nAG4GXi5i39mULnCU5pgCQg9yN0L74y0tw3lLJGlP4AXCe3AlsICQPrhvA04Ty+PGlqt3rlgeePNP\noW34CrRfA62LO07SBEJg2Y0oqEhqBjROcs4tJbUppte7gr/2KMcQAusWZjakmPN+D3SWpLhe754l\ntCMV+xB6s7fGCiQ1L2K/JpKamtlv0ePdCe/LBDP7Q9J0YFszG1DG9rg854HXQbgg9JakqcBLwCrC\nhaHdzexaM/tR0ruEr9FdCb3PB1jXCy3KR4Sv4q9KupKQl20J1DazgYRcZw1JhxECbqGZTZT0HNBf\n0tWE4L0pITf9i5m9RrjodTXwoKRHgB0JvcxUNIjWMIs3K2pbE0lnEtIWRwCnF3H8UuBpSVcR0jCP\nAoPN7KeovgfwcJR2eJuQ6tkZaGJmd6bYRlcJ+DAXh5m9R1hy/iBCHvdL4DrCxaKYLsBkQq51EGG1\n5CklnHMN0BH4DHgWmAA8BFSL6j8nBK7/EdIZ10aHnkcY2XAP8APwFrA/MDU67lfgROBIYCwhNXBd\nii/1VEKQj9+uMrNBwL3Ag8A4woXB7kUcP4WQkhgUvQ+/RO2NveYnCCMjzo7a9gkh7TM5xfa5SsIn\nQnfOuQzzHq9zzmWYB17nnMswD7zOOZdhHnidcy7DPPA651yGeeB1zrkM88DrnHMZ5oHXOecy7P8B\nXuhMsRyi8dQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1778037fa58>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "conf_mx = confusion_matrix(y_test, y_pred)\n",
    "plot_conf_matrix(conf_mx, train.author.unique(), title=\"MNB_GS Confusion Matrix\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "# http://scikit-learn.org/stable/modules/feature_extraction.html\n",
    "\n",
    "class LemmaTokenizer(object):\n",
    "    def __init__(self):\n",
    "        self.wnl = WordNetLemmatizer()\n",
    "    def __call__(self, doc):\n",
    "        return [self.wnl.lemmatize(t) for t in word_tokenize(doc)]\n",
    "        \n",
    "class LemmaTokenizer_c(object):\n",
    "    def __init__(self):\n",
    "        self.wnl = WordNetLemmatizer()\n",
    "    def __call__(self, doc):\n",
    "        # TODO check this\n",
    "        tokenized_text = nltk.word_tokenize(text)\n",
    "        tagged_text = nltk.pos_tag(tokenized_text)\n",
    "        lemma_text = []\n",
    "        for item in tagged_text:\n",
    "            print(item[0], item[1])\n",
    "            lemma_text.append(wnl.lemmatize(item[0], pos=get_wordnet_pos(item[1])))\n",
    "        return lemma_text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnb_clf = Pipeline([\n",
    "    ('vect', CountVectorizer(tokenizer=LemmaTokenizer_c(), binary=True, ngram_range=(1, 2), token_pattern='[,]*\\w+', max_df=0.5)),\n",
    "    ('tfidf', TfidfTransformer(norm='l2', use_idf=False)),\n",
    "    ('clf', MultinomialNB(alpha=0.01)),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('vect', CountVectorizer(analyzer='word', binary=True, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=0.5, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 2), preprocessor=None, stop_words=None,\n",
       "        strip_...lse,\n",
       "         use_idf=False)), ('clf', MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True))])"
      ]
     },
     "execution_count": 334,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnb_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score: 0.863125638407\n",
      "Log Loss: 0.34708790383\n"
     ]
    }
   ],
   "source": [
    "y_pred = mnb_clf.predict(X_test)\n",
    "y_pred_prob = mnb_clf.predict_proba(X_test)\n",
    "print(\"Accuracy score:\", accuracy_score(y_test, y_pred))\n",
    "print(\"Log Loss:\", log_loss(y_test, y_pred_prob))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 581,
   "metadata": {},
   "outputs": [],
   "source": [
    "# max_features=None, max_df=0.5, ngram_range=(1,2) norm='l2', use_idf=False alpha=0.01 0.852093973442 0.3763402815\n",
    "\n",
    "# token_pattern='[,]*\\w+', binary=True, ngram_range=(1,3) use_idf=False alpha=0.01 0.850663942799 0.385644965161\n",
    "# token_pattern='[,]*\\w+', binary=True, ngram_range=(1,2) use_idf=False alpha=0.01 0.857201225741 0.367171610854\n",
    "\n",
    "#token_pattern='[,]*\\w+', binary=True, max_df=0.5, ngram_range=(1,2) norm='l2', use_idf=False alpha=0.01\n",
    "# 0.857405515832 0.367096405468\n",
    "\n",
    "#tokenizer=LemmaTokenizer(), binary=True, ngram_range=(1,2), token_pattern='[,]*\\w+' norm='l2', use_idf=False alpha=0.01\n",
    "# 0.853115423902 0.357860375266\n",
    "\n",
    "#tokenizer=LemmaTokenizer(), binary=True, ngram_range=(1,2), token_pattern='[,]*\\w+', max_df=0.5 norm='l2', use_idf=False alpha=0.01\n",
    "# 0.866394279877 0.341726684642\n",
    "\n",
    "#tokenizer=LemmaTokenizer_c(), binary=True, ngram_range=(1, 2), token_pattern='[,]*\\w+', max_df=0.5 norm='l2', use_idf=False alpha=0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 628,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_submission_file(y_pred_prob, name):\n",
    "    result = pd.DataFrame(y_pred_prob, columns=['EAP', 'HPL', 'MWS'])\n",
    "    result.insert(0, 'id', X_ID)\n",
    "    result.to_csv(name+'.csv', index=False, float_format='%.20f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 407,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnb_clf = Pipeline([\n",
    "    ('vect', CountVectorizer(binary=True, ngram_range=(1,2), token_pattern='[,]*\\w+', max_df=0.5)),\n",
    "    ('tfidf', TfidfTransformer(norm='l2', use_idf=False)),\n",
    "    ('clf', MultinomialNB(alpha=0.01)),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 408,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('vect', CountVectorizer(analyzer='word', binary=True, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=0.5, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 2), preprocessor=None, stop_words=None,\n",
       "        strip_...lse,\n",
       "         use_idf=False)), ('clf', MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True))])"
      ]
     },
     "execution_count": 408,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnb_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 409,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = mnb_clf.predict(X_test)\n",
    "y_pred_prob = mnb_clf.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 410,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score: 0.855158324821\n",
      "Log Loss: 0.361059190387\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy score:\", accuracy_score(y_test, y_pred))\n",
    "print(\"Log Loss:\", log_loss(y_test, y_pred_prob))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 643,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create_submission_file(y_pred_prob, 'MNB_Lemma_max')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "class StemTokenizer(object):\n",
    "    def __init__(self):\n",
    "        self.stemmer = EnglishStemmer()\n",
    "    def __call__(self, doc):\n",
    "        return [self.stemmer.stem(t) for t in word_tokenize(doc)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnb_clf = Pipeline([\n",
    "    ('vect', CountVectorizer(tokenizer=LemmaTokenizer()), binary=True, ngram_range=(1,2), token_pattern='[,]*\\w+', max_df=0.5)),\n",
    "    ('tfidf', TfidfTransformer(norm='l2', use_idf=False)),\n",
    "    ('clf', MultinomialNB(alpha=0.01)),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('vect', CountVectorizer(analyzer='word', binary=True, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=0.5, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 2), preprocessor=None, stop_words=None,\n",
       "        strip_...lse,\n",
       "         use_idf=False)), ('clf', MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True))])"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnb_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score: 0.863942798774\n",
      "Log Loss: 0.348606592351\n"
     ]
    }
   ],
   "source": [
    "y_pred = mnb_clf.predict(X_test)\n",
    "y_pred_prob = mnb_clf.predict_proba(X_test)\n",
    "print(\"Accuracy score:\", accuracy_score(y_test, y_pred))\n",
    "print(\"Log Loss:\", log_loss(y_test, y_pred_prob))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}