4_AutoML_Classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://automl.github.io/auto-sklearn/stable/\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the path of the spooky author dataset\n",
    "SPOOKY_PATH = 'spooky'\n",
    "\n",
    "\n",
    "def load_spooky_dataset(dataset_type):\n",
    "    filepath = os.path.join(SPOOKY_PATH, dataset_type, dataset_type + \".csv\")\n",
    "    print(filepath)\n",
    "    return pd.read_csv(filepath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spooky/train/train.csv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>author</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id26305</td>\n",
       "      <td>This process, however, afforded me no means of...</td>\n",
       "      <td>EAP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id17569</td>\n",
       "      <td>It never once occurred to me that the fumbling...</td>\n",
       "      <td>HPL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id11008</td>\n",
       "      <td>In his left hand was a gold snuff box, from wh...</td>\n",
       "      <td>EAP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id27763</td>\n",
       "      <td>How lovely is spring As we looked from Windsor...</td>\n",
       "      <td>MWS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id12958</td>\n",
       "      <td>Finding nothing else, not even gold, the Super...</td>\n",
       "      <td>HPL</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                               text author\n",
       "0  id26305  This process, however, afforded me no means of...    EAP\n",
       "1  id17569  It never once occurred to me that the fumbling...    HPL\n",
       "2  id11008  In his left hand was a gold snuff box, from wh...    EAP\n",
       "3  id27763  How lovely is spring As we looked from Windsor...    MWS\n",
       "4  id12958  Finding nothing else, not even gold, the Super...    HPL"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = load_spooky_dataset(\"train\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "train[\"author\"] = train.author.map({'EAP':0, 'HPL':1, 'MWS':2})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>author</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id26305</td>\n",
       "      <td>This process, however, afforded me no means of...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id17569</td>\n",
       "      <td>It never once occurred to me that the fumbling...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id11008</td>\n",
       "      <td>In his left hand was a gold snuff box, from wh...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id27763</td>\n",
       "      <td>How lovely is spring As we looked from Windsor...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id12958</td>\n",
       "      <td>Finding nothing else, not even gold, the Super...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                               text  author\n",
       "0  id26305  This process, however, afforded me no means of...       0\n",
       "1  id17569  It never once occurred to me that the fumbling...       1\n",
       "2  id11008  In his left hand was a gold snuff box, from wh...       0\n",
       "3  id27763  How lovely is spring As we looked from Windsor...       2\n",
       "4  id12958  Finding nothing else, not even gold, the Super...       1"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(train['text'], train['author'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "import autosklearn.classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "automl = autosklearn.classification.AutoSklearnClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((14684, 22286), (14684,))"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "count_vect = CountVectorizer()\n",
    "X_train_counts = count_vect.fit_transform(X_train)\n",
    "X_train_counts.shape, X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14684, 22286)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import  TfidfTransformer\n",
    "tf_transformer = TfidfTransformer()\n",
    "X_train_tfidf = tf_transformer.fit_transform(X_train_counts)\n",
    "X_train_tfidf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "automl.fit(X_train_tfidf, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_counts = count_vect.transform(X_test)\n",
    "X_test_tfidf = tf_transformer.transform(X_test_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = automl.predict(X_test_tfidf)\n",
    "y_pred_prob = automl.predict_proba(X_test_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy Score: 0.845352400409\n"
     ]
    }
   ],
   "source": [
    "# Accuracy\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "print(\"Accuracy Score:\", accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log Loss: 0.494008053742\n"
     ]
    }
   ],
   "source": [
    "# Log loss\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "print(\"Log Loss:\", log_loss(y_test, y_pred_prob))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0.140000, SimpleClassificationPipeline({'imputation:strategy': 'median', 'classifier:sgd:loss': 'squared_hinge', 'rescaling:__choice__': 'normalize', 'classifier:sgd:fit_intercept': 'True', 'classifier:sgd:learning_rate': 'invscaling', 'classifier:sgd:average': 'True', 'classifier:sgd:n_iter': 41, 'classifier:sgd:power_t': 0.07376212096528387, 'classifier:sgd:l1_ratio': 4.493338624232962e-06, 'one_hot_encoding:use_minimum_fraction': 'False', 'preprocessor:__choice__': 'no_preprocessing', 'classifier:__choice__': 'sgd', 'classifier:sgd:alpha': 0.00043300391715554303, 'balancing:strategy': 'weighting', 'classifier:sgd:eta0': 0.0236040895596051, 'classifier:sgd:penalty': 'elasticnet'},\n",
      "dataset_properties={\n",
      "  'multilabel': False,\n",
      "  'signed': False,\n",
      "  'target_type': 'classification',\n",
      "  'multiclass': True,\n",
      "  'sparse': True,\n",
      "  'task': 2})),\n",
      "(0.120000, SimpleClassificationPipeline({'classifier:liblinear_svc:tol': 0.00010000000000000009, 'classifier:liblinear_svc:loss': 'squared_hinge', 'classifier:liblinear_svc:dual': 'False', 'rescaling:__choice__': 'normalize', 'classifier:liblinear_svc:intercept_scaling': 1, 'classifier:liblinear_svc:penalty': 'l2', 'classifier:liblinear_svc:multi_class': 'ovr', 'one_hot_encoding:use_minimum_fraction': 'True', 'preprocessor:__choice__': 'no_preprocessing', 'one_hot_encoding:minimum_fraction': 0.010000000000000004, 'classifier:__choice__': 'liblinear_svc', 'imputation:strategy': 'median', 'classifier:liblinear_svc:C': 1.0, 'balancing:strategy': 'weighting', 'classifier:liblinear_svc:fit_intercept': 'True'},\n",
      "dataset_properties={\n",
      "  'multilabel': False,\n",
      "  'signed': False,\n",
      "  'target_type': 'classification',\n",
      "  'multiclass': True,\n",
      "  'sparse': True,\n",
      "  'task': 2})),\n",
      "(0.080000, SimpleClassificationPipeline({'classifier:liblinear_svc:tol': 4.099137522401286e-05, 'classifier:liblinear_svc:loss': 'squared_hinge', 'classifier:liblinear_svc:dual': 'False', 'rescaling:__choice__': 'normalize', 'classifier:liblinear_svc:intercept_scaling': 1, 'classifier:liblinear_svc:penalty': 'l2', 'classifier:liblinear_svc:multi_class': 'ovr', 'one_hot_encoding:use_minimum_fraction': 'True', 'preprocessor:__choice__': 'no_preprocessing', 'one_hot_encoding:minimum_fraction': 0.011235702047056394, 'classifier:__choice__': 'liblinear_svc', 'imputation:strategy': 'median', 'classifier:liblinear_svc:C': 4.800999198391194, 'balancing:strategy': 'none', 'classifier:liblinear_svc:fit_intercept': 'True'},\n",
      "dataset_properties={\n",
      "  'multilabel': False,\n",
      "  'signed': False,\n",
      "  'target_type': 'classification',\n",
      "  'multiclass': True,\n",
      "  'sparse': True,\n",
      "  'task': 2})),\n",
      "(0.080000, SimpleClassificationPipeline({'classifier:bernoulli_nb:alpha': 0.029195127081933305, 'rescaling:__choice__': 'standardize', 'one_hot_encoding:use_minimum_fraction': 'False', 'preprocessor:__choice__': 'no_preprocessing', 'classifier:__choice__': 'bernoulli_nb', 'imputation:strategy': 'most_frequent', 'classifier:bernoulli_nb:fit_prior': 'True', 'balancing:strategy': 'none'},\n",
      "dataset_properties={\n",
      "  'multilabel': False,\n",
      "  'signed': False,\n",
      "  'target_type': 'classification',\n",
      "  'multiclass': True,\n",
      "  'sparse': True,\n",
      "  'task': 2})),\n",
      "(0.080000, SimpleClassificationPipeline({'classifier:bernoulli_nb:alpha': 0.671764330315441, 'rescaling:__choice__': 'standardize', 'one_hot_encoding:use_minimum_fraction': 'True', 'preprocessor:__choice__': 'no_preprocessing', 'one_hot_encoding:minimum_fraction': 0.00017313167556765513, 'classifier:__choice__': 'bernoulli_nb', 'imputation:strategy': 'median', 'classifier:bernoulli_nb:fit_prior': 'False', 'balancing:strategy': 'weighting'},\n",
      "dataset_properties={\n",
      "  'multilabel': False,\n",
      "  'signed': False,\n",
      "  'target_type': 'classification',\n",
      "  'multiclass': True,\n",
      "  'sparse': True,\n",
      "  'task': 2})),\n",
      "(0.060000, SimpleClassificationPipeline({'imputation:strategy': 'mean', 'classifier:sgd:loss': 'modified_huber', 'rescaling:__choice__': 'normalize', 'classifier:sgd:fit_intercept': 'True', 'classifier:sgd:learning_rate': 'invscaling', 'classifier:sgd:average': 'False', 'classifier:sgd:n_iter': 65, 'classifier:sgd:power_t': 0.14709162750315702, 'classifier:sgd:epsilon': 0.021298254696745126, 'one_hot_encoding:use_minimum_fraction': 'False', 'preprocessor:__choice__': 'no_preprocessing', 'classifier:__choice__': 'sgd', 'classifier:sgd:alpha': 4.4934522753284643e-05, 'balancing:strategy': 'weighting', 'classifier:sgd:eta0': 0.05693436553536525, 'classifier:sgd:penalty': 'l2'},\n",
      "dataset_properties={\n",
      "  'multilabel': False,\n",
      "  'signed': False,\n",
      "  'target_type': 'classification',\n",
      "  'multiclass': True,\n",
      "  'sparse': True,\n",
      "  'task': 2})),\n",
      "(0.060000, SimpleClassificationPipeline({'imputation:strategy': 'median', 'classifier:sgd:loss': 'squared_hinge', 'rescaling:__choice__': 'normalize', 'classifier:sgd:fit_intercept': 'True', 'classifier:sgd:learning_rate': 'invscaling', 'classifier:sgd:average': 'True', 'classifier:sgd:n_iter': 41, 'classifier:sgd:power_t': 0.07376212096528387, 'classifier:sgd:l1_ratio': 4.493338624232962e-06, 'one_hot_encoding:use_minimum_fraction': 'True', 'preprocessor:__choice__': 'no_preprocessing', 'one_hot_encoding:minimum_fraction': 0.03247913944984338, 'classifier:__choice__': 'sgd', 'classifier:sgd:alpha': 9.51907170515723e-06, 'balancing:strategy': 'weighting', 'classifier:sgd:eta0': 0.0236040895596051, 'classifier:sgd:penalty': 'elasticnet'},\n",
      "dataset_properties={\n",
      "  'multilabel': False,\n",
      "  'signed': False,\n",
      "  'target_type': 'classification',\n",
      "  'multiclass': True,\n",
      "  'sparse': True,\n",
      "  'task': 2})),\n",
      "(0.060000, SimpleClassificationPipeline({'imputation:strategy': 'most_frequent', 'classifier:sgd:loss': 'modified_huber', 'rescaling:__choice__': 'normalize', 'classifier:sgd:fit_intercept': 'True', 'classifier:sgd:learning_rate': 'invscaling', 'classifier:sgd:average': 'False', 'classifier:sgd:n_iter': 65, 'classifier:sgd:power_t': 0.14709162750315702, 'classifier:sgd:epsilon': 0.021298254696745126, 'one_hot_encoding:use_minimum_fraction': 'True', 'preprocessor:__choice__': 'no_preprocessing', 'one_hot_encoding:minimum_fraction': 0.010000000000000004, 'classifier:__choice__': 'sgd', 'classifier:sgd:alpha': 4.4934522753284643e-05, 'balancing:strategy': 'weighting', 'classifier:sgd:eta0': 0.04587036554807559, 'classifier:sgd:penalty': 'l2'},\n",
      "dataset_properties={\n",
      "  'multilabel': False,\n",
      "  'signed': False,\n",
      "  'target_type': 'classification',\n",
      "  'multiclass': True,\n",
      "  'sparse': True,\n",
      "  'task': 2})),\n",
      "(0.040000, SimpleClassificationPipeline({'classifier:liblinear_svc:tol': 0.00010000000000000009, 'classifier:liblinear_svc:loss': 'squared_hinge', 'classifier:liblinear_svc:dual': 'False', 'rescaling:__choice__': 'normalize', 'classifier:liblinear_svc:intercept_scaling': 1, 'classifier:liblinear_svc:penalty': 'l2', 'classifier:liblinear_svc:multi_class': 'ovr', 'one_hot_encoding:use_minimum_fraction': 'False', 'preprocessor:__choice__': 'no_preprocessing', 'classifier:__choice__': 'liblinear_svc', 'imputation:strategy': 'most_frequent', 'classifier:liblinear_svc:C': 1.0, 'balancing:strategy': 'none', 'classifier:liblinear_svc:fit_intercept': 'True'},\n",
      "dataset_properties={\n",
      "  'multilabel': False,\n",
      "  'signed': False,\n",
      "  'target_type': 'classification',\n",
      "  'multiclass': True,\n",
      "  'sparse': True,\n",
      "  'task': 2})),\n",
      "(0.040000, SimpleClassificationPipeline({'imputation:strategy': 'most_frequent', 'classifier:sgd:loss': 'modified_huber', 'rescaling:__choice__': 'normalize', 'classifier:sgd:fit_intercept': 'True', 'classifier:sgd:learning_rate': 'invscaling', 'classifier:sgd:average': 'False', 'classifier:sgd:n_iter': 65, 'classifier:sgd:power_t': 0.14709162750315702, 'classifier:sgd:epsilon': 0.021298254696745126, 'one_hot_encoding:use_minimum_fraction': 'False', 'preprocessor:__choice__': 'no_preprocessing', 'classifier:__choice__': 'sgd', 'classifier:sgd:alpha': 4.4934522753284643e-05, 'balancing:strategy': 'weighting', 'classifier:sgd:eta0': 0.06460594556542994, 'classifier:sgd:penalty': 'l2'},\n",
      "dataset_properties={\n",
      "  'multilabel': False,\n",
      "  'signed': False,\n",
      "  'target_type': 'classification',\n",
      "  'multiclass': True,\n",
      "  'sparse': True,\n",
      "  'task': 2})),\n",
      "(0.040000, SimpleClassificationPipeline({'classifier:bernoulli_nb:alpha': 0.013047595338774143, 'rescaling:__choice__': 'none', 'one_hot_encoding:use_minimum_fraction': 'False', 'preprocessor:__choice__': 'no_preprocessing', 'classifier:__choice__': 'bernoulli_nb', 'imputation:strategy': 'most_frequent', 'classifier:bernoulli_nb:fit_prior': 'False', 'balancing:strategy': 'none'},\n",
      "dataset_properties={\n",
      "  'multilabel': False,\n",
      "  'signed': False,\n",
      "  'target_type': 'classification',\n",
      "  'multiclass': True,\n",
      "  'sparse': True,\n",
      "  'task': 2})),\n",
      "(0.040000, SimpleClassificationPipeline({'imputation:strategy': 'most_frequent', 'classifier:sgd:loss': 'modified_huber', 'rescaling:__choice__': 'normalize', 'classifier:sgd:fit_intercept': 'True', 'classifier:sgd:learning_rate': 'invscaling', 'classifier:sgd:average': 'False', 'classifier:sgd:n_iter': 65, 'classifier:sgd:power_t': 0.14709162750315702, 'classifier:sgd:epsilon': 0.021298254696745126, 'one_hot_encoding:use_minimum_fraction': 'False', 'preprocessor:__choice__': 'no_preprocessing', 'classifier:__choice__': 'sgd', 'classifier:sgd:alpha': 4.4934522753284643e-05, 'balancing:strategy': 'none', 'classifier:sgd:eta0': 0.04451507242487386, 'classifier:sgd:penalty': 'l2'},\n",
      "dataset_properties={\n",
      "  'multilabel': False,\n",
      "  'signed': False,\n",
      "  'target_type': 'classification',\n",
      "  'multiclass': True,\n",
      "  'sparse': True,\n",
      "  'task': 2})),\n",
      "(0.040000, SimpleClassificationPipeline({'rescaling:__choice__': 'normalize', 'classifier:libsvm_svc:C': 9.808344562754746, 'classifier:libsvm_svc:gamma': 0.11300006779003254, 'classifier:libsvm_svc:kernel': 'sigmoid', 'balancing:strategy': 'none', 'classifier:libsvm_svc:tol': 0.0050165278381641195, 'classifier:libsvm_svc:coef0': 0.10630145950573788, 'one_hot_encoding:use_minimum_fraction': 'True', 'preprocessor:__choice__': 'no_preprocessing', 'one_hot_encoding:minimum_fraction': 0.022079598894746907, 'classifier:libsvm_svc:shrinking': 'False', 'imputation:strategy': 'most_frequent', 'classifier:libsvm_svc:max_iter': -1, 'classifier:__choice__': 'libsvm_svc'},\n",
      "dataset_properties={\n",
      "  'multilabel': False,\n",
      "  'signed': False,\n",
      "  'target_type': 'classification',\n",
      "  'multiclass': True,\n",
      "  'sparse': True,\n",
      "  'task': 2})),\n",
      "(0.020000, SimpleClassificationPipeline({'imputation:strategy': 'most_frequent', 'classifier:sgd:loss': 'modified_huber', 'rescaling:__choice__': 'normalize', 'classifier:sgd:fit_intercept': 'True', 'classifier:sgd:learning_rate': 'invscaling', 'classifier:sgd:average': 'False', 'classifier:sgd:n_iter': 65, 'classifier:sgd:power_t': 0.14709162750315702, 'classifier:sgd:epsilon': 0.021298254696745126, 'one_hot_encoding:use_minimum_fraction': 'True', 'preprocessor:__choice__': 'no_preprocessing', 'one_hot_encoding:minimum_fraction': 0.23720046509814136, 'classifier:__choice__': 'sgd', 'classifier:sgd:alpha': 4.4934522753284643e-05, 'balancing:strategy': 'weighting', 'classifier:sgd:eta0': 0.024803832963600635, 'classifier:sgd:penalty': 'l2'},\n",
      "dataset_properties={\n",
      "  'multilabel': False,\n",
      "  'signed': False,\n",
      "  'target_type': 'classification',\n",
      "  'multiclass': True,\n",
      "  'sparse': True,\n",
      "  'task': 2})),\n",
      "(0.020000, SimpleClassificationPipeline({'preprocessor:liblinear_svc_preprocessor:loss': 'squared_hinge', 'preprocessor:liblinear_svc_preprocessor:fit_intercept': 'True', 'rescaling:__choice__': 'standardize', 'preprocessor:liblinear_svc_preprocessor:penalty': 'l1', 'preprocessor:liblinear_svc_preprocessor:C': 7254.485838948399, 'classifier:bernoulli_nb:fit_prior': 'True', 'preprocessor:liblinear_svc_preprocessor:multi_class': 'ovr', 'classifier:bernoulli_nb:alpha': 0.8409761311439689, 'preprocessor:liblinear_svc_preprocessor:intercept_scaling': 1, 'one_hot_encoding:use_minimum_fraction': 'True', 'preprocessor:__choice__': 'liblinear_svc_preprocessor', 'one_hot_encoding:minimum_fraction': 0.1317081446219844, 'classifier:__choice__': 'bernoulli_nb', 'imputation:strategy': 'most_frequent', 'balancing:strategy': 'none', 'preprocessor:liblinear_svc_preprocessor:dual': 'False', 'preprocessor:liblinear_svc_preprocessor:tol': 0.0001065557328221281},\n",
      "dataset_properties={\n",
      "  'multilabel': False,\n",
      "  'signed': False,\n",
      "  'target_type': 'classification',\n",
      "  'multiclass': True,\n",
      "  'sparse': True,\n",
      "  'task': 2})),\n",
      "(0.020000, SimpleClassificationPipeline({'imputation:strategy': 'most_frequent', 'classifier:sgd:loss': 'modified_huber', 'rescaling:__choice__': 'normalize', 'classifier:sgd:fit_intercept': 'True', 'classifier:sgd:learning_rate': 'invscaling', 'classifier:sgd:average': 'False', 'classifier:sgd:n_iter': 65, 'classifier:sgd:power_t': 0.14709162750315702, 'classifier:sgd:epsilon': 0.021298254696745126, 'one_hot_encoding:use_minimum_fraction': 'False', 'preprocessor:__choice__': 'no_preprocessing', 'classifier:__choice__': 'sgd', 'classifier:sgd:alpha': 4.4934522753284643e-05, 'balancing:strategy': 'weighting', 'classifier:sgd:eta0': 0.06078516105219284, 'classifier:sgd:penalty': 'l2'},\n",
      "dataset_properties={\n",
      "  'multilabel': False,\n",
      "  'signed': False,\n",
      "  'target_type': 'classification',\n",
      "  'multiclass': True,\n",
      "  'sparse': True,\n",
      "  'task': 2})),\n",
      "(0.020000, SimpleClassificationPipeline({'classifier:liblinear_svc:tol': 0.03653041025642362, 'classifier:liblinear_svc:loss': 'squared_hinge', 'classifier:liblinear_svc:dual': 'False', 'rescaling:__choice__': 'normalize', 'classifier:liblinear_svc:intercept_scaling': 1, 'classifier:liblinear_svc:penalty': 'l2', 'classifier:liblinear_svc:multi_class': 'ovr', 'one_hot_encoding:use_minimum_fraction': 'True', 'preprocessor:__choice__': 'no_preprocessing', 'one_hot_encoding:minimum_fraction': 0.003369853840679501, 'classifier:__choice__': 'liblinear_svc', 'imputation:strategy': 'mean', 'classifier:liblinear_svc:C': 9.437220991007523, 'balancing:strategy': 'weighting', 'classifier:liblinear_svc:fit_intercept': 'True'},\n",
      "dataset_properties={\n",
      "  'multilabel': False,\n",
      "  'signed': False,\n",
      "  'target_type': 'classification',\n",
      "  'multiclass': True,\n",
      "  'sparse': True,\n",
      "  'task': 2})),\n",
      "(0.020000, SimpleClassificationPipeline({'imputation:strategy': 'most_frequent', 'classifier:sgd:loss': 'modified_huber', 'rescaling:__choice__': 'normalize', 'classifier:sgd:fit_intercept': 'True', 'classifier:sgd:learning_rate': 'invscaling', 'classifier:sgd:average': 'False', 'classifier:sgd:n_iter': 65, 'classifier:sgd:power_t': 0.14709162750315702, 'classifier:sgd:epsilon': 0.021298254696745126, 'one_hot_encoding:use_minimum_fraction': 'False', 'preprocessor:__choice__': 'no_preprocessing', 'classifier:__choice__': 'sgd', 'classifier:sgd:alpha': 0.000505311702523002, 'balancing:strategy': 'weighting', 'classifier:sgd:eta0': 0.04399988928550367, 'classifier:sgd:penalty': 'l2'},\n",
      "dataset_properties={\n",
      "  'multilabel': False,\n",
      "  'signed': False,\n",
      "  'target_type': 'classification',\n",
      "  'multiclass': True,\n",
      "  'sparse': True,\n",
      "  'task': 2})),\n",
      "(0.020000, SimpleClassificationPipeline({'classifier:bernoulli_nb:alpha': 0.012165773179745503, 'rescaling:__choice__': 'none', 'one_hot_encoding:use_minimum_fraction': 'False', 'preprocessor:__choice__': 'no_preprocessing', 'classifier:__choice__': 'bernoulli_nb', 'imputation:strategy': 'median', 'classifier:bernoulli_nb:fit_prior': 'True', 'balancing:strategy': 'weighting'},\n",
      "dataset_properties={\n",
      "  'multilabel': False,\n",
      "  'signed': False,\n",
      "  'target_type': 'classification',\n",
      "  'multiclass': True,\n",
      "  'sparse': True,\n",
      "  'task': 2})),\n",
      "]\n"
     ]
    }
   ],
   "source": [
    "print(automl.show_models())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.84      0.86      0.85      1993\n",
      "          1       0.86      0.82      0.84      1398\n",
      "          2       0.84      0.84      0.84      1504\n",
      "\n",
      "avg / total       0.85      0.85      0.85      4895\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1719,  109,  165],\n",
       "       [ 170, 1153,   75],\n",
       "       [ 163,   75, 1266]])"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Confusion Matrix\n",
    "conf_mx = confusion_matrix(y_test, y_pred)\n",
    "conf_mx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}