FS-Random-Forest.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest Feature Importance\n",
    "As a side-effect of buiding a random forest ensemble, we get a very useful estimate of feature importance. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_selection import mutual_info_classif\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import matplotlib.pyplot as plt \n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Segmentation Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seg_data = pd.read_csv('segmentation-all.csv')\n",
    "print(seg_data.shape)\n",
    "seg_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seg_data['Class'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load the data, scale it and divide into train and test sets.  \n",
    "The filters are *trained* using the training data and then a classifier is trained on the feature subset and tested on the test set. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = seg_data.pop('Class').values\n",
    "X_raw = seg_data.values\n",
    "\n",
    "X_tr_raw, X_ts_raw, y_train, y_test = train_test_split(X_raw, y, \n",
    "                                                       random_state=1, test_size=1/2)\n",
    "scaler = MinMaxScaler()\n",
    "X_train = scaler.fit_transform(X_tr_raw)\n",
    "X_test = scaler.transform(X_ts_raw)\n",
    "\n",
    "feature_names = seg_data.columns\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build the Random Forest and calculate the scores.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_trees = 1000\n",
    "RF = RandomForestClassifier(n_estimators=n_trees, max_depth=2, random_state=0)\n",
    "RF.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_scores = RF.feature_importances_\n",
    "rf_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate the I-gain scores for comparison."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "i_scores = mutual_info_classif(X_train,y_train)\n",
    "i_scores\n",
    "# The i-gain scores for the features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.DataFrame({'Mutual Info.':i_scores,'RF Score':rf_scores,'Feature':feature_names})\n",
    "df.set_index('Feature', inplace = True)\n",
    "df.sort_values('Mutual Info.', inplace = True, ascending = False)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plotting the two sets of scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = len(df.index)\n",
    "rr = range(0,n)\n",
    "fig, ax = plt.subplots(figsize=(6,5))\n",
    "ax2 = ax.twinx()\n",
    "ax.bar(df.index, df[\"RF Score\"], label='RF Score',width=.35, color = 'g')\n",
    "\n",
    "ax2.set_xticks(rr)\n",
    "ax2.plot(df.index, df[\"Mutual Info.\"], label='I-Gain', color = 'navy')\n",
    "\n",
    "ax.set_xticklabels(list(df.index), rotation = 90)\n",
    "ax.set_xlabel('Features')\n",
    "ax.set_ylabel('I-Gain')\n",
    "ax2.set_ylabel('RF Score')\n",
    "fig.legend(loc=\"upper right\", bbox_to_anchor=(1,1), bbox_transform=ax.transAxes)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import stats\n",
    "stats.spearmanr(rf_scores, i_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Penguins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "penguins_df = pd.read_csv('penguins.csv', index_col = 0)\n",
    "\n",
    "feature_names = penguins_df.columns\n",
    "print(penguins_df.shape)\n",
    "penguins_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = penguins_df.pop('species').values\n",
    "X = penguins_df.values\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, \n",
    "                                                       random_state=1, test_size=1/2)\n",
    "feature_names = penguins_df.columns\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RF = RandomForestClassifier(n_estimators=n_trees, max_depth=2, random_state=0)\n",
    "RF.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_scores = RF.feature_importances_\n",
    "rf_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "i_scores = mutual_info_classif(X_train,y_train)\n",
    "i_scores\n",
    "# The i-gain scores for the features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pen_df=pd.DataFrame({'Mutual Info.':i_scores,'RF Score':rf_scores,'Feature':feature_names})\n",
    "pen_df.set_index('Feature', inplace = True)\n",
    "pen_df.sort_values('Mutual Info.', inplace = True, ascending = False)\n",
    "pen_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = len(pen_df.index)\n",
    "rr = range(0,n)\n",
    "fig, ax = plt.subplots(figsize=(2.5,5))\n",
    "ax2 = ax.twinx()\n",
    "ax.bar(pen_df.index, pen_df[\"RF Score\"], label='RF Score',width=.35, color = 'g')\n",
    "\n",
    "ax2.set_xticks(rr)\n",
    "ax2.plot(pen_df.index, pen_df[\"Mutual Info.\"], label='I-Gain', color = 'navy')\n",
    "\n",
    "ax.set_xticklabels(list(pen_df.index), rotation = 90)\n",
    "ax.set_xlabel('Features')\n",
    "ax.set_ylabel('I-Gain')\n",
    "ax2.set_ylabel('RF Score')\n",
    "fig.legend(loc=\"upper right\", bbox_to_anchor=(1,1), bbox_transform=ax.transAxes)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats.spearmanr(rf_scores, i_scores)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}