diff --git a/.ipynb_checkpoints/lab-hyper-tuning-checkpoint.ipynb b/.ipynb_checkpoints/lab-hyper-tuning-checkpoint.ipynb new file mode 100644 index 0000000..127f2d8 --- /dev/null +++ b/.ipynb_checkpoints/lab-hyper-tuning-checkpoint.ipynb @@ -0,0 +1,2162 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# LAB | Hyperparameter Tuning" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Load the data**\n", + "\n", + "Finally step in order to maximize the performance on your Spaceship Titanic model.\n", + "\n", + "The data can be found here:\n", + "\n", + "https://raw.githubusercontent.com/data-bootcamp-v4/data/main/spaceship_titanic.csv\n", + "\n", + "Metadata\n", + "\n", + "https://github.com/data-bootcamp-v4/data/blob/main/spaceship_titanic.md" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "So far we've been training and evaluating models with default values for hyperparameters.\n", + "\n", + "Today we will perform the same feature engineering as before, and then compare the best working models you got so far, but now fine tuning it's hyperparameters." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "#Libraries\n", + "import pandas as pd\n", + "import numpy as np\n", + "import time\n", + "\n", + "import scipy.stats as st\n", + "\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.preprocessing import OneHotEncoder\n", + "\n", + "\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.tree import DecisionTreeClassifier\n", + "from sklearn.ensemble import BaggingClassifier, RandomForestClassifier,AdaBoostClassifier, GradientBoostingClassifier\n", + "\n", + "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n", + "\n", + "from sklearn.metrics import accuracy_score, recall_score, f1_score, precision_score\n", + "\n", + "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
PassengerIdHomePlanetCryoSleepCabinDestinationAgeVIPRoomServiceFoodCourtShoppingMallSpaVRDeckNameTransported
00001_01EuropaFalseB/0/PTRAPPIST-1e39.0False0.00.00.00.00.0Maham OfracculyFalse
10002_01EarthFalseF/0/STRAPPIST-1e24.0False109.09.025.0549.044.0Juanna VinesTrue
20003_01EuropaFalseA/0/STRAPPIST-1e58.0True43.03576.00.06715.049.0Altark SusentFalse
30003_02EuropaFalseA/0/STRAPPIST-1e33.0False0.01283.0371.03329.0193.0Solam SusentFalse
40004_01EarthFalseF/1/STRAPPIST-1e16.0False303.070.0151.0565.02.0Willy SantantinesTrue
\n", + "
" + ], + "text/plain": [ + " PassengerId HomePlanet CryoSleep Cabin Destination Age VIP \\\n", + "0 0001_01 Europa False B/0/P TRAPPIST-1e 39.0 False \n", + "1 0002_01 Earth False F/0/S TRAPPIST-1e 24.0 False \n", + "2 0003_01 Europa False A/0/S TRAPPIST-1e 58.0 True \n", + "3 0003_02 Europa False A/0/S TRAPPIST-1e 33.0 False \n", + "4 0004_01 Earth False F/1/S TRAPPIST-1e 16.0 False \n", + "\n", + " RoomService FoodCourt ShoppingMall Spa VRDeck Name \\\n", + "0 0.0 0.0 0.0 0.0 0.0 Maham Ofracculy \n", + "1 109.0 9.0 25.0 549.0 44.0 Juanna Vines \n", + "2 43.0 3576.0 0.0 6715.0 49.0 Altark Susent \n", + "3 0.0 1283.0 371.0 3329.0 193.0 Solam Susent \n", + "4 303.0 70.0 151.0 565.0 2.0 Willy Santantines \n", + "\n", + " Transported \n", + "0 False \n", + "1 True \n", + "2 False \n", + "3 False \n", + "4 True " + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "spaceship = pd.read_csv(\"https://raw.githubusercontent.com/data-bootcamp-v4/data/main/spaceship_titanic.csv\")\n", + "spaceship.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now perform the same as before:\n", + "- Feature Scaling\n", + "- Feature Selection\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "PassengerId 0.000000\n", + "HomePlanet 2.312205\n", + "CryoSleep 2.496261\n", + "Cabin 2.289198\n", + "Destination 2.093639\n", + "Age 2.059128\n", + "VIP 2.335212\n", + "RoomService 2.082135\n", + "FoodCourt 2.105142\n", + "ShoppingMall 2.392730\n", + "Spa 2.105142\n", + "VRDeck 2.162660\n", + "Name 2.300702\n", + "Transported 0.000000\n", + "dtype: float64" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#your code here\n", + "missing_pct = spaceship.isna().mean() * 100\n", + "missing_pct" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "spaceship = spaceship.dropna(subset=[\"HomePlanet\", \"CryoSleep\", \"Cabin\", \"Destination\", \"Age\", \"VIP\", \"RoomService\", \"FoodCourt\", \"ShoppingMall\", \"Spa\", \"VRDeck\", \"Name\"])\n", + "spaceship = spaceship.drop(['PassengerId', 'Name'], axis=1)\n", + "spaceship[\"Transported\"] = np.where(spaceship[\"Transported\"] == True, 1, 0)\n", + "spaceship[\"Cabin\"] = spaceship[\"Cabin\"].str[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "features = spaceship.drop(columns=[\"Transported\"])\n", + "target = spaceship[\"Transported\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "X_train, X_test, y_train, y_test = train_test_split(features, target, test_size = 0.20, random_state=0)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/1q/2t5621cd5kz7blvfr_nw9dl80000gn/T/ipykernel_2877/1876233162.py:1: Pandas4Warning: For backward compatibility, 'str' dtypes are included by select_dtypes when 'object' dtype is specified. This behavior is deprecated and will be removed in a future version. Explicitly pass 'str' to `include` to select them, or to `exclude` to remove them and silence this warning.\n", + "See https://pandas.pydata.org/docs/user_guide/migration-3-strings.html#string-migration-select-dtypes for details on how to write code that works with pandas 2 and 3.\n", + " categ_cols = X_train.select_dtypes(include=['object'])\n" + ] + } + ], + "source": [ + "categ_cols = X_train.select_dtypes(include=['object'])" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "ohe = OneHotEncoder(sparse_output=False, drop='first') \n", + "ohe.fit(X_train[categ_cols.columns])\n", + "X_train_trans_np = ohe.transform(X_train[categ_cols.columns])\n", + "X_test_trans_np = ohe.transform(X_test[categ_cols.columns])\n", + "X_train_trans_df = pd.DataFrame(X_train_trans_np, columns=ohe.get_feature_names_out(), index=X_train.index)\n", + "X_test_trans_np = ohe.transform(X_test[categ_cols.columns])\n", + "X_test_trans_df = pd.DataFrame(X_test_trans_np, columns=ohe.get_feature_names_out(), index=X_test.index)\n", + "X_train_num = X_train.select_dtypes(include='number')\n", + "X_train_num = X_train_num[X_train_num.columns]\n", + "X_test_num = X_test.select_dtypes('number')\n", + "X_test_num = X_test_num[X_test_num.columns]\n", + "X_train_full = pd.concat([X_train_trans_df, X_train_num], axis=1)\n", + "X_test_full = pd.concat([X_test_trans_df, X_test_num], axis=1)\n", + "std_scaler = StandardScaler()\n", + "std_scaler.fit(X_train_full)\n", + "X_train_full_np = std_scaler.transform(X_train_full)\n", + "X_test_full_np = std_scaler.transform(X_test_full)\n", + "X_train_full_np_df = pd.DataFrame(X_train_full_np, columns=X_train_full.columns, index=X_train_full.index)\n", + "X_test_full_np_df = pd.DataFrame(X_test_full_np, columns=X_test_full.columns, index=X_test_full.index)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Now let's use the best model we got so far in order to see how it can improve when we fine tune it's hyperparameters." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "#your code here\n", + "gb_reg = GradientBoostingClassifier(max_depth=20,\n", + " n_estimators=100)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Evaluate your model" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
GradientBoostingClassifier(max_depth=20)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" + ], + "text/plain": [ + "GradientBoostingClassifier(max_depth=20)" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#your code here\n", + "gb_reg.fit(X_train_full, y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "accuracy 78.37\n", + "recall 0.80\n", + "F1-score 0.79\n", + "Precision_score 0.77\n" + ] + } + ], + "source": [ + "y_pred_test_gb = gb_reg.predict(X_test_full)\n", + "\n", + "print(f\"accuracy {accuracy_score(y_test, y_pred_test_gb)*100: .2f}\")\n", + "print(f\"recall {recall_score(y_test, y_pred_test_gb): .2f}\")\n", + "print(f\"F1-score {f1_score(y_test, y_pred_test_gb): .2f}\")\n", + "print(f\"Precision_score {precision_score(y_test, y_pred_test_gb): .2f}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Grid/Random Search**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For this lab we will use Grid Search." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Define hyperparameters to fine tune." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "#your code here\n", + "parameter_grid = {\"max_depth\": [3, 10],\n", + " \"n_estimators\": [10, 200]}\n", + "\n", + "gb_classif = GradientBoostingClassifier(random_state=123)\n", + "\n", + "confidence_level = 0.95\n", + "folds = 10" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Run Grid Search" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Fitting 10 folds for each of 4 candidates, totalling 40 fits\n", + "[CV 1/10; 1/4] START max_depth=3, n_estimators=10...............................\n", + "[CV 1/10; 1/4] END max_depth=3, n_estimators=10;, score=0.737 total time= 0.0s\n", + "[CV 2/10; 1/4] START max_depth=3, n_estimators=10...............................\n", + "[CV 2/10; 1/4] END max_depth=3, n_estimators=10;, score=0.777 total time= 0.0s\n", + "[CV 3/10; 1/4] START max_depth=3, n_estimators=10...............................\n", + "[CV 3/10; 1/4] END max_depth=3, n_estimators=10;, score=0.798 total time= 0.0s\n", + "[CV 4/10; 1/4] START max_depth=3, n_estimators=10...............................\n", + "[CV 4/10; 1/4] END max_depth=3, n_estimators=10;, score=0.771 total time= 0.0s\n", + "[CV 5/10; 1/4] START max_depth=3, n_estimators=10...............................\n", + "[CV 5/10; 1/4] END max_depth=3, n_estimators=10;, score=0.758 total time= 0.0s\n", + "[CV 6/10; 1/4] START max_depth=3, n_estimators=10...............................\n", + "[CV 6/10; 1/4] END max_depth=3, n_estimators=10;, score=0.775 total time= 0.0s\n", + "[CV 7/10; 1/4] START max_depth=3, n_estimators=10...............................\n", + "[CV 7/10; 1/4] END max_depth=3, n_estimators=10;, score=0.758 total time= 0.0s\n", + "[CV 8/10; 1/4] START max_depth=3, n_estimators=10...............................\n", + "[CV 8/10; 1/4] END max_depth=3, n_estimators=10;, score=0.771 total time= 0.0s\n", + "[CV 9/10; 1/4] START max_depth=3, n_estimators=10...............................\n", + "[CV 9/10; 1/4] END max_depth=3, n_estimators=10;, score=0.771 total time= 0.0s\n", + "[CV 10/10; 1/4] START max_depth=3, n_estimators=10..............................\n", + "[CV 10/10; 1/4] END max_depth=3, n_estimators=10;, score=0.765 total time= 0.0s\n", + "[CV 1/10; 2/4] START max_depth=3, n_estimators=200..............................\n", + "[CV 1/10; 2/4] END max_depth=3, n_estimators=200;, score=0.783 total time= 0.5s\n", + "[CV 2/10; 2/4] START max_depth=3, n_estimators=200..............................\n", + "[CV 2/10; 2/4] END max_depth=3, n_estimators=200;, score=0.817 total time= 0.5s\n", + "[CV 3/10; 2/4] START max_depth=3, n_estimators=200..............................\n", + "[CV 3/10; 2/4] END max_depth=3, n_estimators=200;, score=0.828 total time= 0.5s\n", + "[CV 4/10; 2/4] START max_depth=3, n_estimators=200..............................\n", + "[CV 4/10; 2/4] END max_depth=3, n_estimators=200;, score=0.805 total time= 0.5s\n", + "[CV 5/10; 2/4] START max_depth=3, n_estimators=200..............................\n", + "[CV 5/10; 2/4] END max_depth=3, n_estimators=200;, score=0.795 total time= 0.5s\n", + "[CV 6/10; 2/4] START max_depth=3, n_estimators=200..............................\n", + "[CV 6/10; 2/4] END max_depth=3, n_estimators=200;, score=0.812 total time= 0.5s\n", + "[CV 7/10; 2/4] START max_depth=3, n_estimators=200..............................\n", + "[CV 7/10; 2/4] END max_depth=3, n_estimators=200;, score=0.807 total time= 0.5s\n", + "[CV 8/10; 2/4] START max_depth=3, n_estimators=200..............................\n", + "[CV 8/10; 2/4] END max_depth=3, n_estimators=200;, score=0.778 total time= 0.5s\n", + "[CV 9/10; 2/4] START max_depth=3, n_estimators=200..............................\n", + "[CV 9/10; 2/4] END max_depth=3, n_estimators=200;, score=0.795 total time= 0.5s\n", + "[CV 10/10; 2/4] START max_depth=3, n_estimators=200.............................\n", + "[CV 10/10; 2/4] END max_depth=3, n_estimators=200;, score=0.799 total time= 0.5s\n", + "[CV 1/10; 3/4] START max_depth=10, n_estimators=10..............................\n", + "[CV 1/10; 3/4] END max_depth=10, n_estimators=10;, score=0.769 total time= 0.1s\n", + "[CV 2/10; 3/4] START max_depth=10, n_estimators=10..............................\n", + "[CV 2/10; 3/4] END max_depth=10, n_estimators=10;, score=0.802 total time= 0.1s\n", + "[CV 3/10; 3/4] START max_depth=10, n_estimators=10..............................\n", + "[CV 3/10; 3/4] END max_depth=10, n_estimators=10;, score=0.803 total time= 0.1s\n", + "[CV 4/10; 3/4] START max_depth=10, n_estimators=10..............................\n", + "[CV 4/10; 3/4] END max_depth=10, n_estimators=10;, score=0.786 total time= 0.1s\n", + "[CV 5/10; 3/4] START max_depth=10, n_estimators=10..............................\n", + "[CV 5/10; 3/4] END max_depth=10, n_estimators=10;, score=0.767 total time= 0.1s\n", + "[CV 6/10; 3/4] START max_depth=10, n_estimators=10..............................\n", + "[CV 6/10; 3/4] END max_depth=10, n_estimators=10;, score=0.801 total time= 0.1s\n", + "[CV 7/10; 3/4] START max_depth=10, n_estimators=10..............................\n", + "[CV 7/10; 3/4] END max_depth=10, n_estimators=10;, score=0.778 total time= 0.1s\n", + "[CV 8/10; 3/4] START max_depth=10, n_estimators=10..............................\n", + "[CV 8/10; 3/4] END max_depth=10, n_estimators=10;, score=0.777 total time= 0.1s\n", + "[CV 9/10; 3/4] START max_depth=10, n_estimators=10..............................\n", + "[CV 9/10; 3/4] END max_depth=10, n_estimators=10;, score=0.784 total time= 0.1s\n", + "[CV 10/10; 3/4] START max_depth=10, n_estimators=10.............................\n", + "[CV 10/10; 3/4] END max_depth=10, n_estimators=10;, score=0.773 total time= 0.1s\n", + "[CV 1/10; 4/4] START max_depth=10, n_estimators=200.............................\n", + "[CV 1/10; 4/4] END max_depth=10, n_estimators=200;, score=0.775 total time= 1.9s\n", + "[CV 2/10; 4/4] START max_depth=10, n_estimators=200.............................\n", + "[CV 2/10; 4/4] END max_depth=10, n_estimators=200;, score=0.803 total time= 1.9s\n", + "[CV 3/10; 4/4] START max_depth=10, n_estimators=200.............................\n", + "[CV 3/10; 4/4] END max_depth=10, n_estimators=200;, score=0.798 total time= 1.9s\n", + "[CV 4/10; 4/4] START max_depth=10, n_estimators=200.............................\n", + "[CV 4/10; 4/4] END max_depth=10, n_estimators=200;, score=0.792 total time= 1.9s\n", + "[CV 5/10; 4/4] START max_depth=10, n_estimators=200.............................\n", + "[CV 5/10; 4/4] END max_depth=10, n_estimators=200;, score=0.784 total time= 1.9s\n", + "[CV 6/10; 4/4] START max_depth=10, n_estimators=200.............................\n", + "[CV 6/10; 4/4] END max_depth=10, n_estimators=200;, score=0.788 total time= 1.9s\n", + "[CV 7/10; 4/4] START max_depth=10, n_estimators=200.............................\n", + "[CV 7/10; 4/4] END max_depth=10, n_estimators=200;, score=0.786 total time= 1.9s\n", + "[CV 8/10; 4/4] START max_depth=10, n_estimators=200.............................\n", + "[CV 8/10; 4/4] END max_depth=10, n_estimators=200;, score=0.773 total time= 1.9s\n", + "[CV 9/10; 4/4] START max_depth=10, n_estimators=200.............................\n", + "[CV 9/10; 4/4] END max_depth=10, n_estimators=200;, score=0.777 total time= 1.9s\n", + "[CV 10/10; 4/4] START max_depth=10, n_estimators=200............................\n", + "[CV 10/10; 4/4] END max_depth=10, n_estimators=200;, score=0.769 total time= 1.9s\n", + "\n", + "\n", + "Time taken to find the best combination of hyperparameters among the given ones: 26.0840 seconds\n", + "\n", + "\n", + "The best combination of hyperparameters has been: {'max_depth': 3, 'n_estimators': 200}\n", + "The best CV accuracy is: 0.8020\n", + "The R2 confidence interval for the best combination of hyperparameters is: ( 0.7918, 0.8020, 0.8123) \n" + ] + } + ], + "source": [ + "grid_search = GridSearchCV(gb_classif, param_grid=parameter_grid, cv=folds, verbose=10, error_score='raise')\n", + "\n", + "start_time = time.time()\n", + "grid_search.fit(X_train_full, y_train)\n", + "end_time = time.time()\n", + "\n", + "print(\"\\n\")\n", + "print(f\"Time taken to find the best combination of hyperparameters among the given ones: {end_time - start_time: .4f} seconds\")\n", + "print(\"\\n\")\n", + "\n", + "\n", + "print(f\"The best combination of hyperparameters has been: {grid_search.best_params_}\")\n", + "print(f\"The best CV accuracy is: {grid_search.best_score_: .4f}\")\n", + "\n", + "results_gs_df = pd.DataFrame(grid_search.cv_results_).sort_values(by=\"mean_test_score\", ascending=False)\n", + "\n", + "gs_mean_score = results_gs_df.iloc[0,-3]\n", + "gs_sem = results_gs_df.iloc[0,-2] / np.sqrt(folds)\n", + "\n", + "gs_tc = st.t.ppf(1-((1-confidence_level)/2), df=folds-1)\n", + "gs_lower_bound = gs_mean_score - ( gs_tc * gs_sem )\n", + "gs_upper_bound = gs_mean_score + ( gs_tc * gs_sem )\n", + "\n", + "print(f\"The R2 confidence interval for the best combination of hyperparameters is: \\\n", + " ({gs_lower_bound: .4f}, {gs_mean_score: .4f}, {gs_upper_bound: .4f}) \")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Evaluate your model" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "accuracy 78.52\n", + "accuracy_train 43.53\n", + "recall 0.82\n", + "F1-score 0.79\n", + "Precision_score 0.77\n", + "\n", + "\n" + ] + } + ], + "source": [ + "best_model = grid_search.best_estimator_\n", + "y_pred_test_df = best_model.predict(X_test_full)\n", + "y_pred_train_df = best_model.predict(X_train_full_np_df)\n", + "\n", + "print(\"\\n\")\n", + "print(f\"accuracy {accuracy_score(y_test, y_pred_test_df)*100: .2f}\")\n", + "print(f\"accuracy_train {accuracy_score(y_train, y_pred_train_df)*100: .2f}\")\n", + "\n", + "print(f\"recall {recall_score(y_test, y_pred_test_df): .2f}\")\n", + "print(f\"F1-score {f1_score(y_test, y_pred_test_df): .2f}\")\n", + "print(f\"Precision_score {precision_score(y_test, y_pred_test_df): .2f}\")\n", + "print(\"\\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Conclusion**\n", + "We gain a bit better results with \"best model\" parameters and we also can see, that model wasn't overfitted ('accuracy_train' metric is below 'accuracy')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/lab-hyper-tuning.ipynb b/lab-hyper-tuning.ipynb index 847d487..127f2d8 100644 --- a/lab-hyper-tuning.ipynb +++ b/lab-hyper-tuning.ipynb @@ -42,7 +42,23 @@ "#Libraries\n", "import pandas as pd\n", "import numpy as np\n", - "from sklearn.model_selection import train_test_split" + "import time\n", + "\n", + "import scipy.stats as st\n", + "\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.preprocessing import OneHotEncoder\n", + "\n", + "\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.tree import DecisionTreeClassifier\n", + "from sklearn.ensemble import BaggingClassifier, RandomForestClassifier,AdaBoostClassifier, GradientBoostingClassifier\n", + "\n", + "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n", + "\n", + "from sklearn.metrics import accuracy_score, recall_score, f1_score, precision_score\n", + "\n", + "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV" ] }, { @@ -221,11 +237,115 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "PassengerId 0.000000\n", + "HomePlanet 2.312205\n", + "CryoSleep 2.496261\n", + "Cabin 2.289198\n", + "Destination 2.093639\n", + "Age 2.059128\n", + "VIP 2.335212\n", + "RoomService 2.082135\n", + "FoodCourt 2.105142\n", + "ShoppingMall 2.392730\n", + "Spa 2.105142\n", + "VRDeck 2.162660\n", + "Name 2.300702\n", + "Transported 0.000000\n", + "dtype: float64" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#your code here\n", + "missing_pct = spaceship.isna().mean() * 100\n", + "missing_pct" + ] + }, + { + "cell_type": "code", + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ - "#your code here" + "spaceship = spaceship.dropna(subset=[\"HomePlanet\", \"CryoSleep\", \"Cabin\", \"Destination\", \"Age\", \"VIP\", \"RoomService\", \"FoodCourt\", \"ShoppingMall\", \"Spa\", \"VRDeck\", \"Name\"])\n", + "spaceship = spaceship.drop(['PassengerId', 'Name'], axis=1)\n", + "spaceship[\"Transported\"] = np.where(spaceship[\"Transported\"] == True, 1, 0)\n", + "spaceship[\"Cabin\"] = spaceship[\"Cabin\"].str[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "features = spaceship.drop(columns=[\"Transported\"])\n", + "target = spaceship[\"Transported\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "X_train, X_test, y_train, y_test = train_test_split(features, target, test_size = 0.20, random_state=0)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/1q/2t5621cd5kz7blvfr_nw9dl80000gn/T/ipykernel_2877/1876233162.py:1: Pandas4Warning: For backward compatibility, 'str' dtypes are included by select_dtypes when 'object' dtype is specified. This behavior is deprecated and will be removed in a future version. Explicitly pass 'str' to `include` to select them, or to `exclude` to remove them and silence this warning.\n", + "See https://pandas.pydata.org/docs/user_guide/migration-3-strings.html#string-migration-select-dtypes for details on how to write code that works with pandas 2 and 3.\n", + " categ_cols = X_train.select_dtypes(include=['object'])\n" + ] + } + ], + "source": [ + "categ_cols = X_train.select_dtypes(include=['object'])" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "ohe = OneHotEncoder(sparse_output=False, drop='first') \n", + "ohe.fit(X_train[categ_cols.columns])\n", + "X_train_trans_np = ohe.transform(X_train[categ_cols.columns])\n", + "X_test_trans_np = ohe.transform(X_test[categ_cols.columns])\n", + "X_train_trans_df = pd.DataFrame(X_train_trans_np, columns=ohe.get_feature_names_out(), index=X_train.index)\n", + "X_test_trans_np = ohe.transform(X_test[categ_cols.columns])\n", + "X_test_trans_df = pd.DataFrame(X_test_trans_np, columns=ohe.get_feature_names_out(), index=X_test.index)\n", + "X_train_num = X_train.select_dtypes(include='number')\n", + "X_train_num = X_train_num[X_train_num.columns]\n", + "X_test_num = X_test.select_dtypes('number')\n", + "X_test_num = X_test_num[X_test_num.columns]\n", + "X_train_full = pd.concat([X_train_trans_df, X_train_num], axis=1)\n", + "X_test_full = pd.concat([X_test_trans_df, X_test_num], axis=1)\n", + "std_scaler = StandardScaler()\n", + "std_scaler.fit(X_train_full)\n", + "X_train_full_np = std_scaler.transform(X_train_full)\n", + "X_test_full_np = std_scaler.transform(X_test_full)\n", + "X_train_full_np_df = pd.DataFrame(X_train_full_np, columns=X_train_full.columns, index=X_train_full.index)\n", + "X_test_full_np_df = pd.DataFrame(X_test_full_np, columns=X_test_full.columns, index=X_test_full.index)" ] }, { @@ -237,11 +357,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ - "#your code here" + "#your code here\n", + "gb_reg = GradientBoostingClassifier(max_depth=20,\n", + " n_estimators=100)" ] }, { @@ -253,11 +375,1535 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
GradientBoostingClassifier(max_depth=20)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" + ], + "text/plain": [ + "GradientBoostingClassifier(max_depth=20)" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "#your code here" + "#your code here\n", + "gb_reg.fit(X_train_full, y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "accuracy 78.37\n", + "recall 0.80\n", + "F1-score 0.79\n", + "Precision_score 0.77\n" + ] + } + ], + "source": [ + "y_pred_test_gb = gb_reg.predict(X_test_full)\n", + "\n", + "print(f\"accuracy {accuracy_score(y_test, y_pred_test_gb)*100: .2f}\")\n", + "print(f\"recall {recall_score(y_test, y_pred_test_gb): .2f}\")\n", + "print(f\"F1-score {f1_score(y_test, y_pred_test_gb): .2f}\")\n", + "print(f\"Precision_score {precision_score(y_test, y_pred_test_gb): .2f}\")" ] }, { @@ -283,11 +1929,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ - "#your code here" + "#your code here\n", + "parameter_grid = {\"max_depth\": [3, 10],\n", + " \"n_estimators\": [10, 200]}\n", + "\n", + "gb_classif = GradientBoostingClassifier(random_state=123)\n", + "\n", + "confidence_level = 0.95\n", + "folds = 10" ] }, { @@ -299,10 +1952,132 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Fitting 10 folds for each of 4 candidates, totalling 40 fits\n", + "[CV 1/10; 1/4] START max_depth=3, n_estimators=10...............................\n", + "[CV 1/10; 1/4] END max_depth=3, n_estimators=10;, score=0.737 total time= 0.0s\n", + "[CV 2/10; 1/4] START max_depth=3, n_estimators=10...............................\n", + "[CV 2/10; 1/4] END max_depth=3, n_estimators=10;, score=0.777 total time= 0.0s\n", + "[CV 3/10; 1/4] START max_depth=3, n_estimators=10...............................\n", + "[CV 3/10; 1/4] END max_depth=3, n_estimators=10;, score=0.798 total time= 0.0s\n", + "[CV 4/10; 1/4] START max_depth=3, n_estimators=10...............................\n", + "[CV 4/10; 1/4] END max_depth=3, n_estimators=10;, score=0.771 total time= 0.0s\n", + "[CV 5/10; 1/4] START max_depth=3, n_estimators=10...............................\n", + "[CV 5/10; 1/4] END max_depth=3, n_estimators=10;, score=0.758 total time= 0.0s\n", + "[CV 6/10; 1/4] START max_depth=3, n_estimators=10...............................\n", + "[CV 6/10; 1/4] END max_depth=3, n_estimators=10;, score=0.775 total time= 0.0s\n", + "[CV 7/10; 1/4] START max_depth=3, n_estimators=10...............................\n", + "[CV 7/10; 1/4] END max_depth=3, n_estimators=10;, score=0.758 total time= 0.0s\n", + "[CV 8/10; 1/4] START max_depth=3, n_estimators=10...............................\n", + "[CV 8/10; 1/4] END max_depth=3, n_estimators=10;, score=0.771 total time= 0.0s\n", + "[CV 9/10; 1/4] START max_depth=3, n_estimators=10...............................\n", + "[CV 9/10; 1/4] END max_depth=3, n_estimators=10;, score=0.771 total time= 0.0s\n", + "[CV 10/10; 1/4] START max_depth=3, n_estimators=10..............................\n", + "[CV 10/10; 1/4] END max_depth=3, n_estimators=10;, score=0.765 total time= 0.0s\n", + "[CV 1/10; 2/4] START max_depth=3, n_estimators=200..............................\n", + "[CV 1/10; 2/4] END max_depth=3, n_estimators=200;, score=0.783 total time= 0.5s\n", + "[CV 2/10; 2/4] START max_depth=3, n_estimators=200..............................\n", + "[CV 2/10; 2/4] END max_depth=3, n_estimators=200;, score=0.817 total time= 0.5s\n", + "[CV 3/10; 2/4] START max_depth=3, n_estimators=200..............................\n", + "[CV 3/10; 2/4] END max_depth=3, n_estimators=200;, score=0.828 total time= 0.5s\n", + "[CV 4/10; 2/4] START max_depth=3, n_estimators=200..............................\n", + "[CV 4/10; 2/4] END max_depth=3, n_estimators=200;, score=0.805 total time= 0.5s\n", + "[CV 5/10; 2/4] START max_depth=3, n_estimators=200..............................\n", + "[CV 5/10; 2/4] END max_depth=3, n_estimators=200;, score=0.795 total time= 0.5s\n", + "[CV 6/10; 2/4] START max_depth=3, n_estimators=200..............................\n", + "[CV 6/10; 2/4] END max_depth=3, n_estimators=200;, score=0.812 total time= 0.5s\n", + "[CV 7/10; 2/4] START max_depth=3, n_estimators=200..............................\n", + "[CV 7/10; 2/4] END max_depth=3, n_estimators=200;, score=0.807 total time= 0.5s\n", + "[CV 8/10; 2/4] START max_depth=3, n_estimators=200..............................\n", + "[CV 8/10; 2/4] END max_depth=3, n_estimators=200;, score=0.778 total time= 0.5s\n", + "[CV 9/10; 2/4] START max_depth=3, n_estimators=200..............................\n", + "[CV 9/10; 2/4] END max_depth=3, n_estimators=200;, score=0.795 total time= 0.5s\n", + "[CV 10/10; 2/4] START max_depth=3, n_estimators=200.............................\n", + "[CV 10/10; 2/4] END max_depth=3, n_estimators=200;, score=0.799 total time= 0.5s\n", + "[CV 1/10; 3/4] START max_depth=10, n_estimators=10..............................\n", + "[CV 1/10; 3/4] END max_depth=10, n_estimators=10;, score=0.769 total time= 0.1s\n", + "[CV 2/10; 3/4] START max_depth=10, n_estimators=10..............................\n", + "[CV 2/10; 3/4] END max_depth=10, n_estimators=10;, score=0.802 total time= 0.1s\n", + "[CV 3/10; 3/4] START max_depth=10, n_estimators=10..............................\n", + "[CV 3/10; 3/4] END max_depth=10, n_estimators=10;, score=0.803 total time= 0.1s\n", + "[CV 4/10; 3/4] START max_depth=10, n_estimators=10..............................\n", + "[CV 4/10; 3/4] END max_depth=10, n_estimators=10;, score=0.786 total time= 0.1s\n", + "[CV 5/10; 3/4] START max_depth=10, n_estimators=10..............................\n", + "[CV 5/10; 3/4] END max_depth=10, n_estimators=10;, score=0.767 total time= 0.1s\n", + "[CV 6/10; 3/4] START max_depth=10, n_estimators=10..............................\n", + "[CV 6/10; 3/4] END max_depth=10, n_estimators=10;, score=0.801 total time= 0.1s\n", + "[CV 7/10; 3/4] START max_depth=10, n_estimators=10..............................\n", + "[CV 7/10; 3/4] END max_depth=10, n_estimators=10;, score=0.778 total time= 0.1s\n", + "[CV 8/10; 3/4] START max_depth=10, n_estimators=10..............................\n", + "[CV 8/10; 3/4] END max_depth=10, n_estimators=10;, score=0.777 total time= 0.1s\n", + "[CV 9/10; 3/4] START max_depth=10, n_estimators=10..............................\n", + "[CV 9/10; 3/4] END max_depth=10, n_estimators=10;, score=0.784 total time= 0.1s\n", + "[CV 10/10; 3/4] START max_depth=10, n_estimators=10.............................\n", + "[CV 10/10; 3/4] END max_depth=10, n_estimators=10;, score=0.773 total time= 0.1s\n", + "[CV 1/10; 4/4] START max_depth=10, n_estimators=200.............................\n", + "[CV 1/10; 4/4] END max_depth=10, n_estimators=200;, score=0.775 total time= 1.9s\n", + "[CV 2/10; 4/4] START max_depth=10, n_estimators=200.............................\n", + "[CV 2/10; 4/4] END max_depth=10, n_estimators=200;, score=0.803 total time= 1.9s\n", + "[CV 3/10; 4/4] START max_depth=10, n_estimators=200.............................\n", + "[CV 3/10; 4/4] END max_depth=10, n_estimators=200;, score=0.798 total time= 1.9s\n", + "[CV 4/10; 4/4] START max_depth=10, n_estimators=200.............................\n", + "[CV 4/10; 4/4] END max_depth=10, n_estimators=200;, score=0.792 total time= 1.9s\n", + "[CV 5/10; 4/4] START max_depth=10, n_estimators=200.............................\n", + "[CV 5/10; 4/4] END max_depth=10, n_estimators=200;, score=0.784 total time= 1.9s\n", + "[CV 6/10; 4/4] START max_depth=10, n_estimators=200.............................\n", + "[CV 6/10; 4/4] END max_depth=10, n_estimators=200;, score=0.788 total time= 1.9s\n", + "[CV 7/10; 4/4] START max_depth=10, n_estimators=200.............................\n", + "[CV 7/10; 4/4] END max_depth=10, n_estimators=200;, score=0.786 total time= 1.9s\n", + "[CV 8/10; 4/4] START max_depth=10, n_estimators=200.............................\n", + "[CV 8/10; 4/4] END max_depth=10, n_estimators=200;, score=0.773 total time= 1.9s\n", + "[CV 9/10; 4/4] START max_depth=10, n_estimators=200.............................\n", + "[CV 9/10; 4/4] END max_depth=10, n_estimators=200;, score=0.777 total time= 1.9s\n", + "[CV 10/10; 4/4] START max_depth=10, n_estimators=200............................\n", + "[CV 10/10; 4/4] END max_depth=10, n_estimators=200;, score=0.769 total time= 1.9s\n", + "\n", + "\n", + "Time taken to find the best combination of hyperparameters among the given ones: 26.0840 seconds\n", + "\n", + "\n", + "The best combination of hyperparameters has been: {'max_depth': 3, 'n_estimators': 200}\n", + "The best CV accuracy is: 0.8020\n", + "The R2 confidence interval for the best combination of hyperparameters is: ( 0.7918, 0.8020, 0.8123) \n" + ] + } + ], + "source": [ + "grid_search = GridSearchCV(gb_classif, param_grid=parameter_grid, cv=folds, verbose=10, error_score='raise')\n", + "\n", + "start_time = time.time()\n", + "grid_search.fit(X_train_full, y_train)\n", + "end_time = time.time()\n", + "\n", + "print(\"\\n\")\n", + "print(f\"Time taken to find the best combination of hyperparameters among the given ones: {end_time - start_time: .4f} seconds\")\n", + "print(\"\\n\")\n", + "\n", + "\n", + "print(f\"The best combination of hyperparameters has been: {grid_search.best_params_}\")\n", + "print(f\"The best CV accuracy is: {grid_search.best_score_: .4f}\")\n", + "\n", + "results_gs_df = pd.DataFrame(grid_search.cv_results_).sort_values(by=\"mean_test_score\", ascending=False)\n", + "\n", + "gs_mean_score = results_gs_df.iloc[0,-3]\n", + "gs_sem = results_gs_df.iloc[0,-2] / np.sqrt(folds)\n", + "\n", + "gs_tc = st.t.ppf(1-((1-confidence_level)/2), df=folds-1)\n", + "gs_lower_bound = gs_mean_score - ( gs_tc * gs_sem )\n", + "gs_upper_bound = gs_mean_score + ( gs_tc * gs_sem )\n", + "\n", + "print(f\"The R2 confidence interval for the best combination of hyperparameters is: \\\n", + " ({gs_lower_bound: .4f}, {gs_mean_score: .4f}, {gs_upper_bound: .4f}) \")\n" + ] }, { "cell_type": "markdown", @@ -311,6 +2086,50 @@ "- Evaluate your model" ] }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "accuracy 78.52\n", + "accuracy_train 43.53\n", + "recall 0.82\n", + "F1-score 0.79\n", + "Precision_score 0.77\n", + "\n", + "\n" + ] + } + ], + "source": [ + "best_model = grid_search.best_estimator_\n", + "y_pred_test_df = best_model.predict(X_test_full)\n", + "y_pred_train_df = best_model.predict(X_train_full_np_df)\n", + "\n", + "print(\"\\n\")\n", + "print(f\"accuracy {accuracy_score(y_test, y_pred_test_df)*100: .2f}\")\n", + "print(f\"accuracy_train {accuracy_score(y_train, y_pred_train_df)*100: .2f}\")\n", + "\n", + "print(f\"recall {recall_score(y_test, y_pred_test_df): .2f}\")\n", + "print(f\"F1-score {f1_score(y_test, y_pred_test_df): .2f}\")\n", + "print(f\"Precision_score {precision_score(y_test, y_pred_test_df): .2f}\")\n", + "print(\"\\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Conclusion**\n", + "We gain a bit better results with \"best model\" parameters and we also can see, that model wasn't overfitted ('accuracy_train' metric is below 'accuracy')" + ] + }, { "cell_type": "code", "execution_count": null, @@ -321,7 +2140,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -335,9 +2154,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.9" + "version": "3.14.6" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 }