diff --git a/.ipynb_checkpoints/lab-hyper-tuning-checkpoint.ipynb b/.ipynb_checkpoints/lab-hyper-tuning-checkpoint.ipynb new file mode 100644 index 0000000..368cbf6 --- /dev/null +++ b/.ipynb_checkpoints/lab-hyper-tuning-checkpoint.ipynb @@ -0,0 +1,512 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# LAB | Hyperparameter Tuning" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Load the data**\n", + "\n", + "Finally step in order to maximize the performance on your Spaceship Titanic model.\n", + "\n", + "The data can be found here:\n", + "\n", + "https://raw.githubusercontent.com/data-bootcamp-v4/data/main/spaceship_titanic.csv\n", + "\n", + "Metadata\n", + "\n", + "https://github.com/data-bootcamp-v4/data/blob/main/spaceship_titanic.md" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "So far we've been training and evaluating models with default values for hyperparameters.\n", + "\n", + "Today we will perform the same feature engineering as before, and then compare the best working models you got so far, but now fine tuning it's hyperparameters." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "#Libraries\n", + "from sklearn.datasets import fetch_california_housing\n", + "import pandas as pd\n", + "import numpy as np\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.tree import DecisionTreeRegressor\n", + "from sklearn.ensemble import BaggingRegressor, RandomForestRegressor,AdaBoostRegressor, GradientBoostingRegressor\n", + "\n", + "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n", + "from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
PassengerIdHomePlanetCryoSleepCabinDestinationAgeVIPRoomServiceFoodCourtShoppingMallSpaVRDeckNameTransported
00001_01EuropaFalseB/0/PTRAPPIST-1e39.0False0.00.00.00.00.0Maham OfracculyFalse
10002_01EarthFalseF/0/STRAPPIST-1e24.0False109.09.025.0549.044.0Juanna VinesTrue
20003_01EuropaFalseA/0/STRAPPIST-1e58.0True43.03576.00.06715.049.0Altark SusentFalse
30003_02EuropaFalseA/0/STRAPPIST-1e33.0False0.01283.0371.03329.0193.0Solam SusentFalse
40004_01EarthFalseF/1/STRAPPIST-1e16.0False303.070.0151.0565.02.0Willy SantantinesTrue
\n", + "
" + ], + "text/plain": [ + " PassengerId HomePlanet CryoSleep Cabin Destination Age VIP \\\n", + "0 0001_01 Europa False B/0/P TRAPPIST-1e 39.0 False \n", + "1 0002_01 Earth False F/0/S TRAPPIST-1e 24.0 False \n", + "2 0003_01 Europa False A/0/S TRAPPIST-1e 58.0 True \n", + "3 0003_02 Europa False A/0/S TRAPPIST-1e 33.0 False \n", + "4 0004_01 Earth False F/1/S TRAPPIST-1e 16.0 False \n", + "\n", + " RoomService FoodCourt ShoppingMall Spa VRDeck Name \\\n", + "0 0.0 0.0 0.0 0.0 0.0 Maham Ofracculy \n", + "1 109.0 9.0 25.0 549.0 44.0 Juanna Vines \n", + "2 43.0 3576.0 0.0 6715.0 49.0 Altark Susent \n", + "3 0.0 1283.0 371.0 3329.0 193.0 Solam Susent \n", + "4 303.0 70.0 151.0 565.0 2.0 Willy Santantines \n", + "\n", + " Transported \n", + "0 False \n", + "1 True \n", + "2 False \n", + "3 False \n", + "4 True " + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "spaceship = pd.read_csv(\"https://raw.githubusercontent.com/data-bootcamp-v4/data/main/spaceship_titanic.csv\")\n", + "spaceship.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now perform the same as before:\n", + "- Feature Scaling\n", + "- Feature Selection\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# Feature Scaling\n", + "numerical_columns = spaceship.select_dtypes(include=np.number).columns\n", + "\n", + "scaler = MinMaxScaler()\n", + "\n", + "spaceship[numerical_columns] = scaler.fit_transform(\n", + " spaceship[numerical_columns]\n", + ")\n", + "\n", + "# Feature Selection\n", + "X = spaceship.drop(columns=[\"Transported\", \"PassengerId\", \"Name\", \"Cabin\"])\n", + "y = spaceship[\"Transported\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Now let's use the best model we got so far in order to see how it can improve when we fine tune it's hyperparameters." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "\n", + "# Train-test split\n", + "X_train, X_test, y_train, y_test = train_test_split(\n", + " X,\n", + " y,\n", + " test_size=0.2,\n", + " random_state=42\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\sfern\\AppData\\Local\\Temp\\ipykernel_2888\\1131300158.py:7: Pandas4Warning: For backward compatibility, 'str' dtypes are included by select_dtypes when 'object' dtype is specified. This behavior is deprecated and will be removed in a future version. Explicitly pass 'str' to `include` to select them, or to `exclude` to remove them and silence this warning.\n", + "See https://pandas.pydata.org/docs/user_guide/migration-3-strings.html#string-migration-select-dtypes for details on how to write code that works with pandas 2 and 3.\n", + " categorical_columns = X_train.select_dtypes(include=\"object\").columns\n" + ] + } + ], + "source": [ + "from sklearn.impute import SimpleImputer\n", + "\n", + "# Numerical columns\n", + "numerical_columns = X_train.select_dtypes(exclude=\"object\").columns\n", + "\n", + "# Categorical columns\n", + "categorical_columns = X_train.select_dtypes(include=\"object\").columns\n", + "\n", + "# Numerical imputer\n", + "num_imputer = SimpleImputer(strategy=\"mean\")\n", + "\n", + "X_train[numerical_columns] = num_imputer.fit_transform(X_train[numerical_columns])\n", + "X_test[numerical_columns] = num_imputer.transform(X_test[numerical_columns])\n", + "\n", + "# Categorical imputer\n", + "cat_imputer = SimpleImputer(strategy=\"most_frequent\")\n", + "\n", + "X_train[categorical_columns] = cat_imputer.fit_transform(X_train[categorical_columns])\n", + "X_test[categorical_columns] = cat_imputer.transform(X_test[categorical_columns])" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\sfern\\AppData\\Local\\Temp\\ipykernel_2888\\3289519680.py:5: Pandas4Warning: For backward compatibility, 'str' dtypes are included by select_dtypes when 'object' dtype is specified. This behavior is deprecated and will be removed in a future version. Explicitly pass 'str' to `include` to select them, or to `exclude` to remove them and silence this warning.\n", + "See https://pandas.pydata.org/docs/user_guide/migration-3-strings.html#string-migration-select-dtypes for details on how to write code that works with pandas 2 and 3.\n", + " categorical_columns = X_train.select_dtypes(include=[\"object\"]).columns\n" + ] + } + ], + "source": [ + "from sklearn.compose import ColumnTransformer\n", + "from sklearn.preprocessing import OneHotEncoder\n", + "\n", + "# Select categorical columns\n", + "categorical_columns = X_train.select_dtypes(include=[\"object\"]).columns\n", + "\n", + "# Create the One-Hot Encoder\n", + "preprocessor = ColumnTransformer(\n", + " transformers=[\n", + " (\"cat\", OneHotEncoder(handle_unknown=\"ignore\"), categorical_columns)\n", + " ],\n", + " remainder=\"passthrough\"\n", + ")\n", + "\n", + "# Fit on the training set and transform both datasets\n", + "X_train = preprocessor.fit_transform(X_train)\n", + "X_test = preprocessor.transform(X_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Evaluate your model" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy: 0.7832087406555491\n" + ] + } + ], + "source": [ + "from sklearn.ensemble import GradientBoostingClassifier\n", + "\n", + "# Create the Gradient Boosting model\n", + "gradient_boosting = GradientBoostingClassifier(\n", + " n_estimators=100,\n", + " learning_rate=0.1,\n", + " random_state=42\n", + ")\n", + "\n", + "# Train the model\n", + "gradient_boosting.fit(X_train, y_train)\n", + "\n", + "# Evaluate the model\n", + "print(\"Accuracy:\", gradient_boosting.score(X_test, y_test))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Grid/Random Search**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For this lab we will use Grid Search." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Define hyperparameters to fine tune." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.model_selection import GridSearchCV\n", + "\n", + "param_grid = {\n", + " \"n_estimators\": [50, 100, 200],\n", + " \"learning_rate\": [0.01, 0.1, 0.5, 1.0],\n", + " \"max_depth\": [3, 5, 7]\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Run Grid Search" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Best parameters: {'learning_rate': 0.1, 'max_depth': 3, 'n_estimators': 100}\n", + "Best CV accuracy: 0.7947943629565718\n" + ] + } + ], + "source": [ + "grid = GridSearchCV(\n", + " estimator=GradientBoostingClassifier(random_state=42),\n", + " param_grid=param_grid,\n", + " cv=2,\n", + " scoring=\"accuracy\",\n", + " n_jobs=-1\n", + ")\n", + "\n", + "grid.fit(X_train, y_train)\n", + "\n", + "print(\"Best parameters:\", grid.best_params_)\n", + "print(\"Best CV accuracy:\", grid.best_score_)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Evaluate your model" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy: 0.7832087406555491\n" + ] + } + ], + "source": [ + "best_model = grid.best_estimator_\n", + "\n", + "print(\"Accuracy:\", best_model.score(X_test, y_test))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Hyperparameter tuning did not improve the performance because the initial hyperparameters were already the best among the tested combinations." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/lab-hyper-tuning.ipynb b/lab-hyper-tuning.ipynb index 847d487..368cbf6 100644 --- a/lab-hyper-tuning.ipynb +++ b/lab-hyper-tuning.ipynb @@ -35,19 +35,29 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "#Libraries\n", + "from sklearn.datasets import fetch_california_housing\n", "import pandas as pd\n", "import numpy as np\n", - "from sklearn.model_selection import train_test_split" + "\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.tree import DecisionTreeRegressor\n", + "from sklearn.ensemble import BaggingRegressor, RandomForestRegressor,AdaBoostRegressor, GradientBoostingRegressor\n", + "\n", + "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n", + "from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -200,7 +210,7 @@ "4 True " ] }, - "execution_count": 2, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -221,11 +231,22 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ - "#your code here" + "# Feature Scaling\n", + "numerical_columns = spaceship.select_dtypes(include=np.number).columns\n", + "\n", + "scaler = MinMaxScaler()\n", + "\n", + "spaceship[numerical_columns] = scaler.fit_transform(\n", + " spaceship[numerical_columns]\n", + ")\n", + "\n", + "# Feature Selection\n", + "X = spaceship.drop(columns=[\"Transported\", \"PassengerId\", \"Name\", \"Cabin\"])\n", + "y = spaceship[\"Transported\"]" ] }, { @@ -237,11 +258,91 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ - "#your code here" + "from sklearn.model_selection import train_test_split\n", + "\n", + "# Train-test split\n", + "X_train, X_test, y_train, y_test = train_test_split(\n", + " X,\n", + " y,\n", + " test_size=0.2,\n", + " random_state=42\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\sfern\\AppData\\Local\\Temp\\ipykernel_2888\\1131300158.py:7: Pandas4Warning: For backward compatibility, 'str' dtypes are included by select_dtypes when 'object' dtype is specified. This behavior is deprecated and will be removed in a future version. Explicitly pass 'str' to `include` to select them, or to `exclude` to remove them and silence this warning.\n", + "See https://pandas.pydata.org/docs/user_guide/migration-3-strings.html#string-migration-select-dtypes for details on how to write code that works with pandas 2 and 3.\n", + " categorical_columns = X_train.select_dtypes(include=\"object\").columns\n" + ] + } + ], + "source": [ + "from sklearn.impute import SimpleImputer\n", + "\n", + "# Numerical columns\n", + "numerical_columns = X_train.select_dtypes(exclude=\"object\").columns\n", + "\n", + "# Categorical columns\n", + "categorical_columns = X_train.select_dtypes(include=\"object\").columns\n", + "\n", + "# Numerical imputer\n", + "num_imputer = SimpleImputer(strategy=\"mean\")\n", + "\n", + "X_train[numerical_columns] = num_imputer.fit_transform(X_train[numerical_columns])\n", + "X_test[numerical_columns] = num_imputer.transform(X_test[numerical_columns])\n", + "\n", + "# Categorical imputer\n", + "cat_imputer = SimpleImputer(strategy=\"most_frequent\")\n", + "\n", + "X_train[categorical_columns] = cat_imputer.fit_transform(X_train[categorical_columns])\n", + "X_test[categorical_columns] = cat_imputer.transform(X_test[categorical_columns])" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\sfern\\AppData\\Local\\Temp\\ipykernel_2888\\3289519680.py:5: Pandas4Warning: For backward compatibility, 'str' dtypes are included by select_dtypes when 'object' dtype is specified. This behavior is deprecated and will be removed in a future version. Explicitly pass 'str' to `include` to select them, or to `exclude` to remove them and silence this warning.\n", + "See https://pandas.pydata.org/docs/user_guide/migration-3-strings.html#string-migration-select-dtypes for details on how to write code that works with pandas 2 and 3.\n", + " categorical_columns = X_train.select_dtypes(include=[\"object\"]).columns\n" + ] + } + ], + "source": [ + "from sklearn.compose import ColumnTransformer\n", + "from sklearn.preprocessing import OneHotEncoder\n", + "\n", + "# Select categorical columns\n", + "categorical_columns = X_train.select_dtypes(include=[\"object\"]).columns\n", + "\n", + "# Create the One-Hot Encoder\n", + "preprocessor = ColumnTransformer(\n", + " transformers=[\n", + " (\"cat\", OneHotEncoder(handle_unknown=\"ignore\"), categorical_columns)\n", + " ],\n", + " remainder=\"passthrough\"\n", + ")\n", + "\n", + "# Fit on the training set and transform both datasets\n", + "X_train = preprocessor.fit_transform(X_train)\n", + "X_test = preprocessor.transform(X_test)" ] }, { @@ -253,11 +354,32 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy: 0.7832087406555491\n" + ] + } + ], "source": [ - "#your code here" + "from sklearn.ensemble import GradientBoostingClassifier\n", + "\n", + "# Create the Gradient Boosting model\n", + "gradient_boosting = GradientBoostingClassifier(\n", + " n_estimators=100,\n", + " learning_rate=0.1,\n", + " random_state=42\n", + ")\n", + "\n", + "# Train the model\n", + "gradient_boosting.fit(X_train, y_train)\n", + "\n", + "# Evaluate the model\n", + "print(\"Accuracy:\", gradient_boosting.score(X_test, y_test))" ] }, { @@ -283,11 +405,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ - "#your code here" + "from sklearn.model_selection import GridSearchCV\n", + "\n", + "param_grid = {\n", + " \"n_estimators\": [50, 100, 200],\n", + " \"learning_rate\": [0.01, 0.1, 0.5, 1.0],\n", + " \"max_depth\": [3, 5, 7]\n", + "}" ] }, { @@ -299,10 +427,32 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Best parameters: {'learning_rate': 0.1, 'max_depth': 3, 'n_estimators': 100}\n", + "Best CV accuracy: 0.7947943629565718\n" + ] + } + ], + "source": [ + "grid = GridSearchCV(\n", + " estimator=GradientBoostingClassifier(random_state=42),\n", + " param_grid=param_grid,\n", + " cv=2,\n", + " scoring=\"accuracy\",\n", + " n_jobs=-1\n", + ")\n", + "\n", + "grid.fit(X_train, y_train)\n", + "\n", + "print(\"Best parameters:\", grid.best_params_)\n", + "print(\"Best CV accuracy:\", grid.best_score_)" + ] }, { "cell_type": "markdown", @@ -313,15 +463,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy: 0.7832087406555491\n" + ] + } + ], + "source": [ + "best_model = grid.best_estimator_\n", + "\n", + "print(\"Accuracy:\", best_model.score(X_test, y_test))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Hyperparameter tuning did not improve the performance because the initial hyperparameters were already the best among the tested combinations." + ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -335,9 +504,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.9" + "version": "3.14.5" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 }