From e4bbdbb2136648a28cfd13415ca5c345e7b2cd82 Mon Sep 17 00:00:00 2001 From: Jan Noel Vero Date: Thu, 3 Sep 2026 14:13:42 +0200 Subject: [PATCH 1/2] Solved lab --- .../lab-hyper-tuning-checkpoint.ipynb | 2645 +++++++++++++++++ .virtual_documents/lab-hyper-tuning.ipynb | 267 ++ anaconda_projects/db/project_filebrowser.db | Bin 0 -> 32768 bytes lab-hyper-tuning.ipynb | 2346 ++++++++++++++- 4 files changed, 5236 insertions(+), 22 deletions(-) create mode 100644 .ipynb_checkpoints/lab-hyper-tuning-checkpoint.ipynb create mode 100644 .virtual_documents/lab-hyper-tuning.ipynb create mode 100644 anaconda_projects/db/project_filebrowser.db diff --git a/.ipynb_checkpoints/lab-hyper-tuning-checkpoint.ipynb b/.ipynb_checkpoints/lab-hyper-tuning-checkpoint.ipynb new file mode 100644 index 0000000..e6e9053 --- /dev/null +++ b/.ipynb_checkpoints/lab-hyper-tuning-checkpoint.ipynb @@ -0,0 +1,2645 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# LAB | Hyperparameter Tuning" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Load the data**\n", + "\n", + "Finally step in order to maximize the performance on your Spaceship Titanic model.\n", + "\n", + "The data can be found here:\n", + "\n", + "https://raw.githubusercontent.com/data-bootcamp-v4/data/main/spaceship_titanic.csv\n", + "\n", + "Metadata\n", + "\n", + "https://github.com/data-bootcamp-v4/data/blob/main/spaceship_titanic.md" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "So far we've been training and evaluating models with default values for hyperparameters.\n", + "\n", + "Today we will perform the same feature engineering as before, and then compare the best working models you got so far, but now fine tuning it's hyperparameters." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "#Libraries\n", + "import pandas as pd\n", + "import numpy as np\n", + "from sklearn.model_selection import train_test_split" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
PassengerIdHomePlanetCryoSleepCabinDestinationAgeVIPRoomServiceFoodCourtShoppingMallSpaVRDeckNameTransported
00001_01EuropaFalseB/0/PTRAPPIST-1e39.0False0.00.00.00.00.0Maham OfracculyFalse
10002_01EarthFalseF/0/STRAPPIST-1e24.0False109.09.025.0549.044.0Juanna VinesTrue
20003_01EuropaFalseA/0/STRAPPIST-1e58.0True43.03576.00.06715.049.0Altark SusentFalse
30003_02EuropaFalseA/0/STRAPPIST-1e33.0False0.01283.0371.03329.0193.0Solam SusentFalse
40004_01EarthFalseF/1/STRAPPIST-1e16.0False303.070.0151.0565.02.0Willy SantantinesTrue
\n", + "
" + ], + "text/plain": [ + " PassengerId HomePlanet CryoSleep Cabin Destination Age VIP \\\n", + "0 0001_01 Europa False B/0/P TRAPPIST-1e 39.0 False \n", + "1 0002_01 Earth False F/0/S TRAPPIST-1e 24.0 False \n", + "2 0003_01 Europa False A/0/S TRAPPIST-1e 58.0 True \n", + "3 0003_02 Europa False A/0/S TRAPPIST-1e 33.0 False \n", + "4 0004_01 Earth False F/1/S TRAPPIST-1e 16.0 False \n", + "\n", + " RoomService FoodCourt ShoppingMall Spa VRDeck Name \\\n", + "0 0.0 0.0 0.0 0.0 0.0 Maham Ofracculy \n", + "1 109.0 9.0 25.0 549.0 44.0 Juanna Vines \n", + "2 43.0 3576.0 0.0 6715.0 49.0 Altark Susent \n", + "3 0.0 1283.0 371.0 3329.0 193.0 Solam Susent \n", + "4 303.0 70.0 151.0 565.0 2.0 Willy Santantines \n", + "\n", + " Transported \n", + "0 False \n", + "1 True \n", + "2 False \n", + "3 False \n", + "4 True " + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "spaceship = pd.read_csv(\"https://raw.githubusercontent.com/data-bootcamp-v4/data/main/spaceship_titanic.csv\")\n", + "df = pd.read_csv(\"https://raw.githubusercontent.com/data-bootcamp-v4/data/main/spaceship_titanic.csv\")\n", + "spaceship.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 8693 entries, 0 to 8692\n", + "Data columns (total 14 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 PassengerId 8693 non-null str \n", + " 1 HomePlanet 8492 non-null str \n", + " 2 CryoSleep 8476 non-null object \n", + " 3 Cabin 8494 non-null str \n", + " 4 Destination 8511 non-null str \n", + " 5 Age 8514 non-null float64\n", + " 6 VIP 8490 non-null object \n", + " 7 RoomService 8512 non-null float64\n", + " 8 FoodCourt 8510 non-null float64\n", + " 9 ShoppingMall 8485 non-null float64\n", + " 10 Spa 8510 non-null float64\n", + " 11 VRDeck 8505 non-null float64\n", + " 12 Name 8493 non-null str \n", + " 13 Transported 8693 non-null bool \n", + "dtypes: bool(1), float64(6), object(2), str(5)\n", + "memory usage: 891.5+ KB\n" + ] + } + ], + "source": [ + "df.info()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "PassengerId 0\n", + "HomePlanet 201\n", + "CryoSleep 217\n", + "Cabin 199\n", + "Destination 182\n", + "Age 179\n", + "VIP 203\n", + "RoomService 181\n", + "FoodCourt 183\n", + "ShoppingMall 208\n", + "Spa 183\n", + "VRDeck 188\n", + "Name 200\n", + "Transported 0\n", + "dtype: int64" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.isnull().sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
HomePlanetCryoSleepDestinationAgeVIPRoomServiceFoodCourtShoppingMallSpaVRDeckTransported
0EuropaFalseTRAPPIST-1e39.0False0.00.00.00.00.0False
1EarthFalseTRAPPIST-1e24.0False109.09.025.0549.044.0True
2EuropaFalseTRAPPIST-1e58.0True43.03576.00.06715.049.0False
3EuropaFalseTRAPPIST-1e33.0False0.01283.0371.03329.0193.0False
4EarthFalseTRAPPIST-1e16.0False303.070.0151.0565.02.0True
\n", + "
" + ], + "text/plain": [ + " HomePlanet CryoSleep Destination Age VIP RoomService FoodCourt \\\n", + "0 Europa False TRAPPIST-1e 39.0 False 0.0 0.0 \n", + "1 Earth False TRAPPIST-1e 24.0 False 109.0 9.0 \n", + "2 Europa False TRAPPIST-1e 58.0 True 43.0 3576.0 \n", + "3 Europa False TRAPPIST-1e 33.0 False 0.0 1283.0 \n", + "4 Earth False TRAPPIST-1e 16.0 False 303.0 70.0 \n", + "\n", + " ShoppingMall Spa VRDeck Transported \n", + "0 0.0 0.0 0.0 False \n", + "1 25.0 549.0 44.0 True \n", + "2 0.0 6715.0 49.0 False \n", + "3 371.0 3329.0 193.0 False \n", + "4 151.0 565.0 2.0 True " + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Remove identifier and high-cardinality text columns\n", + "df = df.drop(columns=[\"PassengerId\", \"Cabin\", \"Name\"])\n", + "\n", + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
AgeRoomServiceFoodCourtShoppingMallSpaVRDeckHomePlanet_EuropaHomePlanet_MarsCryoSleep_TrueDestination_PSO J318.5-22Destination_TRAPPIST-1eVIP_True
3600-1.996149-0.33634-0.279076-0.305732-0.267311-0.264632FalseFalseFalseFalseTrueFalse
1262-0.811703-0.33634-0.279076-0.305732-0.267311-0.264632FalseFalseTrueFalseTrueFalse
86120.442416-0.33634-0.279076-0.305732-0.267311-0.264632FalseFalseFalseTrueFalseFalse
5075-0.184643-0.33634-0.279076-0.305732-0.267311-0.264632TrueFalseTrueFalseFalseFalse
4758-1.090396-0.33634-0.279076-0.195161-0.2664274.423937FalseFalseFalseFalseTrueFalse
\n", + "
" + ], + "text/plain": [ + " Age RoomService FoodCourt ShoppingMall Spa VRDeck \\\n", + "3600 -1.996149 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", + "1262 -0.811703 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", + "8612 0.442416 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", + "5075 -0.184643 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", + "4758 -1.090396 -0.33634 -0.279076 -0.195161 -0.266427 4.423937 \n", + "\n", + " HomePlanet_Europa HomePlanet_Mars CryoSleep_True \\\n", + "3600 False False False \n", + "1262 False False True \n", + "8612 False False False \n", + "5075 True False True \n", + "4758 False False False \n", + "\n", + " Destination_PSO J318.5-22 Destination_TRAPPIST-1e VIP_True \n", + "3600 False True False \n", + "1262 False True False \n", + "8612 True False False \n", + "5075 False False False \n", + "4758 False True False " + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from sklearn.preprocessing import StandardScaler\n", + "\n", + "numerical_columns = [\n", + " \"Age\",\n", + " \"RoomService\",\n", + " \"FoodCourt\",\n", + " \"ShoppingMall\",\n", + " \"Spa\",\n", + " \"VRDeck\"\n", + "]\n", + "\n", + "scaler = StandardScaler()\n", + "\n", + "X_train_scaled = X_train.copy()\n", + "X_test_scaled = X_test.copy()\n", + "\n", + "X_train_scaled[numerical_columns] = scaler.fit_transform(\n", + " X_train[numerical_columns]\n", + ")\n", + "\n", + "X_test_scaled[numerical_columns] = scaler.transform(\n", + " X_test[numerical_columns]\n", + ")\n", + "\n", + "X_train_scaled.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "# Fill numerical missing values with the median\n", + "for column in numerical_columns:\n", + " df[column] = df[column].fillna(df[column].median())\n", + "\n", + "# Fill categorical missing values with the mode\n", + "for column in categorical_columns:\n", + " df[column] = df[column].fillna(df[column].mode()[0])" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "HomePlanet 0\n", + "CryoSleep 0\n", + "Destination 0\n", + "Age 0\n", + "VIP 0\n", + "RoomService 0\n", + "FoodCourt 0\n", + "ShoppingMall 0\n", + "Spa 0\n", + "VRDeck 0\n", + "Transported 0\n", + "dtype: int64" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.isnull().sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "X shape: (8693, 10)\n", + "y shape: (8693,)\n" + ] + } + ], + "source": [ + "X = df.drop(columns=[\"Transported\"])\n", + "y = df[\"Transported\"]\n", + "\n", + "print(\"X shape:\", X.shape)\n", + "print(\"y shape:\", y.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
AgeRoomServiceFoodCourtShoppingMallSpaVRDeckHomePlanet_EuropaHomePlanet_MarsCryoSleep_TrueDestination_PSO J318.5-22Destination_TRAPPIST-1eVIP_True
039.00.00.00.00.00.0TrueFalseFalseFalseTrueFalse
124.0109.09.025.0549.044.0FalseFalseFalseFalseTrueFalse
258.043.03576.00.06715.049.0TrueFalseFalseFalseTrueTrue
333.00.01283.0371.03329.0193.0TrueFalseFalseFalseTrueFalse
416.0303.070.0151.0565.02.0FalseFalseFalseFalseTrueFalse
\n", + "
" + ], + "text/plain": [ + " Age RoomService FoodCourt ShoppingMall Spa VRDeck \\\n", + "0 39.0 0.0 0.0 0.0 0.0 0.0 \n", + "1 24.0 109.0 9.0 25.0 549.0 44.0 \n", + "2 58.0 43.0 3576.0 0.0 6715.0 49.0 \n", + "3 33.0 0.0 1283.0 371.0 3329.0 193.0 \n", + "4 16.0 303.0 70.0 151.0 565.0 2.0 \n", + "\n", + " HomePlanet_Europa HomePlanet_Mars CryoSleep_True \\\n", + "0 True False False \n", + "1 False False False \n", + "2 True False False \n", + "3 True False False \n", + "4 False False False \n", + "\n", + " Destination_PSO J318.5-22 Destination_TRAPPIST-1e VIP_True \n", + "0 False True False \n", + "1 False True False \n", + "2 False True True \n", + "3 False True False \n", + "4 False True False " + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = pd.get_dummies(\n", + " X,\n", + " columns=[\"HomePlanet\", \"CryoSleep\", \"Destination\", \"VIP\"],\n", + " drop_first=True\n", + ")\n", + "\n", + "X.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now perform the same as before:\n", + "- Feature Scaling\n", + "- Feature Selection\n" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training set: (6954, 12)\n", + "Test set: (1739, 12)\n" + ] + } + ], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "\n", + "X_train, X_test, y_train, y_test = train_test_split(\n", + " X,\n", + " y,\n", + " test_size=0.20,\n", + " random_state=42,\n", + " stratify=y\n", + ")\n", + "\n", + "print(\"Training set:\", X_train.shape)\n", + "print(\"Test set:\", X_test.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Scaled Logistic Regression Accuracy: 0.7884\n" + ] + } + ], + "source": [ + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.metrics import accuracy_score\n", + "\n", + "log_model = LogisticRegression(\n", + " max_iter=1000,\n", + " random_state=42\n", + ")\n", + "\n", + "log_model.fit(X_train_scaled, y_train)\n", + "\n", + "y_pred = log_model.predict(X_test_scaled)\n", + "\n", + "accuracy = accuracy_score(y_test, y_pred)\n", + "\n", + "print(\"Scaled Logistic Regression Accuracy:\", round(accuracy, 4))" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/jannoelvero/Documents/Ironhack/week_7/day_4/lab-hyperparameter-tuning/.venv/lib/python3.13/site-packages/sklearn/linear_model/_logistic.py:599: ConvergenceWarning: lbfgs failed to converge after 1000 iteration(s) (status=1):\n", + "STOP: TOTAL NO. OF ITERATIONS REACHED LIMIT\n", + "\n", + "Increase the number of iterations to improve the convergence (max_iter=1000).\n", + "You might also want to scale the data as shown in:\n", + " https://scikit-learn.org/stable/modules/preprocessing.html\n", + "Please also refer to the documentation for alternative solver options:\n", + " https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n", + " n_iter_i = _check_optimize_result(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logistic Regression Accuracy: 0.7878\n", + "KNN Accuracy: 0.7608\n", + "Decision Tree Accuracy: 0.7338\n", + "Random Forest Accuracy: 0.7849\n" + ] + } + ], + "source": [ + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.neighbors import KNeighborsClassifier\n", + "from sklearn.tree import DecisionTreeClassifier\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from sklearn.metrics import accuracy_score\n", + "\n", + "models = {\n", + " \"Logistic Regression\": LogisticRegression(max_iter=1000),\n", + " \"KNN\": KNeighborsClassifier(),\n", + " \"Decision Tree\": DecisionTreeClassifier(random_state=42),\n", + " \"Random Forest\": RandomForestClassifier(random_state=42)\n", + "}\n", + "\n", + "for name, model in models.items():\n", + "\n", + " model.fit(X_train, y_train)\n", + "\n", + " y_pred = model.predict(X_test)\n", + "\n", + " accuracy = accuracy_score(y_test, y_pred)\n", + "\n", + " print(name, \"Accuracy:\", round(accuracy, 4))" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
AgeRoomServiceFoodCourtShoppingMallSpaVRDeckHomePlanet_EuropaHomePlanet_MarsCryoSleep_TrueDestination_PSO J318.5-22Destination_TRAPPIST-1eVIP_True
3600-1.996149-0.33634-0.279076-0.305732-0.267311-0.264632FalseFalseFalseFalseTrueFalse
1262-0.811703-0.33634-0.279076-0.305732-0.267311-0.264632FalseFalseTrueFalseTrueFalse
86120.442416-0.33634-0.279076-0.305732-0.267311-0.264632FalseFalseFalseTrueFalseFalse
5075-0.184643-0.33634-0.279076-0.305732-0.267311-0.264632TrueFalseTrueFalseFalseFalse
4758-1.090396-0.33634-0.279076-0.195161-0.2664274.423937FalseFalseFalseFalseTrueFalse
\n", + "
" + ], + "text/plain": [ + " Age RoomService FoodCourt ShoppingMall Spa VRDeck \\\n", + "3600 -1.996149 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", + "1262 -0.811703 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", + "8612 0.442416 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", + "5075 -0.184643 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", + "4758 -1.090396 -0.33634 -0.279076 -0.195161 -0.266427 4.423937 \n", + "\n", + " HomePlanet_Europa HomePlanet_Mars CryoSleep_True \\\n", + "3600 False False False \n", + "1262 False False True \n", + "8612 False False False \n", + "5075 True False True \n", + "4758 False False False \n", + "\n", + " Destination_PSO J318.5-22 Destination_TRAPPIST-1e VIP_True \n", + "3600 False True False \n", + "1262 False True False \n", + "8612 True False False \n", + "5075 False False False \n", + "4758 False True False " + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X_train_scaled.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Selected features:\n", + "['Age', 'RoomService', 'FoodCourt', 'Spa', 'VRDeck', 'HomePlanet_Europa', 'CryoSleep_True', 'Destination_TRAPPIST-1e']\n", + "\n", + "Training shape: (6954, 8)\n", + "Test shape: (1739, 8)\n" + ] + } + ], + "source": [ + "from sklearn.feature_selection import SelectKBest, f_classif\n", + "\n", + "selector = SelectKBest(\n", + " score_func=f_classif,\n", + " k=8\n", + ")\n", + "\n", + "X_train_selected = selector.fit_transform(\n", + " X_train_scaled,\n", + " y_train\n", + ")\n", + "\n", + "X_test_selected = selector.transform(\n", + " X_test_scaled\n", + ")\n", + "\n", + "selected_features = X_train_scaled.columns[\n", + " selector.get_support()\n", + "]\n", + "\n", + "print(\"Selected features:\")\n", + "print(selected_features.tolist())\n", + "\n", + "print(\"\\nTraining shape:\", X_train_selected.shape)\n", + "print(\"Test shape:\", X_test_selected.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Now let's use the best model we got so far in order to see how it can improve when we fine tune it's hyperparameters." + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
LogisticRegression(max_iter=1000, random_state=42)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" + ], + "text/plain": [ + "LogisticRegression(max_iter=1000, random_state=42)" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from sklearn.linear_model import LogisticRegression\n", + "\n", + "model = LogisticRegression(\n", + " max_iter=1000,\n", + " random_state=42\n", + ")\n", + "\n", + "model.fit(X_train_selected, y_train)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Evaluate your model" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Baseline Accuracy: 0.7838\n" + ] + } + ], + "source": [ + "from sklearn.metrics import accuracy_score\n", + "\n", + "y_pred = model.predict(X_test_selected)\n", + "\n", + "accuracy = accuracy_score(y_test, y_pred)\n", + "\n", + "print(\"Baseline Accuracy:\", round(accuracy, 4))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Grid/Random Search**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For this lab we will use Grid Search." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Define hyperparameters to fine tune." + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'C': [0.01, 0.1, 1, 10, 100], 'solver': ['liblinear', 'lbfgs']}" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "param_grid = {\n", + " \"C\": [0.01, 0.1, 1, 10, 100],\n", + " \"solver\": [\"liblinear\", \"lbfgs\"]\n", + "}\n", + "\n", + "param_grid" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Run Grid Search" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Best parameters: {'C': 100, 'solver': 'liblinear'}\n", + "Best CV accuracy: 0.786\n" + ] + } + ], + "source": [ + "from sklearn.model_selection import GridSearchCV\n", + "\n", + "grid_search = GridSearchCV(\n", + " estimator=LogisticRegression(\n", + " max_iter=1000,\n", + " random_state=42\n", + " ),\n", + " param_grid=param_grid,\n", + " cv=5,\n", + " scoring=\"accuracy\",\n", + " n_jobs=-1\n", + ")\n", + "\n", + "grid_search.fit(\n", + " X_train_selected,\n", + " y_train\n", + ")\n", + "\n", + "print(\"Best parameters:\", grid_search.best_params_)\n", + "print(\"Best CV accuracy:\", round(grid_search.best_score_, 4))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Evaluate your model" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Baseline Accuracy: 0.7838\n", + "Tuned Accuracy: 0.7844\n" + ] + } + ], + "source": [ + "best_model = grid_search.best_estimator_\n", + "\n", + "y_pred_tuned = best_model.predict(\n", + " X_test_selected\n", + ")\n", + "\n", + "tuned_accuracy = accuracy_score(\n", + " y_test,\n", + " y_pred_tuned\n", + ")\n", + "\n", + "print(\"Baseline Accuracy:\", round(accuracy, 4))\n", + "print(\"Tuned Accuracy:\", round(tuned_accuracy, 4))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Model Evaluation\n", + "\n", + "Grid Search identified the best Logistic Regression hyperparameters as `C = 100` and `solver = 'liblinear'`.\n", + "\n", + "The baseline model achieved an accuracy of 0.7838, while the tuned model achieved an accuracy of 0.7844.\n", + "\n", + "Hyperparameter tuning therefore produced a small improvement of 0.0006, equivalent to 0.06 percentage points. This indicates that the baseline Logistic Regression model was already performing close to its optimal configuration for the selected features.\n", + "\n", + "Although the improvement is small, Grid Search provides a systematic method for selecting hyperparameters using cross validation rather than relying on manually chosen settings." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python (lab-hyperparameter-tuning)", + "language": "python", + "name": "lab-hyperparameter-tuning" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.11" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/.virtual_documents/lab-hyper-tuning.ipynb b/.virtual_documents/lab-hyper-tuning.ipynb new file mode 100644 index 0000000..e34948d --- /dev/null +++ b/.virtual_documents/lab-hyper-tuning.ipynb @@ -0,0 +1,267 @@ + + + + + + + + + +#Libraries +import pandas as pd +import numpy as np +from sklearn.model_selection import train_test_split + + +spaceship = pd.read_csv("https://raw.githubusercontent.com/data-bootcamp-v4/data/main/spaceship_titanic.csv") +df = pd.read_csv("https://raw.githubusercontent.com/data-bootcamp-v4/data/main/spaceship_titanic.csv") +spaceship.head() + + +df.info() + + +df.isnull().sum() + + +# Remove identifier and high-cardinality text columns +df = df.drop(columns=["PassengerId", "Cabin", "Name"]) + +df.head() + + +from sklearn.preprocessing import StandardScaler + +numerical_columns = [ + "Age", + "RoomService", + "FoodCourt", + "ShoppingMall", + "Spa", + "VRDeck" +] + +scaler = StandardScaler() + +X_train_scaled = X_train.copy() +X_test_scaled = X_test.copy() + +X_train_scaled[numerical_columns] = scaler.fit_transform( + X_train[numerical_columns] +) + +X_test_scaled[numerical_columns] = scaler.transform( + X_test[numerical_columns] +) + +X_train_scaled.head() + + +# Fill numerical missing values with the median +for column in numerical_columns: + df[column] = df[column].fillna(df[column].median()) + +# Fill categorical missing values with the mode +for column in categorical_columns: + df[column] = df[column].fillna(df[column].mode()[0]) + + +df.isnull().sum() + + +X = df.drop(columns=["Transported"]) +y = df["Transported"] + +print("X shape:", X.shape) +print("y shape:", y.shape) + + +X = pd.get_dummies( + X, + columns=["HomePlanet", "CryoSleep", "Destination", "VIP"], + drop_first=True +) + +X.head() + + + + + +from sklearn.model_selection import train_test_split + +X_train, X_test, y_train, y_test = train_test_split( + X, + y, + test_size=0.20, + random_state=42, + stratify=y +) + +print("Training set:", X_train.shape) +print("Test set:", X_test.shape) + + +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import accuracy_score + +log_model = LogisticRegression( + max_iter=1000, + random_state=42 +) + +log_model.fit(X_train_scaled, y_train) + +y_pred = log_model.predict(X_test_scaled) + +accuracy = accuracy_score(y_test, y_pred) + +print("Scaled Logistic Regression Accuracy:", round(accuracy, 4)) + + +from sklearn.linear_model import LogisticRegression +from sklearn.neighbors import KNeighborsClassifier +from sklearn.tree import DecisionTreeClassifier +from sklearn.ensemble import RandomForestClassifier +from sklearn.metrics import accuracy_score + +models = { + "Logistic Regression": LogisticRegression(max_iter=1000), + "KNN": KNeighborsClassifier(), + "Decision Tree": DecisionTreeClassifier(random_state=42), + "Random Forest": RandomForestClassifier(random_state=42) +} + +for name, model in models.items(): + + model.fit(X_train, y_train) + + y_pred = model.predict(X_test) + + accuracy = accuracy_score(y_test, y_pred) + + print(name, "Accuracy:", round(accuracy, 4)) + + +X_train_scaled.head() + + +from sklearn.feature_selection import SelectKBest, f_classif + +selector = SelectKBest( + score_func=f_classif, + k=8 +) + +X_train_selected = selector.fit_transform( + X_train_scaled, + y_train +) + +X_test_selected = selector.transform( + X_test_scaled +) + +selected_features = X_train_scaled.columns[ + selector.get_support() +] + +print("Selected features:") +print(selected_features.tolist()) + +print("\nTraining shape:", X_train_selected.shape) +print("Test shape:", X_test_selected.shape) + + + + + +from sklearn.linear_model import LogisticRegression + +model = LogisticRegression( + max_iter=1000, + random_state=42 +) + +model.fit(X_train_selected, y_train) + + + + + +from sklearn.metrics import accuracy_score + +y_pred = model.predict(X_test_selected) + +accuracy = accuracy_score(y_test, y_pred) + +print("Baseline Accuracy:", round(accuracy, 4)) + + + + + + + + + + + +param_grid = { + "C": [0.01, 0.1, 1, 10, 100], + "solver": ["liblinear", "lbfgs"] +} + +param_grid + + + + + +from sklearn.model_selection import GridSearchCV + +grid_search = GridSearchCV( + estimator=LogisticRegression( + max_iter=1000, + random_state=42 + ), + param_grid=param_grid, + cv=5, + scoring="accuracy", + n_jobs=-1 +) + +grid_search.fit( + X_train_selected, + y_train +) + +print("Best parameters:", grid_search.best_params_) +print("Best CV accuracy:", round(grid_search.best_score_, 4)) + + + + + +best_model = grid_search.best_estimator_ + +y_pred_tuned = best_model.predict( + X_test_selected +) + +tuned_accuracy = accuracy_score( + y_test, + y_pred_tuned +) + +print("Baseline Accuracy:", round(accuracy, 4)) +print("Tuned Accuracy:", round(tuned_accuracy, 4)) + + + + + + + + + diff --git a/anaconda_projects/db/project_filebrowser.db b/anaconda_projects/db/project_filebrowser.db new file mode 100644 index 0000000000000000000000000000000000000000..ffdfcb381eb2fc2c8dac4ca0c2d19c403f107368 GIT binary patch literal 32768 zcmeI)!B5jr9Ki8*z{Un;dfI8p3khM%h-5=F9!#vW#>F|LbVg!K)3JvsZtd7Q^eUb` z@JIMZ_?LL`kMPLRcA;b}uy|#BU-nD;-fLg_KK)%@+I4TY?gmmE`u(mGh%M!wqN>U> zArwVf3S%*hWEhDsGGWYz5f?SynX+9{9{f1hwBO3Y@|l{|vcHzkw4W<~v?J|k%2{+F zfB*srAb-$IYAQ;5k`KoE?mLV+tdEF4Jar3Gu zWYTV1ywlCL_N4o2g{3npYLqYs;^L<4(9@vz=k! zkLq@8V3*^)l#)GH7w$&BOqu28;(xJKVz$L^MKzTxmDJPpDEFL>?6%wk`$YB!uJ27$ z6IU{EqBBazi_>gx*o`x(uvICJrO=gZm>9wCeLfIZ5@>9^FejcSlDXZ|O*t_+Sb3sh zE@X?=_KFsUn*FPUKMDc}Ab^!WNedj222^iM$m0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_Kd+b3=lv70R#|0009ILKmY** z5V%$#`u{)g|6fZ(BLWB@fB*srAb\n", + "RangeIndex: 8693 entries, 0 to 8692\n", + "Data columns (total 14 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 PassengerId 8693 non-null str \n", + " 1 HomePlanet 8492 non-null str \n", + " 2 CryoSleep 8476 non-null object \n", + " 3 Cabin 8494 non-null str \n", + " 4 Destination 8511 non-null str \n", + " 5 Age 8514 non-null float64\n", + " 6 VIP 8490 non-null object \n", + " 7 RoomService 8512 non-null float64\n", + " 8 FoodCourt 8510 non-null float64\n", + " 9 ShoppingMall 8485 non-null float64\n", + " 10 Spa 8510 non-null float64\n", + " 11 VRDeck 8505 non-null float64\n", + " 12 Name 8493 non-null str \n", + " 13 Transported 8693 non-null bool \n", + "dtypes: bool(1), float64(6), object(2), str(5)\n", + "memory usage: 891.5+ KB\n" + ] + } + ], + "source": [ + "df.info()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "PassengerId 0\n", + "HomePlanet 201\n", + "CryoSleep 217\n", + "Cabin 199\n", + "Destination 182\n", + "Age 179\n", + "VIP 203\n", + "RoomService 181\n", + "FoodCourt 183\n", + "ShoppingMall 208\n", + "Spa 183\n", + "VRDeck 188\n", + "Name 200\n", + "Transported 0\n", + "dtype: int64" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.isnull().sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
HomePlanetCryoSleepDestinationAgeVIPRoomServiceFoodCourtShoppingMallSpaVRDeckTransported
0EuropaFalseTRAPPIST-1e39.0False0.00.00.00.00.0False
1EarthFalseTRAPPIST-1e24.0False109.09.025.0549.044.0True
2EuropaFalseTRAPPIST-1e58.0True43.03576.00.06715.049.0False
3EuropaFalseTRAPPIST-1e33.0False0.01283.0371.03329.0193.0False
4EarthFalseTRAPPIST-1e16.0False303.070.0151.0565.02.0True
\n", + "
" + ], + "text/plain": [ + " HomePlanet CryoSleep Destination Age VIP RoomService FoodCourt \\\n", + "0 Europa False TRAPPIST-1e 39.0 False 0.0 0.0 \n", + "1 Earth False TRAPPIST-1e 24.0 False 109.0 9.0 \n", + "2 Europa False TRAPPIST-1e 58.0 True 43.0 3576.0 \n", + "3 Europa False TRAPPIST-1e 33.0 False 0.0 1283.0 \n", + "4 Earth False TRAPPIST-1e 16.0 False 303.0 70.0 \n", + "\n", + " ShoppingMall Spa VRDeck Transported \n", + "0 0.0 0.0 0.0 False \n", + "1 25.0 549.0 44.0 True \n", + "2 0.0 6715.0 49.0 False \n", + "3 371.0 3329.0 193.0 False \n", + "4 151.0 565.0 2.0 True " + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Remove identifier and high-cardinality text columns\n", + "df = df.drop(columns=[\"PassengerId\", \"Cabin\", \"Name\"])\n", + "\n", + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
AgeRoomServiceFoodCourtShoppingMallSpaVRDeckHomePlanet_EuropaHomePlanet_MarsCryoSleep_TrueDestination_PSO J318.5-22Destination_TRAPPIST-1eVIP_True
3600-1.996149-0.33634-0.279076-0.305732-0.267311-0.264632FalseFalseFalseFalseTrueFalse
1262-0.811703-0.33634-0.279076-0.305732-0.267311-0.264632FalseFalseTrueFalseTrueFalse
86120.442416-0.33634-0.279076-0.305732-0.267311-0.264632FalseFalseFalseTrueFalseFalse
5075-0.184643-0.33634-0.279076-0.305732-0.267311-0.264632TrueFalseTrueFalseFalseFalse
4758-1.090396-0.33634-0.279076-0.195161-0.2664274.423937FalseFalseFalseFalseTrueFalse
\n", + "
" + ], + "text/plain": [ + " Age RoomService FoodCourt ShoppingMall Spa VRDeck \\\n", + "3600 -1.996149 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", + "1262 -0.811703 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", + "8612 0.442416 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", + "5075 -0.184643 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", + "4758 -1.090396 -0.33634 -0.279076 -0.195161 -0.266427 4.423937 \n", + "\n", + " HomePlanet_Europa HomePlanet_Mars CryoSleep_True \\\n", + "3600 False False False \n", + "1262 False False True \n", + "8612 False False False \n", + "5075 True False True \n", + "4758 False False False \n", + "\n", + " Destination_PSO J318.5-22 Destination_TRAPPIST-1e VIP_True \n", + "3600 False True False \n", + "1262 False True False \n", + "8612 True False False \n", + "5075 False False False \n", + "4758 False True False " + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from sklearn.preprocessing import StandardScaler\n", + "\n", + "numerical_columns = [\n", + " \"Age\",\n", + " \"RoomService\",\n", + " \"FoodCourt\",\n", + " \"ShoppingMall\",\n", + " \"Spa\",\n", + " \"VRDeck\"\n", + "]\n", + "\n", + "scaler = StandardScaler()\n", + "\n", + "X_train_scaled = X_train.copy()\n", + "X_test_scaled = X_test.copy()\n", + "\n", + "X_train_scaled[numerical_columns] = scaler.fit_transform(\n", + " X_train[numerical_columns]\n", + ")\n", + "\n", + "X_test_scaled[numerical_columns] = scaler.transform(\n", + " X_test[numerical_columns]\n", + ")\n", + "\n", + "X_train_scaled.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "# Fill numerical missing values with the median\n", + "for column in numerical_columns:\n", + " df[column] = df[column].fillna(df[column].median())\n", + "\n", + "# Fill categorical missing values with the mode\n", + "for column in categorical_columns:\n", + " df[column] = df[column].fillna(df[column].mode()[0])" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "HomePlanet 0\n", + "CryoSleep 0\n", + "Destination 0\n", + "Age 0\n", + "VIP 0\n", + "RoomService 0\n", + "FoodCourt 0\n", + "ShoppingMall 0\n", + "Spa 0\n", + "VRDeck 0\n", + "Transported 0\n", + "dtype: int64" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.isnull().sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "X shape: (8693, 10)\n", + "y shape: (8693,)\n" + ] + } + ], + "source": [ + "X = df.drop(columns=[\"Transported\"])\n", + "y = df[\"Transported\"]\n", + "\n", + "print(\"X shape:\", X.shape)\n", + "print(\"y shape:\", y.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
AgeRoomServiceFoodCourtShoppingMallSpaVRDeckHomePlanet_EuropaHomePlanet_MarsCryoSleep_TrueDestination_PSO J318.5-22Destination_TRAPPIST-1eVIP_True
039.00.00.00.00.00.0TrueFalseFalseFalseTrueFalse
124.0109.09.025.0549.044.0FalseFalseFalseFalseTrueFalse
258.043.03576.00.06715.049.0TrueFalseFalseFalseTrueTrue
333.00.01283.0371.03329.0193.0TrueFalseFalseFalseTrueFalse
416.0303.070.0151.0565.02.0FalseFalseFalseFalseTrueFalse
\n", + "
" + ], + "text/plain": [ + " Age RoomService FoodCourt ShoppingMall Spa VRDeck \\\n", + "0 39.0 0.0 0.0 0.0 0.0 0.0 \n", + "1 24.0 109.0 9.0 25.0 549.0 44.0 \n", + "2 58.0 43.0 3576.0 0.0 6715.0 49.0 \n", + "3 33.0 0.0 1283.0 371.0 3329.0 193.0 \n", + "4 16.0 303.0 70.0 151.0 565.0 2.0 \n", + "\n", + " HomePlanet_Europa HomePlanet_Mars CryoSleep_True \\\n", + "0 True False False \n", + "1 False False False \n", + "2 True False False \n", + "3 True False False \n", + "4 False False False \n", + "\n", + " Destination_PSO J318.5-22 Destination_TRAPPIST-1e VIP_True \n", + "0 False True False \n", + "1 False True False \n", + "2 False True True \n", + "3 False True False \n", + "4 False True False " + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = pd.get_dummies(\n", + " X,\n", + " columns=[\"HomePlanet\", \"CryoSleep\", \"Destination\", \"VIP\"],\n", + " drop_first=True\n", + ")\n", + "\n", + "X.head()" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -221,11 +837,315 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training set: (6954, 12)\n", + "Test set: (1739, 12)\n" + ] + } + ], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "\n", + "X_train, X_test, y_train, y_test = train_test_split(\n", + " X,\n", + " y,\n", + " test_size=0.20,\n", + " random_state=42,\n", + " stratify=y\n", + ")\n", + "\n", + "print(\"Training set:\", X_train.shape)\n", + "print(\"Test set:\", X_test.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Scaled Logistic Regression Accuracy: 0.7884\n" + ] + } + ], + "source": [ + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.metrics import accuracy_score\n", + "\n", + "log_model = LogisticRegression(\n", + " max_iter=1000,\n", + " random_state=42\n", + ")\n", + "\n", + "log_model.fit(X_train_scaled, y_train)\n", + "\n", + "y_pred = log_model.predict(X_test_scaled)\n", + "\n", + "accuracy = accuracy_score(y_test, y_pred)\n", + "\n", + "print(\"Scaled Logistic Regression Accuracy:\", round(accuracy, 4))" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/jannoelvero/Documents/Ironhack/week_7/day_4/lab-hyperparameter-tuning/.venv/lib/python3.13/site-packages/sklearn/linear_model/_logistic.py:599: ConvergenceWarning: lbfgs failed to converge after 1000 iteration(s) (status=1):\n", + "STOP: TOTAL NO. OF ITERATIONS REACHED LIMIT\n", + "\n", + "Increase the number of iterations to improve the convergence (max_iter=1000).\n", + "You might also want to scale the data as shown in:\n", + " https://scikit-learn.org/stable/modules/preprocessing.html\n", + "Please also refer to the documentation for alternative solver options:\n", + " https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n", + " n_iter_i = _check_optimize_result(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logistic Regression Accuracy: 0.7878\n", + "KNN Accuracy: 0.7608\n", + "Decision Tree Accuracy: 0.7338\n", + "Random Forest Accuracy: 0.7849\n" + ] + } + ], + "source": [ + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.neighbors import KNeighborsClassifier\n", + "from sklearn.tree import DecisionTreeClassifier\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from sklearn.metrics import accuracy_score\n", + "\n", + "models = {\n", + " \"Logistic Regression\": LogisticRegression(max_iter=1000),\n", + " \"KNN\": KNeighborsClassifier(),\n", + " \"Decision Tree\": DecisionTreeClassifier(random_state=42),\n", + " \"Random Forest\": RandomForestClassifier(random_state=42)\n", + "}\n", + "\n", + "for name, model in models.items():\n", + "\n", + " model.fit(X_train, y_train)\n", + "\n", + " y_pred = model.predict(X_test)\n", + "\n", + " accuracy = accuracy_score(y_test, y_pred)\n", + "\n", + " print(name, \"Accuracy:\", round(accuracy, 4))" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
AgeRoomServiceFoodCourtShoppingMallSpaVRDeckHomePlanet_EuropaHomePlanet_MarsCryoSleep_TrueDestination_PSO J318.5-22Destination_TRAPPIST-1eVIP_True
3600-1.996149-0.33634-0.279076-0.305732-0.267311-0.264632FalseFalseFalseFalseTrueFalse
1262-0.811703-0.33634-0.279076-0.305732-0.267311-0.264632FalseFalseTrueFalseTrueFalse
86120.442416-0.33634-0.279076-0.305732-0.267311-0.264632FalseFalseFalseTrueFalseFalse
5075-0.184643-0.33634-0.279076-0.305732-0.267311-0.264632TrueFalseTrueFalseFalseFalse
4758-1.090396-0.33634-0.279076-0.195161-0.2664274.423937FalseFalseFalseFalseTrueFalse
\n", + "
" + ], + "text/plain": [ + " Age RoomService FoodCourt ShoppingMall Spa VRDeck \\\n", + "3600 -1.996149 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", + "1262 -0.811703 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", + "8612 0.442416 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", + "5075 -0.184643 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", + "4758 -1.090396 -0.33634 -0.279076 -0.195161 -0.266427 4.423937 \n", + "\n", + " HomePlanet_Europa HomePlanet_Mars CryoSleep_True \\\n", + "3600 False False False \n", + "1262 False False True \n", + "8612 False False False \n", + "5075 True False True \n", + "4758 False False False \n", + "\n", + " Destination_PSO J318.5-22 Destination_TRAPPIST-1e VIP_True \n", + "3600 False True False \n", + "1262 False True False \n", + "8612 True False False \n", + "5075 False False False \n", + "4758 False True False " + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X_train_scaled.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 29, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Selected features:\n", + "['Age', 'RoomService', 'FoodCourt', 'Spa', 'VRDeck', 'HomePlanet_Europa', 'CryoSleep_True', 'Destination_TRAPPIST-1e']\n", + "\n", + "Training shape: (6954, 8)\n", + "Test shape: (1739, 8)\n" + ] + } + ], "source": [ - "#your code here" + "from sklearn.feature_selection import SelectKBest, f_classif\n", + "\n", + "selector = SelectKBest(\n", + " score_func=f_classif,\n", + " k=8\n", + ")\n", + "\n", + "X_train_selected = selector.fit_transform(\n", + " X_train_scaled,\n", + " y_train\n", + ")\n", + "\n", + "X_test_selected = selector.transform(\n", + " X_test_scaled\n", + ")\n", + "\n", + "selected_features = X_train_scaled.columns[\n", + " selector.get_support()\n", + "]\n", + "\n", + "print(\"Selected features:\")\n", + "print(selected_features.tolist())\n", + "\n", + "print(\"\\nTraining shape:\", X_train_selected.shape)\n", + "print(\"Test shape:\", X_test_selected.shape)" ] }, { @@ -237,11 +1157,1281 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
LogisticRegression(max_iter=1000, random_state=42)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" + ], + "text/plain": [ + "LogisticRegression(max_iter=1000, random_state=42)" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "#your code here" + "from sklearn.linear_model import LogisticRegression\n", + "\n", + "model = LogisticRegression(\n", + " max_iter=1000,\n", + " random_state=42\n", + ")\n", + "\n", + "model.fit(X_train_selected, y_train)" ] }, { @@ -253,11 +2443,25 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 31, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Baseline Accuracy: 0.7838\n" + ] + } + ], "source": [ - "#your code here" + "from sklearn.metrics import accuracy_score\n", + "\n", + "y_pred = model.predict(X_test_selected)\n", + "\n", + "accuracy = accuracy_score(y_test, y_pred)\n", + "\n", + "print(\"Baseline Accuracy:\", round(accuracy, 4))" ] }, { @@ -283,11 +2487,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'C': [0.01, 0.1, 1, 10, 100], 'solver': ['liblinear', 'lbfgs']}" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "#your code here" + "param_grid = {\n", + " \"C\": [0.01, 0.1, 1, 10, 100],\n", + " \"solver\": [\"liblinear\", \"lbfgs\"]\n", + "}\n", + "\n", + "param_grid" ] }, { @@ -299,10 +2519,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Best parameters: {'C': 100, 'solver': 'liblinear'}\n", + "Best CV accuracy: 0.786\n" + ] + } + ], + "source": [ + "from sklearn.model_selection import GridSearchCV\n", + "\n", + "grid_search = GridSearchCV(\n", + " estimator=LogisticRegression(\n", + " max_iter=1000,\n", + " random_state=42\n", + " ),\n", + " param_grid=param_grid,\n", + " cv=5,\n", + " scoring=\"accuracy\",\n", + " n_jobs=-1\n", + ")\n", + "\n", + "grid_search.fit(\n", + " X_train_selected,\n", + " y_train\n", + ")\n", + "\n", + "print(\"Best parameters:\", grid_search.best_params_)\n", + "print(\"Best CV accuracy:\", round(grid_search.best_score_, 4))" + ] }, { "cell_type": "markdown", @@ -311,6 +2561,58 @@ "- Evaluate your model" ] }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Baseline Accuracy: 0.7838\n", + "Tuned Accuracy: 0.7844\n" + ] + } + ], + "source": [ + "best_model = grid_search.best_estimator_\n", + "\n", + "y_pred_tuned = best_model.predict(\n", + " X_test_selected\n", + ")\n", + "\n", + "tuned_accuracy = accuracy_score(\n", + " y_test,\n", + " y_pred_tuned\n", + ")\n", + "\n", + "print(\"Baseline Accuracy:\", round(accuracy, 4))\n", + "print(\"Tuned Accuracy:\", round(tuned_accuracy, 4))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Model Evaluation\n", + "\n", + "Grid Search identified the best Logistic Regression hyperparameters as `C = 100` and `solver = 'liblinear'`.\n", + "\n", + "The baseline model achieved an accuracy of 0.7838, while the tuned model achieved an accuracy of 0.7844.\n", + "\n", + "Hyperparameter tuning therefore produced a small improvement of 0.0006, equivalent to 0.06 percentage points. This indicates that the baseline Logistic Regression model was already performing close to its optimal configuration for the selected features.\n", + "\n", + "Although the improvement is small, Grid Search provides a systematic method for selecting hyperparameters using cross validation rather than relying on manually chosen settings." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, @@ -321,9 +2623,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python (lab-hyperparameter-tuning)", "language": "python", - "name": "python3" + "name": "lab-hyperparameter-tuning" }, "language_info": { "codemirror_mode": { @@ -335,9 +2637,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.9" + "version": "3.13.11" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } From 7e42d166b29e2d386f0c99678b0d6fd24832fd79 Mon Sep 17 00:00:00 2001 From: Jan Noel Vero Date: Thu, 3 Sep 2026 14:38:20 +0200 Subject: [PATCH 2/2] Solved lab --- .../lab-hyper-tuning-checkpoint.ipynb | 602 +++++------------- .virtual_documents/lab-hyper-tuning.ipynb | 83 ++- lab-hyper-tuning.ipynb | 602 +++++------------- 3 files changed, 343 insertions(+), 944 deletions(-) diff --git a/.ipynb_checkpoints/lab-hyper-tuning-checkpoint.ipynb b/.ipynb_checkpoints/lab-hyper-tuning-checkpoint.ipynb index e6e9053..5b4a1c0 100644 --- a/.ipynb_checkpoints/lab-hyper-tuning-checkpoint.ipynb +++ b/.ipynb_checkpoints/lab-hyper-tuning-checkpoint.ipynb @@ -35,19 +35,24 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ - "#Libraries\n", "import pandas as pd\n", "import numpy as np\n", - "from sklearn.model_selection import train_test_split" + "\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.preprocessing import StandardScaler\n", + "from sklearn.feature_selection import SelectKBest, f_classif\n", + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.metrics import accuracy_score\n", + "from sklearn.model_selection import GridSearchCV" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -200,7 +205,7 @@ "4 True " ] }, - "execution_count": 12, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -213,7 +218,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -250,7 +255,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -273,7 +278,7 @@ "dtype: int64" ] }, - "execution_count": 14, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -284,7 +289,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -412,169 +417,52 @@ "4 151.0 565.0 2.0 True " ] }, - "execution_count": 15, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "# Remove identifier and high-cardinality text columns\n", - "df = df.drop(columns=[\"PassengerId\", \"Cabin\", \"Name\"])\n", + "df = df.drop(\n", + " columns=[\n", + " \"PassengerId\",\n", + " \"Cabin\",\n", + " \"Name\"\n", + " ]\n", + ")\n", "\n", "df.head()" ] }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 17, "metadata": {}, "outputs": [ { "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AgeRoomServiceFoodCourtShoppingMallSpaVRDeckHomePlanet_EuropaHomePlanet_MarsCryoSleep_TrueDestination_PSO J318.5-22Destination_TRAPPIST-1eVIP_True
3600-1.996149-0.33634-0.279076-0.305732-0.267311-0.264632FalseFalseFalseFalseTrueFalse
1262-0.811703-0.33634-0.279076-0.305732-0.267311-0.264632FalseFalseTrueFalseTrueFalse
86120.442416-0.33634-0.279076-0.305732-0.267311-0.264632FalseFalseFalseTrueFalseFalse
5075-0.184643-0.33634-0.279076-0.305732-0.267311-0.264632TrueFalseTrueFalseFalseFalse
4758-1.090396-0.33634-0.279076-0.195161-0.2664274.423937FalseFalseFalseFalseTrueFalse
\n", - "
" - ], "text/plain": [ - " Age RoomService FoodCourt ShoppingMall Spa VRDeck \\\n", - "3600 -1.996149 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", - "1262 -0.811703 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", - "8612 0.442416 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", - "5075 -0.184643 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", - "4758 -1.090396 -0.33634 -0.279076 -0.195161 -0.266427 4.423937 \n", - "\n", - " HomePlanet_Europa HomePlanet_Mars CryoSleep_True \\\n", - "3600 False False False \n", - "1262 False False True \n", - "8612 False False False \n", - "5075 True False True \n", - "4758 False False False \n", - "\n", - " Destination_PSO J318.5-22 Destination_TRAPPIST-1e VIP_True \n", - "3600 False True False \n", - "1262 False True False \n", - "8612 True False False \n", - "5075 False False False \n", - "4758 False True False " + "HomePlanet 0\n", + "CryoSleep 0\n", + "Destination 0\n", + "Age 0\n", + "VIP 0\n", + "RoomService 0\n", + "FoodCourt 0\n", + "ShoppingMall 0\n", + "Spa 0\n", + "VRDeck 0\n", + "Transported 0\n", + "dtype: int64" ] }, - "execution_count": 28, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "from sklearn.preprocessing import StandardScaler\n", - "\n", + "# Define numerical and categorical columns\n", "numerical_columns = [\n", " \"Age\",\n", " \"RoomService\",\n", @@ -584,71 +472,28 @@ " \"VRDeck\"\n", "]\n", "\n", - "scaler = StandardScaler()\n", - "\n", - "X_train_scaled = X_train.copy()\n", - "X_test_scaled = X_test.copy()\n", - "\n", - "X_train_scaled[numerical_columns] = scaler.fit_transform(\n", - " X_train[numerical_columns]\n", - ")\n", - "\n", - "X_test_scaled[numerical_columns] = scaler.transform(\n", - " X_test[numerical_columns]\n", - ")\n", + "categorical_columns = [\n", + " \"HomePlanet\",\n", + " \"CryoSleep\",\n", + " \"Destination\",\n", + " \"VIP\"\n", + "]\n", "\n", - "X_train_scaled.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "# Fill numerical missing values with the median\n", + "# Fill missing numerical values with the median\n", "for column in numerical_columns:\n", " df[column] = df[column].fillna(df[column].median())\n", "\n", - "# Fill categorical missing values with the mode\n", + "# Fill missing categorical values with the mode\n", "for column in categorical_columns:\n", - " df[column] = df[column].fillna(df[column].mode()[0])" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "HomePlanet 0\n", - "CryoSleep 0\n", - "Destination 0\n", - "Age 0\n", - "VIP 0\n", - "RoomService 0\n", - "FoodCourt 0\n", - "ShoppingMall 0\n", - "Spa 0\n", - "VRDeck 0\n", - "Transported 0\n", - "dtype: int64" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ + " df[column] = df[column].fillna(df[column].mode()[0])\n", + "\n", + "# Verify missing values\n", "df.isnull().sum()" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -670,174 +515,40 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "metadata": {}, "outputs": [ { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AgeRoomServiceFoodCourtShoppingMallSpaVRDeckHomePlanet_EuropaHomePlanet_MarsCryoSleep_TrueDestination_PSO J318.5-22Destination_TRAPPIST-1eVIP_True
039.00.00.00.00.00.0TrueFalseFalseFalseTrueFalse
124.0109.09.025.0549.044.0FalseFalseFalseFalseTrueFalse
258.043.03576.00.06715.049.0TrueFalseFalseFalseTrueTrue
333.00.01283.0371.03329.0193.0TrueFalseFalseFalseTrueFalse
416.0303.070.0151.0565.02.0FalseFalseFalseFalseTrueFalse
\n", - "
" - ], - "text/plain": [ - " Age RoomService FoodCourt ShoppingMall Spa VRDeck \\\n", - "0 39.0 0.0 0.0 0.0 0.0 0.0 \n", - "1 24.0 109.0 9.0 25.0 549.0 44.0 \n", - "2 58.0 43.0 3576.0 0.0 6715.0 49.0 \n", - "3 33.0 0.0 1283.0 371.0 3329.0 193.0 \n", - "4 16.0 303.0 70.0 151.0 565.0 2.0 \n", - "\n", - " HomePlanet_Europa HomePlanet_Mars CryoSleep_True \\\n", - "0 True False False \n", - "1 False False False \n", - "2 True False False \n", - "3 True False False \n", - "4 False False False \n", - "\n", - " Destination_PSO J318.5-22 Destination_TRAPPIST-1e VIP_True \n", - "0 False True False \n", - "1 False True False \n", - "2 False True True \n", - "3 False True False \n", - "4 False True False " - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "X shape after encoding: (8693, 12)\n", + "\n", + "Columns:\n", + "['Age', 'RoomService', 'FoodCourt', 'ShoppingMall', 'Spa', 'VRDeck', 'HomePlanet_Europa', 'HomePlanet_Mars', 'CryoSleep_True', 'Destination_PSO J318.5-22', 'Destination_TRAPPIST-1e', 'VIP_True']\n" + ] } ], "source": [ "X = pd.get_dummies(\n", " X,\n", - " columns=[\"HomePlanet\", \"CryoSleep\", \"Destination\", \"VIP\"],\n", + " columns=[\n", + " \"HomePlanet\",\n", + " \"CryoSleep\",\n", + " \"Destination\",\n", + " \"VIP\"\n", + " ],\n", " drop_first=True\n", ")\n", "\n", - "X.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now perform the same as before:\n", - "- Feature Scaling\n", - "- Feature Selection\n" + "print(\"X shape after encoding:\", X.shape)\n", + "print(\"\\nColumns:\")\n", + "print(X.columns.tolist())" ] }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -850,8 +561,6 @@ } ], "source": [ - "from sklearn.model_selection import train_test_split\n", - "\n", "X_train, X_test, y_train, y_test = train_test_split(\n", " X,\n", " y,\n", @@ -865,95 +574,24 @@ ] }, { - "cell_type": "code", - "execution_count": 25, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Scaled Logistic Regression Accuracy: 0.7884\n" - ] - } - ], "source": [ - "from sklearn.linear_model import LogisticRegression\n", - "from sklearn.metrics import accuracy_score\n", - "\n", - "log_model = LogisticRegression(\n", - " max_iter=1000,\n", - " random_state=42\n", - ")\n", - "\n", - "log_model.fit(X_train_scaled, y_train)\n", - "\n", - "y_pred = log_model.predict(X_test_scaled)\n", - "\n", - "accuracy = accuracy_score(y_test, y_pred)\n", - "\n", - "print(\"Scaled Logistic Regression Accuracy:\", round(accuracy, 4))" + "Now perform the same as before:\n", + "- Feature Scaling\n", + "- Feature Selection\n" ] }, { - "cell_type": "code", - "execution_count": 26, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/jannoelvero/Documents/Ironhack/week_7/day_4/lab-hyperparameter-tuning/.venv/lib/python3.13/site-packages/sklearn/linear_model/_logistic.py:599: ConvergenceWarning: lbfgs failed to converge after 1000 iteration(s) (status=1):\n", - "STOP: TOTAL NO. OF ITERATIONS REACHED LIMIT\n", - "\n", - "Increase the number of iterations to improve the convergence (max_iter=1000).\n", - "You might also want to scale the data as shown in:\n", - " https://scikit-learn.org/stable/modules/preprocessing.html\n", - "Please also refer to the documentation for alternative solver options:\n", - " https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n", - " n_iter_i = _check_optimize_result(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logistic Regression Accuracy: 0.7878\n", - "KNN Accuracy: 0.7608\n", - "Decision Tree Accuracy: 0.7338\n", - "Random Forest Accuracy: 0.7849\n" - ] - } - ], "source": [ - "from sklearn.linear_model import LogisticRegression\n", - "from sklearn.neighbors import KNeighborsClassifier\n", - "from sklearn.tree import DecisionTreeClassifier\n", - "from sklearn.ensemble import RandomForestClassifier\n", - "from sklearn.metrics import accuracy_score\n", - "\n", - "models = {\n", - " \"Logistic Regression\": LogisticRegression(max_iter=1000),\n", - " \"KNN\": KNeighborsClassifier(),\n", - " \"Decision Tree\": DecisionTreeClassifier(random_state=42),\n", - " \"Random Forest\": RandomForestClassifier(random_state=42)\n", - "}\n", - "\n", - "for name, model in models.items():\n", - "\n", - " model.fit(X_train, y_train)\n", - "\n", - " y_pred = model.predict(X_test)\n", - "\n", - " accuracy = accuracy_score(y_test, y_pred)\n", - "\n", - " print(name, \"Accuracy:\", round(accuracy, 4))" + "## Feature Scaling" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -1094,18 +732,38 @@ "4758 False True False " ] }, - "execution_count": 27, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ + "scaler = StandardScaler()\n", + "\n", + "X_train_scaled = X_train.copy()\n", + "X_test_scaled = X_test.copy()\n", + "\n", + "X_train_scaled[numerical_columns] = scaler.fit_transform(\n", + " X_train[numerical_columns]\n", + ")\n", + "\n", + "X_test_scaled[numerical_columns] = scaler.transform(\n", + " X_test[numerical_columns]\n", + ")\n", + "\n", "X_train_scaled.head()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Feature Selection" + ] + }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -1121,8 +779,6 @@ } ], "source": [ - "from sklearn.feature_selection import SelectKBest, f_classif\n", - "\n", "selector = SelectKBest(\n", " score_func=f_classif,\n", " k=8\n", @@ -1157,7 +813,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 24, "metadata": {}, "outputs": [ { @@ -2418,20 +2074,21 @@ "LogisticRegression(max_iter=1000, random_state=42)" ] }, - "execution_count": 30, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "from sklearn.linear_model import LogisticRegression\n", - "\n", "model = LogisticRegression(\n", " max_iter=1000,\n", " random_state=42\n", ")\n", "\n", - "model.fit(X_train_selected, y_train)" + "model.fit(\n", + " X_train_selected,\n", + " y_train\n", + ")" ] }, { @@ -2443,7 +2100,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 25, "metadata": {}, "outputs": [ { @@ -2455,13 +2112,19 @@ } ], "source": [ - "from sklearn.metrics import accuracy_score\n", - "\n", - "y_pred = model.predict(X_test_selected)\n", + "y_pred = model.predict(\n", + " X_test_selected\n", + ")\n", "\n", - "accuracy = accuracy_score(y_test, y_pred)\n", + "baseline_accuracy = accuracy_score(\n", + " y_test,\n", + " y_pred\n", + ")\n", "\n", - "print(\"Baseline Accuracy:\", round(accuracy, 4))" + "print(\n", + " \"Baseline Accuracy:\",\n", + " round(baseline_accuracy, 4)\n", + ")" ] }, { @@ -2487,7 +2150,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 29, "metadata": {}, "outputs": [ { @@ -2496,7 +2159,7 @@ "{'C': [0.01, 0.1, 1, 10, 100], 'solver': ['liblinear', 'lbfgs']}" ] }, - "execution_count": 32, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -2519,7 +2182,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -2532,8 +2195,6 @@ } ], "source": [ - "from sklearn.model_selection import GridSearchCV\n", - "\n", "grid_search = GridSearchCV(\n", " estimator=LogisticRegression(\n", " max_iter=1000,\n", @@ -2563,7 +2224,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 30, "metadata": {}, "outputs": [ { @@ -2587,8 +2248,15 @@ " y_pred_tuned\n", ")\n", "\n", - "print(\"Baseline Accuracy:\", round(accuracy, 4))\n", - "print(\"Tuned Accuracy:\", round(tuned_accuracy, 4))" + "print(\n", + " \"Baseline Accuracy:\",\n", + " round(baseline_accuracy, 4)\n", + ")\n", + "\n", + "print(\n", + " \"Tuned Accuracy:\",\n", + " round(tuned_accuracy, 4)\n", + ")" ] }, { @@ -2613,6 +2281,20 @@ "outputs": [], "source": [] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, diff --git a/.virtual_documents/lab-hyper-tuning.ipynb b/.virtual_documents/lab-hyper-tuning.ipynb index e34948d..e435909 100644 --- a/.virtual_documents/lab-hyper-tuning.ipynb +++ b/.virtual_documents/lab-hyper-tuning.ipynb @@ -85,6 +85,59 @@ X = pd.get_dummies( X.head() +from sklearn.preprocessing import StandardScaler +from sklearn.feature_selection import SelectKBest, f_classif + +# Numerical columns to scale +numerical_columns = [ + "Age", + "RoomService", + "FoodCourt", + "ShoppingMall", + "Spa", + "VRDeck" +] + +# Feature Scaling +scaler = StandardScaler() + +X_train_scaled = X_train.copy() +X_test_scaled = X_test.copy() + +X_train_scaled[numerical_columns] = scaler.fit_transform( + X_train[numerical_columns] +) + +X_test_scaled[numerical_columns] = scaler.transform( + X_test[numerical_columns] +) + +# Feature Selection +selector = SelectKBest( + score_func=f_classif, + k=8 +) + +X_train_selected = selector.fit_transform( + X_train_scaled, + y_train +) + +X_test_selected = selector.transform( + X_test_scaled +) + +selected_features = X_train_scaled.columns[ + selector.get_support() +] + +print("Selected features:") +print(selected_features.tolist()) + +print("\nTraining shape:", X_train_selected.shape) +print("Test shape:", X_test_selected.shape) + + @@ -119,30 +172,6 @@ accuracy = accuracy_score(y_test, y_pred) print("Scaled Logistic Regression Accuracy:", round(accuracy, 4)) -from sklearn.linear_model import LogisticRegression -from sklearn.neighbors import KNeighborsClassifier -from sklearn.tree import DecisionTreeClassifier -from sklearn.ensemble import RandomForestClassifier -from sklearn.metrics import accuracy_score - -models = { - "Logistic Regression": LogisticRegression(max_iter=1000), - "KNN": KNeighborsClassifier(), - "Decision Tree": DecisionTreeClassifier(random_state=42), - "Random Forest": RandomForestClassifier(random_state=42) -} - -for name, model in models.items(): - - model.fit(X_train, y_train) - - y_pred = model.predict(X_test) - - accuracy = accuracy_score(y_test, y_pred) - - print(name, "Accuracy:", round(accuracy, 4)) - - X_train_scaled.head() @@ -265,3 +294,9 @@ print("Tuned Accuracy:", round(tuned_accuracy, 4)) + + + + + + diff --git a/lab-hyper-tuning.ipynb b/lab-hyper-tuning.ipynb index e6e9053..5b4a1c0 100644 --- a/lab-hyper-tuning.ipynb +++ b/lab-hyper-tuning.ipynb @@ -35,19 +35,24 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ - "#Libraries\n", "import pandas as pd\n", "import numpy as np\n", - "from sklearn.model_selection import train_test_split" + "\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.preprocessing import StandardScaler\n", + "from sklearn.feature_selection import SelectKBest, f_classif\n", + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.metrics import accuracy_score\n", + "from sklearn.model_selection import GridSearchCV" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -200,7 +205,7 @@ "4 True " ] }, - "execution_count": 12, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -213,7 +218,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -250,7 +255,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -273,7 +278,7 @@ "dtype: int64" ] }, - "execution_count": 14, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -284,7 +289,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -412,169 +417,52 @@ "4 151.0 565.0 2.0 True " ] }, - "execution_count": 15, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "# Remove identifier and high-cardinality text columns\n", - "df = df.drop(columns=[\"PassengerId\", \"Cabin\", \"Name\"])\n", + "df = df.drop(\n", + " columns=[\n", + " \"PassengerId\",\n", + " \"Cabin\",\n", + " \"Name\"\n", + " ]\n", + ")\n", "\n", "df.head()" ] }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 17, "metadata": {}, "outputs": [ { "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AgeRoomServiceFoodCourtShoppingMallSpaVRDeckHomePlanet_EuropaHomePlanet_MarsCryoSleep_TrueDestination_PSO J318.5-22Destination_TRAPPIST-1eVIP_True
3600-1.996149-0.33634-0.279076-0.305732-0.267311-0.264632FalseFalseFalseFalseTrueFalse
1262-0.811703-0.33634-0.279076-0.305732-0.267311-0.264632FalseFalseTrueFalseTrueFalse
86120.442416-0.33634-0.279076-0.305732-0.267311-0.264632FalseFalseFalseTrueFalseFalse
5075-0.184643-0.33634-0.279076-0.305732-0.267311-0.264632TrueFalseTrueFalseFalseFalse
4758-1.090396-0.33634-0.279076-0.195161-0.2664274.423937FalseFalseFalseFalseTrueFalse
\n", - "
" - ], "text/plain": [ - " Age RoomService FoodCourt ShoppingMall Spa VRDeck \\\n", - "3600 -1.996149 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", - "1262 -0.811703 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", - "8612 0.442416 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", - "5075 -0.184643 -0.33634 -0.279076 -0.305732 -0.267311 -0.264632 \n", - "4758 -1.090396 -0.33634 -0.279076 -0.195161 -0.266427 4.423937 \n", - "\n", - " HomePlanet_Europa HomePlanet_Mars CryoSleep_True \\\n", - "3600 False False False \n", - "1262 False False True \n", - "8612 False False False \n", - "5075 True False True \n", - "4758 False False False \n", - "\n", - " Destination_PSO J318.5-22 Destination_TRAPPIST-1e VIP_True \n", - "3600 False True False \n", - "1262 False True False \n", - "8612 True False False \n", - "5075 False False False \n", - "4758 False True False " + "HomePlanet 0\n", + "CryoSleep 0\n", + "Destination 0\n", + "Age 0\n", + "VIP 0\n", + "RoomService 0\n", + "FoodCourt 0\n", + "ShoppingMall 0\n", + "Spa 0\n", + "VRDeck 0\n", + "Transported 0\n", + "dtype: int64" ] }, - "execution_count": 28, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "from sklearn.preprocessing import StandardScaler\n", - "\n", + "# Define numerical and categorical columns\n", "numerical_columns = [\n", " \"Age\",\n", " \"RoomService\",\n", @@ -584,71 +472,28 @@ " \"VRDeck\"\n", "]\n", "\n", - "scaler = StandardScaler()\n", - "\n", - "X_train_scaled = X_train.copy()\n", - "X_test_scaled = X_test.copy()\n", - "\n", - "X_train_scaled[numerical_columns] = scaler.fit_transform(\n", - " X_train[numerical_columns]\n", - ")\n", - "\n", - "X_test_scaled[numerical_columns] = scaler.transform(\n", - " X_test[numerical_columns]\n", - ")\n", + "categorical_columns = [\n", + " \"HomePlanet\",\n", + " \"CryoSleep\",\n", + " \"Destination\",\n", + " \"VIP\"\n", + "]\n", "\n", - "X_train_scaled.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "# Fill numerical missing values with the median\n", + "# Fill missing numerical values with the median\n", "for column in numerical_columns:\n", " df[column] = df[column].fillna(df[column].median())\n", "\n", - "# Fill categorical missing values with the mode\n", + "# Fill missing categorical values with the mode\n", "for column in categorical_columns:\n", - " df[column] = df[column].fillna(df[column].mode()[0])" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "HomePlanet 0\n", - "CryoSleep 0\n", - "Destination 0\n", - "Age 0\n", - "VIP 0\n", - "RoomService 0\n", - "FoodCourt 0\n", - "ShoppingMall 0\n", - "Spa 0\n", - "VRDeck 0\n", - "Transported 0\n", - "dtype: int64" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ + " df[column] = df[column].fillna(df[column].mode()[0])\n", + "\n", + "# Verify missing values\n", "df.isnull().sum()" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -670,174 +515,40 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "metadata": {}, "outputs": [ { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AgeRoomServiceFoodCourtShoppingMallSpaVRDeckHomePlanet_EuropaHomePlanet_MarsCryoSleep_TrueDestination_PSO J318.5-22Destination_TRAPPIST-1eVIP_True
039.00.00.00.00.00.0TrueFalseFalseFalseTrueFalse
124.0109.09.025.0549.044.0FalseFalseFalseFalseTrueFalse
258.043.03576.00.06715.049.0TrueFalseFalseFalseTrueTrue
333.00.01283.0371.03329.0193.0TrueFalseFalseFalseTrueFalse
416.0303.070.0151.0565.02.0FalseFalseFalseFalseTrueFalse
\n", - "
" - ], - "text/plain": [ - " Age RoomService FoodCourt ShoppingMall Spa VRDeck \\\n", - "0 39.0 0.0 0.0 0.0 0.0 0.0 \n", - "1 24.0 109.0 9.0 25.0 549.0 44.0 \n", - "2 58.0 43.0 3576.0 0.0 6715.0 49.0 \n", - "3 33.0 0.0 1283.0 371.0 3329.0 193.0 \n", - "4 16.0 303.0 70.0 151.0 565.0 2.0 \n", - "\n", - " HomePlanet_Europa HomePlanet_Mars CryoSleep_True \\\n", - "0 True False False \n", - "1 False False False \n", - "2 True False False \n", - "3 True False False \n", - "4 False False False \n", - "\n", - " Destination_PSO J318.5-22 Destination_TRAPPIST-1e VIP_True \n", - "0 False True False \n", - "1 False True False \n", - "2 False True True \n", - "3 False True False \n", - "4 False True False " - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "X shape after encoding: (8693, 12)\n", + "\n", + "Columns:\n", + "['Age', 'RoomService', 'FoodCourt', 'ShoppingMall', 'Spa', 'VRDeck', 'HomePlanet_Europa', 'HomePlanet_Mars', 'CryoSleep_True', 'Destination_PSO J318.5-22', 'Destination_TRAPPIST-1e', 'VIP_True']\n" + ] } ], "source": [ "X = pd.get_dummies(\n", " X,\n", - " columns=[\"HomePlanet\", \"CryoSleep\", \"Destination\", \"VIP\"],\n", + " columns=[\n", + " \"HomePlanet\",\n", + " \"CryoSleep\",\n", + " \"Destination\",\n", + " \"VIP\"\n", + " ],\n", " drop_first=True\n", ")\n", "\n", - "X.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now perform the same as before:\n", - "- Feature Scaling\n", - "- Feature Selection\n" + "print(\"X shape after encoding:\", X.shape)\n", + "print(\"\\nColumns:\")\n", + "print(X.columns.tolist())" ] }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -850,8 +561,6 @@ } ], "source": [ - "from sklearn.model_selection import train_test_split\n", - "\n", "X_train, X_test, y_train, y_test = train_test_split(\n", " X,\n", " y,\n", @@ -865,95 +574,24 @@ ] }, { - "cell_type": "code", - "execution_count": 25, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Scaled Logistic Regression Accuracy: 0.7884\n" - ] - } - ], "source": [ - "from sklearn.linear_model import LogisticRegression\n", - "from sklearn.metrics import accuracy_score\n", - "\n", - "log_model = LogisticRegression(\n", - " max_iter=1000,\n", - " random_state=42\n", - ")\n", - "\n", - "log_model.fit(X_train_scaled, y_train)\n", - "\n", - "y_pred = log_model.predict(X_test_scaled)\n", - "\n", - "accuracy = accuracy_score(y_test, y_pred)\n", - "\n", - "print(\"Scaled Logistic Regression Accuracy:\", round(accuracy, 4))" + "Now perform the same as before:\n", + "- Feature Scaling\n", + "- Feature Selection\n" ] }, { - "cell_type": "code", - "execution_count": 26, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/jannoelvero/Documents/Ironhack/week_7/day_4/lab-hyperparameter-tuning/.venv/lib/python3.13/site-packages/sklearn/linear_model/_logistic.py:599: ConvergenceWarning: lbfgs failed to converge after 1000 iteration(s) (status=1):\n", - "STOP: TOTAL NO. OF ITERATIONS REACHED LIMIT\n", - "\n", - "Increase the number of iterations to improve the convergence (max_iter=1000).\n", - "You might also want to scale the data as shown in:\n", - " https://scikit-learn.org/stable/modules/preprocessing.html\n", - "Please also refer to the documentation for alternative solver options:\n", - " https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n", - " n_iter_i = _check_optimize_result(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logistic Regression Accuracy: 0.7878\n", - "KNN Accuracy: 0.7608\n", - "Decision Tree Accuracy: 0.7338\n", - "Random Forest Accuracy: 0.7849\n" - ] - } - ], "source": [ - "from sklearn.linear_model import LogisticRegression\n", - "from sklearn.neighbors import KNeighborsClassifier\n", - "from sklearn.tree import DecisionTreeClassifier\n", - "from sklearn.ensemble import RandomForestClassifier\n", - "from sklearn.metrics import accuracy_score\n", - "\n", - "models = {\n", - " \"Logistic Regression\": LogisticRegression(max_iter=1000),\n", - " \"KNN\": KNeighborsClassifier(),\n", - " \"Decision Tree\": DecisionTreeClassifier(random_state=42),\n", - " \"Random Forest\": RandomForestClassifier(random_state=42)\n", - "}\n", - "\n", - "for name, model in models.items():\n", - "\n", - " model.fit(X_train, y_train)\n", - "\n", - " y_pred = model.predict(X_test)\n", - "\n", - " accuracy = accuracy_score(y_test, y_pred)\n", - "\n", - " print(name, \"Accuracy:\", round(accuracy, 4))" + "## Feature Scaling" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -1094,18 +732,38 @@ "4758 False True False " ] }, - "execution_count": 27, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ + "scaler = StandardScaler()\n", + "\n", + "X_train_scaled = X_train.copy()\n", + "X_test_scaled = X_test.copy()\n", + "\n", + "X_train_scaled[numerical_columns] = scaler.fit_transform(\n", + " X_train[numerical_columns]\n", + ")\n", + "\n", + "X_test_scaled[numerical_columns] = scaler.transform(\n", + " X_test[numerical_columns]\n", + ")\n", + "\n", "X_train_scaled.head()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Feature Selection" + ] + }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -1121,8 +779,6 @@ } ], "source": [ - "from sklearn.feature_selection import SelectKBest, f_classif\n", - "\n", "selector = SelectKBest(\n", " score_func=f_classif,\n", " k=8\n", @@ -1157,7 +813,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 24, "metadata": {}, "outputs": [ { @@ -2418,20 +2074,21 @@ "LogisticRegression(max_iter=1000, random_state=42)" ] }, - "execution_count": 30, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "from sklearn.linear_model import LogisticRegression\n", - "\n", "model = LogisticRegression(\n", " max_iter=1000,\n", " random_state=42\n", ")\n", "\n", - "model.fit(X_train_selected, y_train)" + "model.fit(\n", + " X_train_selected,\n", + " y_train\n", + ")" ] }, { @@ -2443,7 +2100,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 25, "metadata": {}, "outputs": [ { @@ -2455,13 +2112,19 @@ } ], "source": [ - "from sklearn.metrics import accuracy_score\n", - "\n", - "y_pred = model.predict(X_test_selected)\n", + "y_pred = model.predict(\n", + " X_test_selected\n", + ")\n", "\n", - "accuracy = accuracy_score(y_test, y_pred)\n", + "baseline_accuracy = accuracy_score(\n", + " y_test,\n", + " y_pred\n", + ")\n", "\n", - "print(\"Baseline Accuracy:\", round(accuracy, 4))" + "print(\n", + " \"Baseline Accuracy:\",\n", + " round(baseline_accuracy, 4)\n", + ")" ] }, { @@ -2487,7 +2150,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 29, "metadata": {}, "outputs": [ { @@ -2496,7 +2159,7 @@ "{'C': [0.01, 0.1, 1, 10, 100], 'solver': ['liblinear', 'lbfgs']}" ] }, - "execution_count": 32, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -2519,7 +2182,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -2532,8 +2195,6 @@ } ], "source": [ - "from sklearn.model_selection import GridSearchCV\n", - "\n", "grid_search = GridSearchCV(\n", " estimator=LogisticRegression(\n", " max_iter=1000,\n", @@ -2563,7 +2224,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 30, "metadata": {}, "outputs": [ { @@ -2587,8 +2248,15 @@ " y_pred_tuned\n", ")\n", "\n", - "print(\"Baseline Accuracy:\", round(accuracy, 4))\n", - "print(\"Tuned Accuracy:\", round(tuned_accuracy, 4))" + "print(\n", + " \"Baseline Accuracy:\",\n", + " round(baseline_accuracy, 4)\n", + ")\n", + "\n", + "print(\n", + " \"Tuned Accuracy:\",\n", + " round(tuned_accuracy, 4)\n", + ")" ] }, { @@ -2613,6 +2281,20 @@ "outputs": [], "source": [] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null,