diff --git a/lab-hyper-tuning.ipynb b/lab-hyper-tuning.ipynb index 847d487..3b045e5 100644 --- a/lab-hyper-tuning.ipynb +++ b/lab-hyper-tuning.ipynb @@ -35,23 +35,211 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 41, "metadata": {}, "outputs": [], "source": [ "#Libraries\n", "import pandas as pd\n", "import numpy as np\n", - "from sklearn.model_selection import train_test_split" + "from sklearn.model_selection import train_test_split\n", + "from sklearn.preprocessing import OneHotEncoder, StandardScaler\n", + "from sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier\n", + "from sklearn.ensemble import BaggingClassifier, RandomForestRegressor,AdaBoostClassifier, GradientBoostingClassifier, RandomForestClassifier \n", + "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n", + "from xgboost import XGBRegressor, XGBClassifier\n", + "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n", + "\n", + "\n", + "\n", + "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n", + "from sklearn.metrics import (\n", + " mean_absolute_error,\n", + " root_mean_squared_error,\n", + " r2_score\n", + ")\n", + "\n", + "SEED = 1" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 42, "metadata": {}, "outputs": [ { "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "PassengerId", + "rawType": "str", + "type": "string" + }, + { + "name": "HomePlanet", + "rawType": "str", + "type": "string" + }, + { + "name": "CryoSleep", + "rawType": "object", + "type": "unknown" + }, + { + "name": "Cabin", + "rawType": "str", + "type": "string" + }, + { + "name": "Destination", + "rawType": "str", + "type": "string" + }, + { + "name": "Age", + "rawType": "float64", + "type": "float" + }, + { + "name": "VIP", + "rawType": "object", + "type": "unknown" + }, + { + "name": "RoomService", + "rawType": "float64", + "type": "float" + }, + { + "name": "FoodCourt", + "rawType": "float64", + "type": "float" + }, + { + "name": "ShoppingMall", + "rawType": "float64", + "type": "float" + }, + { + "name": "Spa", + "rawType": "float64", + "type": "float" + }, + { + "name": "VRDeck", + "rawType": "float64", + "type": "float" + }, + { + "name": "Name", + "rawType": "str", + "type": "string" + }, + { + "name": "Transported", + "rawType": "bool", + "type": "boolean" + } + ], + "ref": "11d2f35b-6615-4395-8af6-f87ccdd39e3e", + "rows": [ + [ + "0", + "0001_01", + "Europa", + "False", + "B/0/P", + "TRAPPIST-1e", + "39.0", + "False", + "0.0", + "0.0", + "0.0", + "0.0", + "0.0", + "Maham Ofracculy", + "False" + ], + [ + "1", + "0002_01", + "Earth", + "False", + "F/0/S", + "TRAPPIST-1e", + "24.0", + "False", + "109.0", + "9.0", + "25.0", + "549.0", + "44.0", + "Juanna Vines", + "True" + ], + [ + "2", + "0003_01", + "Europa", + "False", + "A/0/S", + "TRAPPIST-1e", + "58.0", + "True", + "43.0", + "3576.0", + "0.0", + "6715.0", + "49.0", + "Altark Susent", + "False" + ], + [ + "3", + "0003_02", + "Europa", + "False", + "A/0/S", + "TRAPPIST-1e", + "33.0", + "False", + "0.0", + "1283.0", + "371.0", + "3329.0", + "193.0", + "Solam Susent", + "False" + ], + [ + "4", + "0004_01", + "Earth", + "False", + "F/1/S", + "TRAPPIST-1e", + "16.0", + "False", + "303.0", + "70.0", + "151.0", + "565.0", + "2.0", + "Willy Santantines", + "True" + ] + ], + "shape": { + "columns": 14, + "rows": 5 + } + }, "text/html": [ "
\n", "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
PassengerIdHomePlanetCryoSleepCabinDestinationAgeVIPRoomServiceFoodCourtShoppingMallSpaVRDeckNameTransported
00001_01EuropaFalseBTRAPPIST-1e39.0False0.00.00.00.00.0Maham OfracculyFalse
10002_01EarthFalseFTRAPPIST-1e24.0False109.09.025.0549.044.0Juanna VinesTrue
20003_01EuropaFalseATRAPPIST-1e58.0True43.03576.00.06715.049.0Altark SusentFalse
30003_02EuropaFalseATRAPPIST-1e33.0False0.01283.0371.03329.0193.0Solam SusentFalse
40004_01EarthFalseFTRAPPIST-1e16.0False303.070.0151.0565.02.0Willy SantantinesTrue
\n", + "
" + ], + "text/plain": [ + " PassengerId HomePlanet CryoSleep Cabin Destination Age VIP \\\n", + "0 0001_01 Europa False B TRAPPIST-1e 39.0 False \n", + "1 0002_01 Earth False F TRAPPIST-1e 24.0 False \n", + "2 0003_01 Europa False A TRAPPIST-1e 58.0 True \n", + "3 0003_02 Europa False A TRAPPIST-1e 33.0 False \n", + "4 0004_01 Earth False F TRAPPIST-1e 16.0 False \n", + "\n", + " RoomService FoodCourt ShoppingMall Spa VRDeck Name \\\n", + "0 0.0 0.0 0.0 0.0 0.0 Maham Ofracculy \n", + "1 109.0 9.0 25.0 549.0 44.0 Juanna Vines \n", + "2 43.0 3576.0 0.0 6715.0 49.0 Altark Susent \n", + "3 0.0 1283.0 371.0 3329.0 193.0 Solam Susent \n", + "4 303.0 70.0 151.0 565.0 2.0 Willy Santantines \n", + "\n", + " Transported \n", + "0 False \n", + "1 True \n", + "2 False \n", + "3 False \n", + "4 True " + ] + }, + "execution_count": 46, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Cabin is too granular - transform it in order to obtain {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'T'}\n", + "\n", + "spaceship[\"Cabin\"] = spaceship[\"Cabin\"].apply(lambda x: x.split(\"/\")[0])\n", + "spaceship.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": {}, + "outputs": [], + "source": [ + "# Drop PassengerId and Name\n", + "spaceship = spaceship.drop(columns=['PassengerId', 'Name'])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "- Now let's use the best model we got so far in order to see how it can improve when we fine tune it's hyperparameters." + "**Perform Train Test Split**" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "X_train shape: (5284, 11)\n", + "X_test shape: (1322, 11)\n" + ] + } + ], + "source": [ + "# Select the features (X) and the target (y)\n", + "X = spaceship[['HomePlanet', 'CryoSleep', 'Cabin', 'Destination', 'Age', 'VIP','RoomService', 'FoodCourt', 'ShoppingMall', 'Spa', 'VRDeck']]\n", + "y = spaceship['Transported']\n", + "\n", + "# ============================================================\n", + "# Train / Test Split\n", + "# ============================================================\n", + "\n", + "# Split the data into training (80%) and testing (20%)\n", + "# stratify=y keeps the same proportion of the target classes\n", + "# in both the training and test sets\n", + "\n", + "X_train, X_test, y_train, y_test = train_test_split(\n", + " X,\n", + " y,\n", + " test_size=0.2,\n", + " random_state=SEED,\n", + " stratify=y\n", + ")\n", + "\n", + "print(f\"X_train shape: {X_train.shape}\")\n", + "print(f\"X_test shape: {X_test.shape}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": {}, + "outputs": [], + "source": [ + "# ============================================================\n", + "# Separate Categorical and Numerical Features\n", + "# ============================================================\n", + "\n", + "# Categorical variables that need One-Hot Encoding\n", + "categorical_cols = [\n", + " 'HomePlanet',\n", + " 'CryoSleep',\n", + " 'Cabin',\n", + " 'Destination',\n", + " 'VIP'\n", + "]\n", + "\n", + "# Numerical variables that will be kept as numbers\n", + "numeric_cols = [\n", + " 'Age',\n", + " 'RoomService',\n", + " 'FoodCourt',\n", + " 'ShoppingMall',\n", + " 'Spa',\n", + " 'VRDeck'\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": {}, + "outputs": [], + "source": [ + "# ============================================================\n", + "# One-Hot Encoding\n", + "# ============================================================\n", + "\n", + "# Create the encoder\n", + "# drop='first' avoids multicollinearity\n", + "# handle_unknown='ignore' prevents errors if new categories\n", + "# appear in the test set\n", + "\n", + "encoder = OneHotEncoder(\n", + " sparse_output=False,\n", + " drop='first',\n", + " handle_unknown='ignore'\n", + ")\n", + "\n", + "# Learn the categories from the training data and transform it\n", + "X_train_cat = encoder.fit_transform(X_train[categorical_cols])\n", + "\n", + "# Apply the same transformation to the test data\n", + "X_test_cat = encoder.transform(X_test[categorical_cols])\n", + "\n", + "# Retrieve the names of the newly created columns\n", + "encoded_cols = encoder.get_feature_names_out(categorical_cols)\n", + "\n", + "# Convert the encoded arrays back into DataFrames\n", + "# (this keeps the column names and original row indexes)\n", + "\n", + "X_train_cat = pd.DataFrame(\n", + " X_train_cat,\n", + " columns=encoded_cols,\n", + " index=X_train.index\n", + ")\n", + "\n", + "X_test_cat = pd.DataFrame(\n", + " X_test_cat,\n", + " columns=encoded_cols,\n", + " index=X_test.index\n", + ")\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 51, "metadata": {}, "outputs": [], "source": [ - "#your code here" + "# ============================================================\n", + "# Keep the Numerical Features\n", + "# ============================================================\n", + "\n", + "# Extract the numerical columns from the original dataset\n", + "\n", + "X_train_num = X_train[numeric_cols]\n", + "X_test_num = X_test[numeric_cols]\n", + "\n", + "\n", + "# ============================================================\n", + "# Combine Numerical and Encoded Features\n", + "# ============================================================\n", + "\n", + "# Join the numerical variables with the encoded categorical ones\n", + "\n", + "X_train_final = pd.concat(\n", + " [X_train_num, X_train_cat],\n", + " axis=1\n", + ")\n", + "\n", + "X_test_final = pd.concat(\n", + " [X_test_num, X_test_cat],\n", + " axis=1\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " Age RoomService FoodCourt ShoppingMall Spa VRDeck \\\n", + "6608 -0.062685 -0.347862 -0.194775 -0.298866 1.109513 1.010481 \n", + "7683 0.696693 -0.318144 1.929981 -0.298866 0.101711 0.197504 \n", + "8083 -0.545925 -0.347862 -0.293514 -0.298866 -0.278132 -0.264883 \n", + "4011 1.801242 -0.347862 -0.137999 -0.297178 -0.278132 0.229701 \n", + "6539 -0.269788 -0.347862 0.587738 -0.298866 1.059890 -0.264883 \n", + "\n", + " HomePlanet_Europa HomePlanet_Mars CryoSleep_True Cabin_B Cabin_C \\\n", + "6608 1.704522 -0.513627 -0.742856 -0.328263 3.192948 \n", + "7683 1.704522 -0.513627 -0.742856 -0.328263 3.192948 \n", + "8083 -0.586675 1.946939 1.346155 -0.328263 -0.313190 \n", + "4011 -0.586675 -0.513627 -0.742856 -0.328263 -0.313190 \n", + "6539 -0.586675 -0.513627 -0.742856 -0.328263 -0.313190 \n", + "\n", + " Cabin_D Cabin_E Cabin_F Cabin_G Cabin_T \\\n", + "6608 -0.250502 -0.336339 -0.690808 -0.65253 -0.019459 \n", + "7683 -0.250502 -0.336339 -0.690808 -0.65253 -0.019459 \n", + "8083 -0.250502 2.973189 -0.690808 -0.65253 -0.019459 \n", + "4011 -0.250502 -0.336339 1.447580 -0.65253 -0.019459 \n", + "6539 -0.250502 -0.336339 1.447580 -0.65253 -0.019459 \n", + "\n", + " Destination_PSO J318.5-22 Destination_TRAPPIST-1e VIP_True \n", + "6608 -0.318264 -1.507930 -0.158191 \n", + "7683 -0.318264 0.663161 -0.158191 \n", + "8083 -0.318264 0.663161 -0.158191 \n", + "4011 -0.318264 0.663161 -0.158191 \n", + "6539 3.142042 -1.507930 -0.158191 \n", + "(5284, 19)\n", + "(1322, 19)\n" + ] + } + ], + "source": [ + "# ============================================================\n", + "# Feature Scaling\n", + "# ============================================================\n", + "\n", + "scaler = StandardScaler()\n", + "\n", + "# Return a pandas DataFrame instead of a NumPy array\n", + "scaler.set_output(transform=\"pandas\")\n", + "\n", + "# Compute the mean and standard deviation using ONLY the training set\n", + "# and standardize the training data\n", + "\n", + "X_train_final = scaler.fit_transform(X_train_final)\n", + "\n", + "# Apply the same scaling to the test data\n", + "\n", + "X_test_final = scaler.transform(X_test_final)\n", + "\n", + "\n", + "# ============================================================\n", + "# Check the Result\n", + "# ============================================================\n", + "\n", + "print(X_train_final.head())\n", + "print(X_train_final.shape)\n", + "print(X_test_final.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Model trainings" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Now let's use the best model we got so far in order to see how it can improve when we fine tune it's hyperparameters." + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model Trained!\n" + ] + } + ], + "source": [ + "#your code here\n", + "\n", + "xgb_clf = XGBClassifier(\n", + " n_estimators=300,\n", + " learning_rate=0.03,\n", + " max_depth=5,\n", + " min_child_weight=2,\n", + " subsample=0.9,\n", + " colsample_bytree=1.0,\n", + " gamma=0.5,\n", + " reg_alpha=0.1,\n", + " reg_lambda=1,\n", + " random_state=SEED,\n", + " eval_metric=\"logloss\"\n", + ")\n", + "\n", + "# Train the model\n", + "xgb_clf.fit(X_train_final, y_train)\n", + "\n", + "print(\"Model Trained!\")" ] }, { @@ -253,11 +1212,57 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 66, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training Accuracy: 84.33%\n", + "Testing Accuracy: 78.97%\n", + "\n", + " precision recall f1-score support\n", + "\n", + "Not Transported 0.80 0.77 0.78 656\n", + " Transported 0.78 0.81 0.80 666\n", + "\n", + " accuracy 0.79 1322\n", + " macro avg 0.79 0.79 0.79 1322\n", + " weighted avg 0.79 0.79 0.79 1322\n", + "\n" + ] + } + ], "source": [ - "#your code here" + "# ============================================================\n", + "# Make Predictions\n", + "# ============================================================\n", + "\n", + "y_pred_train = xgb_clf.predict(X_train_final)\n", + "y_pred_test = xgb_clf.predict(X_test_final)\n", + "\n", + "# ============================================================\n", + "# Evaluate the Model\n", + "# ============================================================\n", + "\n", + "accuracy_train = accuracy_score(y_train, y_pred_train)\n", + "print(f\"Training Accuracy: {accuracy_train * 100:.2f}%\")\n", + "\n", + "accuracy_test = accuracy_score(y_test, y_pred_test)\n", + "print(f\"Testing Accuracy: {accuracy_test * 100:.2f}%\")\n", + "\n", + "# Save baseline accuracies\n", + "baseline_train_accuracy = accuracy_train\n", + "baseline_test_accuracy = accuracy_test\n", + "\n", + "print()\n", + "\n", + "print(classification_report(\n", + " y_test,\n", + " y_pred_test,\n", + " target_names=[\"Not Transported\", \"Transported\"]\n", + "))" ] }, { @@ -283,11 +1288,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 60, "metadata": {}, "outputs": [], "source": [ - "#your code here" + "param_grid = {\n", + " \"n_estimators\": [200, 300],\n", + " \"learning_rate\": [0.03, 0.1],\n", + " \"max_depth\": [3, 5],\n", + " \"min_child_weight\": [1, 3],\n", + " \"subsample\": [0.8, 1.0],\n", + " \"colsample_bytree\": [0.8, 1.0],\n", + " \"gamma\": [0, 0.5],\n", + " \"reg_alpha\": [0, 0.1],\n", + " \"reg_lambda\": [1, 10]\n", + "}" ] }, { @@ -299,10 +1314,35 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 63, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Best Parameters: {'colsample_bytree': 1.0, 'gamma': 0, 'learning_rate': 0.03, 'max_depth': 5, 'min_child_weight': 3, 'n_estimators': 300, 'reg_alpha': 0.1, 'reg_lambda': 10, 'subsample': 1.0}\n", + "Best CV Score: 0.8120741862225587\n" + ] + } + ], + "source": [ + "grid = GridSearchCV(\n", + " estimator=XGBClassifier(\n", + " random_state=SEED,\n", + " eval_metric=\"logloss\"\n", + " ),\n", + " param_grid=param_grid,\n", + " cv=4,\n", + " scoring=\"accuracy\",\n", + " n_jobs=-1,\n", + ")\n", + "\n", + "grid.fit(X_train_final, y_train)\n", + "\n", + "print(\"Best Parameters:\", grid.best_params_)\n", + "print(\"Best CV Score:\", grid.best_score_)" + ] }, { "cell_type": "markdown", @@ -313,15 +1353,100 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 67, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training Accuracy: 82.89%\n", + "Testing Accuracy: 79.27%\n", + " precision recall f1-score support\n", + "\n", + "Not Transported 0.81 0.77 0.79 656\n", + " Transported 0.78 0.82 0.80 666\n", + "\n", + " accuracy 0.79 1322\n", + " macro avg 0.79 0.79 0.79 1322\n", + " weighted avg 0.79 0.79 0.79 1322\n", + "\n" + ] + } + ], + "source": [ + "best_xgb = grid.best_estimator_\n", + "\n", + "y_pred_train = best_xgb.predict(X_train_final)\n", + "y_pred_test = best_xgb.predict(X_test_final)\n", + "\n", + "print(f\"Training Accuracy: {accuracy_score(y_train, y_pred_train)*100:.2f}%\")\n", + "print(f\"Testing Accuracy: {accuracy_score(y_test, y_pred_test)*100:.2f}%\")\n", + "\n", + "accuracy_train = accuracy_score(y_train, y_pred_train)\n", + "accuracy_test = accuracy_score(y_test, y_pred_test)\n", + "\n", + "print(classification_report(\n", + " y_test,\n", + " y_pred_test,\n", + " target_names=[\"Not Transported\", \"Transported\"]\n", + "))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's compare them" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "=== Comparison ===\n", + "Training Accuracy: 84.33% → 82.89% (-1.44%)\n", + "Testing Accuracy : 78.97% → 79.27% (+0.30%)\n", + "\n", + "✅ Grid Search improved the model's generalization.\n" + ] + } + ], + "source": [ + "print(\"\\n=== Comparison ===\")\n", + "print(f\"Training Accuracy: {baseline_train_accuracy:.2%} → {accuracy_train:.2%} ({accuracy_train - baseline_train_accuracy:+.2%})\")\n", + "print(f\"Testing Accuracy : {baseline_test_accuracy:.2%} → {accuracy_test:.2%} ({accuracy_test - baseline_test_accuracy:+.2%})\")\n", + "\n", + "if accuracy_test > baseline_test_accuracy:\n", + " print(\"\\n✅ Grid Search improved the model's generalization.\")\n", + "else:\n", + " print(\"\\n⚠️ Grid Search did not improve the model's generalization.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Conclusion" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "After applying Grid Search, the XGBoost classifier achieved a 0.30% improvement in testing accuracy, while the training accuracy decreased by 1.44%. This indicates that the tuned model generalizes better to unseen data and is less prone to overfitting than the baseline model." + ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": ".venv (3.14.4)", "language": "python", "name": "python3" }, @@ -335,7 +1460,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.9" + "version": "3.14.4" } }, "nbformat": 4,