diff --git a/lab-hyper-tuning.ipynb b/lab-hyper-tuning.ipynb index 847d487..c2b4e1b 100644 --- a/lab-hyper-tuning.ipynb +++ b/lab-hyper-tuning.ipynb @@ -35,19 +35,22 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 43, "metadata": {}, "outputs": [], "source": [ "#Libraries\n", "import pandas as pd\n", "import numpy as np\n", - "from sklearn.model_selection import train_test_split" + "from sklearn.model_selection import train_test_split\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from scipy.stats import chi2_contingency\n", + "SEED=1" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 44, "metadata": {}, "outputs": [ { @@ -200,7 +203,7 @@ "4 True " ] }, - "execution_count": 2, + "execution_count": 44, "metadata": {}, "output_type": "execute_result" } @@ -221,11 +224,232 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 45, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
AgeRoomServiceFoodCourtShoppingMallSpaVRDeckHomePlanet_EarthHomePlanet_EuropaHomePlanet_MarsCryoSleep_False...Cabin_DCabin_ECabin_FCabin_GCabin_TDestination_55 Cancri eDestination_PSO J318.5-22Destination_TRAPPIST-1eVIP_FalseVIP_True
039.00.00.00.00.00.00101...0000000110
124.0109.09.025.0549.044.01001...0010000110
258.043.03576.00.06715.049.00101...0000000101
333.00.01283.0371.03329.0193.00101...0000000110
416.0303.070.0151.0565.02.01001...0010000110
\n", + "

5 rows × 24 columns

\n", + "
" + ], + "text/plain": [ + " Age RoomService FoodCourt ShoppingMall Spa VRDeck \\\n", + "0 39.0 0.0 0.0 0.0 0.0 0.0 \n", + "1 24.0 109.0 9.0 25.0 549.0 44.0 \n", + "2 58.0 43.0 3576.0 0.0 6715.0 49.0 \n", + "3 33.0 0.0 1283.0 371.0 3329.0 193.0 \n", + "4 16.0 303.0 70.0 151.0 565.0 2.0 \n", + "\n", + " HomePlanet_Earth HomePlanet_Europa HomePlanet_Mars CryoSleep_False ... \\\n", + "0 0 1 0 1 ... \n", + "1 1 0 0 1 ... \n", + "2 0 1 0 1 ... \n", + "3 0 1 0 1 ... \n", + "4 1 0 0 1 ... \n", + "\n", + " Cabin_D Cabin_E Cabin_F Cabin_G Cabin_T Destination_55 Cancri e \\\n", + "0 0 0 0 0 0 0 \n", + "1 0 0 1 0 0 0 \n", + "2 0 0 0 0 0 0 \n", + "3 0 0 0 0 0 0 \n", + "4 0 0 1 0 0 0 \n", + "\n", + " Destination_PSO J318.5-22 Destination_TRAPPIST-1e VIP_False VIP_True \n", + "0 0 1 1 0 \n", + "1 0 1 1 0 \n", + "2 0 1 0 1 \n", + "3 0 1 1 0 \n", + "4 0 1 1 0 \n", + "\n", + "[5 rows x 24 columns]" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "#your code here" + "spaceship.dropna(inplace=True)\n", + "\n", + "X_train_spaceship_cabin = spaceship.copy()\n", + "\n", + "X_train_spaceship_cabin['Cabin'] = (\n", + " X_train_spaceship_cabin['Cabin'].str.split('/').str[0]\n", + ")\n", + "\n", + "X_train_spaceship_cabin['Cabin'].value_counts(dropna=False)\n", + "X_train_spaceship_cabin = X_train_spaceship_cabin.drop(columns=['PassengerId', 'Name'])\n", + "X = X_train_spaceship_cabin.drop(columns='Transported')\n", + "y = X_train_spaceship_cabin['Transported'].astype(int)\n", + "\n", + "X = pd.get_dummies(X, dtype=int)\n", + "X.head()" ] }, { @@ -237,11 +461,57 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 46, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "X_train shape: (5284, 24)\n", + "X_test shape: (1322, 24)\n" + ] + } + ], "source": [ - "#your code here" + "# Step 2: Train / Test Split\n", + "# ---------------------------\n", + "# Train / Test Split (without stratify)\n", + "X_train, X_test, y_train, y_test = train_test_split(\n", + " X, y,\n", + " test_size=0.2,\n", + " random_state=SEED)\n", + "print(f'X_train shape: {X_train.shape}')\n", + "print(f'X_test shape: {X_test.shape}')" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model Trained!\n" + ] + } + ], + "source": [ + "# XGBoost NEEDS TUNING to show its power:\n", + "from xgboost import XGBRegressor, XGBClassifier\n", + "xgb_clf = XGBClassifier(\n", + " n_estimators=500,\n", + " learning_rate=0.05,\n", + " subsample=0.8,\n", + " colsample_bytree=0.8,\n", + " reg_lambda=10,\n", + " random_state=SEED,\n", + " eval_metric='logloss')\n", + "\n", + "xgb_clf.fit(X_train, y_train)\n", + "print(\"Model Trained!\")" ] }, { @@ -253,11 +523,45 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 48, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy (Train): 87.40%\n", + "Accuracy (Test): 78.97%\n", + "\n", + "Classification Report (Test):\n", + " precision recall f1-score support\n", + "\n", + " 0 0.79 0.77 0.78 648\n", + " 1 0.79 0.80 0.80 674\n", + "\n", + " accuracy 0.79 1322\n", + " macro avg 0.79 0.79 0.79 1322\n", + "weighted avg 0.79 0.79 0.79 1322\n", + "\n" + ] + } + ], "source": [ - "#your code here" + "#your code here# Step 5 - Results\n", + "# -------------------\n", + "from sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n", + "\n", + "y_pred_train = xgb_clf.predict(X_train)\n", + "y_pred_test = xgb_clf.predict(X_test)\n", + "\n", + "acc_train = accuracy_score(y_train, y_pred_train)\n", + "acc_test = accuracy_score(y_test, y_pred_test)\n", + "\n", + "print(f'Accuracy (Train): {acc_train * 100:.2f}%')\n", + "print(f'Accuracy (Test): {acc_test * 100:.2f}%')\n", + "print()\n", + "print('Classification Report (Test):')\n", + "print(classification_report(y_test, y_pred_test))" ] }, { @@ -283,11 +587,42 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 49, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model Trained!\n" + ] + } + ], "source": [ - "#your code here" + "# The grid: the same 5 hyperparameters we actually tuned by hand in Section 4\n", + "from sklearn.model_selection import RandomizedSearchCV\n", + "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n", + "\n", + "\n", + "grid = {\n", + " 'n_estimators': [200, 300, 500, 600],\n", + " 'learning_rate': [0.01, 0.03, 0.05, 0.1],\n", + " 'subsample': [0.7, 0.8, 1.0],\n", + " 'colsample_bytree': [0.6, 0.8, 1.0],\n", + " 'reg_lambda': [1, 3, 10, 20]}\n", + "xgb_clf = XGBClassifier(random_state=SEED)\n", + "\n", + "# cv=4 --> same Cross Validation with K=4\n", + "# n_iter=10 --> instead of trying all combinations, only 10 random ones are sampled\n", + "model = RandomizedSearchCV(estimator=xgb_clf,\n", + " param_distributions=grid,\n", + " n_iter=10,\n", + " cv=4,\n", + " random_state=SEED, # So all the random iterations are the same\n", + " n_jobs=-1) # n_jobs=-1 --> allows multicore CPU processing\n", + "model.fit(X_train, y_train)\n", + "\n", + "print(\"Model Trained!\")" ] }, { @@ -311,6 +646,155 @@ "- Evaluate your model" ] }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Best hyperparameters found:\n", + "{'subsample': 1.0, 'reg_lambda': 1, 'n_estimators': 500, 'learning_rate': 0.01, 'colsample_bytree': 1.0}\n", + "Accuracy (Train): 83.74%\n", + "Accuracy (Test): 80.03%\n", + "\n", + "Classification Report (Test):\n", + " precision recall f1-score support\n", + "\n", + " 0 0.81 0.77 0.79 648\n", + " 1 0.79 0.83 0.81 674\n", + "\n", + " accuracy 0.80 1322\n", + " macro avg 0.80 0.80 0.80 1322\n", + "weighted avg 0.80 0.80 0.80 1322\n", + "\n" + ] + } + ], + "source": [ + "\n", + "\n", + "\n", + "\n", + "from sklearn.metrics import accuracy_score, classification_report\n", + "\n", + "print(\"Best hyperparameters found:\")\n", + "print(model.best_params_)\n", + "\n", + "best_xgb_clf = model.best_estimator_\n", + "\n", + "y_pred_train = best_xgb_clf.predict(X_train)\n", + "y_pred_test = best_xgb_clf.predict(X_test)\n", + "\n", + "acc_train = accuracy_score(y_train, y_pred_train)\n", + "acc_test = accuracy_score(y_test, y_pred_test)\n", + "\n", + "print(f\"Accuracy (Train): {acc_train * 100:.2f}%\")\n", + "print(f\"Accuracy (Test): {acc_test * 100:.2f}%\")\n", + "print()\n", + "print(\"Classification Report (Test):\")\n", + "print(classification_report(y_test, y_pred_test))" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model Trained!\n", + "Best parameters: {'colsample_bytree': 1.0, 'learning_rate': 0.01, 'n_estimators': 600, 'reg_lambda': 1, 'subsample': 1.0}\n", + "Best CV accuracy: 0.8046934140802422\n" + ] + } + ], + "source": [ + "from sklearn.model_selection import train_test_split, GridSearchCV\n", + "\n", + "\n", + "\n", + "from sklearn.model_selection import GridSearchCV\n", + "from xgboost import XGBClassifier\n", + "\n", + "grid = {\n", + " \"n_estimators\": [600, 800, 900],\n", + " \"learning_rate\": [0.01],\n", + " \"subsample\": [1.0],\n", + " \"colsample_bytree\": [0.5, 1.0],\n", + " \"reg_lambda\": [1],\n", + "}\n", + "\n", + "xgb_clf = XGBClassifier(\n", + " random_state=SEED,\n", + " eval_metric=\"logloss\"\n", + ")\n", + "\n", + "model = GridSearchCV(\n", + " estimator=xgb_clf,\n", + " param_grid=grid,\n", + " scoring=\"accuracy\",\n", + " cv=4,\n", + " n_jobs=-1\n", + ")\n", + "\n", + "model.fit(X_train, y_train)\n", + "\n", + "print(\"Model Trained!\")\n", + "print(\"Best parameters:\", model.best_params_)\n", + "print(\"Best CV accuracy:\", model.best_score_)" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Best hyperparameters found:\n", + "{'colsample_bytree': 1.0, 'learning_rate': 0.01, 'n_estimators': 600, 'reg_lambda': 1, 'subsample': 1.0}\n", + "Accuracy (Train): 83.82%\n", + "Accuracy (Test): 79.80%\n", + "\n", + "Classification Report (Test):\n", + " precision recall f1-score support\n", + "\n", + " 0 0.81 0.76 0.79 648\n", + " 1 0.78 0.83 0.81 674\n", + "\n", + " accuracy 0.80 1322\n", + " macro avg 0.80 0.80 0.80 1322\n", + "weighted avg 0.80 0.80 0.80 1322\n", + "\n" + ] + } + ], + "source": [ + "print(\"Best hyperparameters found:\")\n", + "print(model.best_params_)\n", + "\n", + "best_xgb_clf = model.best_estimator_\n", + "\n", + "y_pred_train = best_xgb_clf.predict(X_train)\n", + "y_pred_test = best_xgb_clf.predict(X_test)\n", + "\n", + "acc_train = accuracy_score(y_train, y_pred_train)\n", + "acc_test = accuracy_score(y_test, y_pred_test)\n", + "\n", + "print(f\"Accuracy (Train): {acc_train * 100:.2f}%\")\n", + "print(f\"Accuracy (Test): {acc_test * 100:.2f}%\")\n", + "print()\n", + "print(\"Classification Report (Test):\")\n", + "print(classification_report(y_test, y_pred_test))" + ] + }, { "cell_type": "code", "execution_count": null, @@ -321,7 +805,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "base", "language": "python", "name": "python3" }, @@ -335,7 +819,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.9" + "version": "3.13.9" } }, "nbformat": 4,