hyperparameters-tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b0c7acad",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "from sklearn.compose import make_column_transformer\n",
    "from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "import os\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6c9c16ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore') \n",
    "os.environ['PYTHONWARNINGS']='ignore'\n",
    "\n",
    "def warn(*args, **kwargs):\n",
    "    pass\n",
    "import warnings\n",
    "warnings.warn = warn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "417610c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataframe = pd.read_csv(\"processed.csv\")\n",
    "features = dataframe.columns.to_list()\n",
    "features.remove('Unnamed: 0')\n",
    "features.remove('HOME_TEAM_WINS')\n",
    "\n",
    "data = dataframe.values\n",
    "X = dataframe[features]\n",
    "y = dataframe.HOME_TEAM_WINS\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "896808ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score: 0.5712040272589736\n",
      "Best Hyperparameters: {'C': 1e-05, 'penalty': 'none', 'solver': 'lbfgs'}\n",
      "Best Score: 0.5670589495387663\n",
      "Best Hyperparameters: {'solver': 'liblinear', 'penalty': 'l2', 'C': 100}\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "model = LogisticRegression()\n",
    "\n",
    "#Search Space \n",
    "space = dict()\n",
    "space['solver'] = ['newton-cg', 'lbfgs', 'liblinear',]\n",
    "space['penalty'] = ['none', 'l1', 'l2', 'elasticnet']\n",
    "space['C'] = [1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10, 100, 500, 1000]\n",
    "space['max_iter']: [100, 500, 1000, 2000]\n",
    "\n",
    "#Grid Search \n",
    "grid_search = GridSearchCV(model, space, scoring='accuracy', n_jobs=-1)\n",
    "grid_result = grid_search.fit(X, y) #Replace x and y with input output variables later \n",
    "\n",
    "print('Best Score: %s' % grid_result.best_score_)\n",
    "print('Best Hyperparameters: %s' % grid_result.best_params_)\n",
    "\n",
    "#Random Search \n",
    "random_search = RandomizedSearchCV(model, space, scoring='accuracy', n_jobs=-1)\n",
    "random_result = random_search.fit(X, y) #Replace x and y with input output variables later \n",
    "\n",
    "print('Best Score: %s' % random_result.best_score_)\n",
    "print('Best Hyperparameters: %s' % random_result.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5e23d72d",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[5], line 25\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[39m# model = LogisticRegression()\u001b[39;00m\n\u001b[1;32m     16\u001b[0m \u001b[39m# grid_search = GridSearchCV(model, param_grid, cv=10, verbose=1,n_jobs=-1)\u001b[39;00m\n\u001b[1;32m     17\u001b[0m \u001b[39m# grid_search.fit(X_train, y_train)\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     22\u001b[0m \n\u001b[1;32m     23\u001b[0m \u001b[39m#Grid Search \u001b[39;00m\n\u001b[1;32m     24\u001b[0m grid_search \u001b[39m=\u001b[39m GridSearchCV(model, param_grid, scoring\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39maccuracy\u001b[39m\u001b[39m'\u001b[39m, n_jobs\u001b[39m=\u001b[39m\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m)\n\u001b[0;32m---> 25\u001b[0m grid_result \u001b[39m=\u001b[39m grid_search\u001b[39m.\u001b[39;49mfit(X_train, y_train) \u001b[39m#Replace x and y with input output variables later \u001b[39;00m\n\u001b[1;32m     27\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m'\u001b[39m\u001b[39mBest Score: \u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m'\u001b[39m \u001b[39m%\u001b[39m grid_result\u001b[39m.\u001b[39mbest_score_)\n\u001b[1;32m     28\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m'\u001b[39m\u001b[39mBest Hyperparameters: \u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m'\u001b[39m \u001b[39m%\u001b[39m grid_result\u001b[39m.\u001b[39mbest_params_)\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/sklearn/model_selection/_search.py:875\u001b[0m, in \u001b[0;36mBaseSearchCV.fit\u001b[0;34m(self, X, y, groups, **fit_params)\u001b[0m\n\u001b[1;32m    869\u001b[0m     results \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_format_results(\n\u001b[1;32m    870\u001b[0m         all_candidate_params, n_splits, all_out, all_more_results\n\u001b[1;32m    871\u001b[0m     )\n\u001b[1;32m    873\u001b[0m     \u001b[39mreturn\u001b[39;00m results\n\u001b[0;32m--> 875\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_run_search(evaluate_candidates)\n\u001b[1;32m    877\u001b[0m \u001b[39m# multimetric is determined here because in the case of a callable\u001b[39;00m\n\u001b[1;32m    878\u001b[0m \u001b[39m# self.scoring the return type is only known after calling\u001b[39;00m\n\u001b[1;32m    879\u001b[0m first_test_score \u001b[39m=\u001b[39m all_out[\u001b[39m0\u001b[39m][\u001b[39m\"\u001b[39m\u001b[39mtest_scores\u001b[39m\u001b[39m\"\u001b[39m]\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/sklearn/model_selection/_search.py:1389\u001b[0m, in \u001b[0;36mGridSearchCV._run_search\u001b[0;34m(self, evaluate_candidates)\u001b[0m\n\u001b[1;32m   1387\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_run_search\u001b[39m(\u001b[39mself\u001b[39m, evaluate_candidates):\n\u001b[1;32m   1388\u001b[0m     \u001b[39m\"\"\"Search all candidates in param_grid\"\"\"\u001b[39;00m\n\u001b[0;32m-> 1389\u001b[0m     evaluate_candidates(ParameterGrid(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mparam_grid))\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/sklearn/model_selection/_search.py:822\u001b[0m, in \u001b[0;36mBaseSearchCV.fit.<locals>.evaluate_candidates\u001b[0;34m(candidate_params, cv, more_results)\u001b[0m\n\u001b[1;32m    814\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mverbose \u001b[39m>\u001b[39m \u001b[39m0\u001b[39m:\n\u001b[1;32m    815\u001b[0m     \u001b[39mprint\u001b[39m(\n\u001b[1;32m    816\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mFitting \u001b[39m\u001b[39m{0}\u001b[39;00m\u001b[39m folds for each of \u001b[39m\u001b[39m{1}\u001b[39;00m\u001b[39m candidates,\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    817\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39m totalling \u001b[39m\u001b[39m{2}\u001b[39;00m\u001b[39m fits\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m.\u001b[39mformat(\n\u001b[1;32m    818\u001b[0m             n_splits, n_candidates, n_candidates \u001b[39m*\u001b[39m n_splits\n\u001b[1;32m    819\u001b[0m         )\n\u001b[1;32m    820\u001b[0m     )\n\u001b[0;32m--> 822\u001b[0m out \u001b[39m=\u001b[39m parallel(\n\u001b[1;32m    823\u001b[0m     delayed(_fit_and_score)(\n\u001b[1;32m    824\u001b[0m         clone(base_estimator),\n\u001b[1;32m    825\u001b[0m         X,\n\u001b[1;32m    826\u001b[0m         y,\n\u001b[1;32m    827\u001b[0m         train\u001b[39m=\u001b[39;49mtrain,\n\u001b[1;32m    828\u001b[0m         test\u001b[39m=\u001b[39;49mtest,\n\u001b[1;32m    829\u001b[0m         parameters\u001b[39m=\u001b[39;49mparameters,\n\u001b[1;32m    830\u001b[0m         split_progress\u001b[39m=\u001b[39;49m(split_idx, n_splits),\n\u001b[1;32m    831\u001b[0m         candidate_progress\u001b[39m=\u001b[39;49m(cand_idx, n_candidates),\n\u001b[1;32m    832\u001b[0m         \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mfit_and_score_kwargs,\n\u001b[1;32m    833\u001b[0m     )\n\u001b[1;32m    834\u001b[0m     \u001b[39mfor\u001b[39;49;00m (cand_idx, parameters), (split_idx, (train, test)) \u001b[39min\u001b[39;49;00m product(\n\u001b[1;32m    835\u001b[0m         \u001b[39menumerate\u001b[39;49m(candidate_params), \u001b[39menumerate\u001b[39;49m(cv\u001b[39m.\u001b[39;49msplit(X, y, groups))\n\u001b[1;32m    836\u001b[0m     )\n\u001b[1;32m    837\u001b[0m )\n\u001b[1;32m    839\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(out) \u001b[39m<\u001b[39m \u001b[39m1\u001b[39m:\n\u001b[1;32m    840\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\n\u001b[1;32m    841\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mNo fits were performed. \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    842\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mWas the CV iterator empty? \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    843\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mWere there no candidates?\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    844\u001b[0m     )\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/joblib/parallel.py:1061\u001b[0m, in \u001b[0;36mParallel.__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m   1058\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_iterating \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n\u001b[1;32m   1060\u001b[0m \u001b[39mwith\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backend\u001b[39m.\u001b[39mretrieval_context():\n\u001b[0;32m-> 1061\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mretrieve()\n\u001b[1;32m   1062\u001b[0m \u001b[39m# Make sure that we get a last message telling us we are done\u001b[39;00m\n\u001b[1;32m   1063\u001b[0m elapsed_time \u001b[39m=\u001b[39m time\u001b[39m.\u001b[39mtime() \u001b[39m-\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_start_time\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/joblib/parallel.py:938\u001b[0m, in \u001b[0;36mParallel.retrieve\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    936\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m    937\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mgetattr\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backend, \u001b[39m'\u001b[39m\u001b[39msupports_timeout\u001b[39m\u001b[39m'\u001b[39m, \u001b[39mFalse\u001b[39;00m):\n\u001b[0;32m--> 938\u001b[0m         \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_output\u001b[39m.\u001b[39mextend(job\u001b[39m.\u001b[39;49mget(timeout\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtimeout))\n\u001b[1;32m    939\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    940\u001b[0m         \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_output\u001b[39m.\u001b[39mextend(job\u001b[39m.\u001b[39mget())\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/joblib/_parallel_backends.py:542\u001b[0m, in \u001b[0;36mLokyBackend.wrap_future_result\u001b[0;34m(future, timeout)\u001b[0m\n\u001b[1;32m    539\u001b[0m \u001b[39m\"\"\"Wrapper for Future.result to implement the same behaviour as\u001b[39;00m\n\u001b[1;32m    540\u001b[0m \u001b[39mAsyncResults.get from multiprocessing.\"\"\"\u001b[39;00m\n\u001b[1;32m    541\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 542\u001b[0m     \u001b[39mreturn\u001b[39;00m future\u001b[39m.\u001b[39;49mresult(timeout\u001b[39m=\u001b[39;49mtimeout)\n\u001b[1;32m    543\u001b[0m \u001b[39mexcept\u001b[39;00m CfTimeoutError \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m    544\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mTimeoutError\u001b[39;00m \u001b[39mfrom\u001b[39;00m \u001b[39me\u001b[39;00m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/concurrent/futures/_base.py:441\u001b[0m, in \u001b[0;36mFuture.result\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    438\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_state \u001b[39m==\u001b[39m FINISHED:\n\u001b[1;32m    439\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__get_result()\n\u001b[0;32m--> 441\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_condition\u001b[39m.\u001b[39;49mwait(timeout)\n\u001b[1;32m    443\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_state \u001b[39min\u001b[39;00m [CANCELLED, CANCELLED_AND_NOTIFIED]:\n\u001b[1;32m    444\u001b[0m     \u001b[39mraise\u001b[39;00m CancelledError()\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/threading.py:312\u001b[0m, in \u001b[0;36mCondition.wait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    310\u001b[0m \u001b[39mtry\u001b[39;00m:    \u001b[39m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[39;00m\n\u001b[1;32m    311\u001b[0m     \u001b[39mif\u001b[39;00m timeout \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m--> 312\u001b[0m         waiter\u001b[39m.\u001b[39;49macquire()\n\u001b[1;32m    313\u001b[0m         gotit \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[1;32m    314\u001b[0m     \u001b[39melse\u001b[39;00m:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "param_grid = [\n",
    "    {\n",
    "        'solver': ['newton-cg', 'lbfgs', 'liblinear',],\n",
    "        'penalty':['none', 'l1', 'l2', 'elasticnet'],\n",
    "        'C': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10, 100, 500, 1000],\n",
    "        'max_iter': [100, 500, 1000, 2000]\n",
    "    }\n",
    "#     {\n",
    "#         \"preprocesser__num_transform__imputer__strategy\": [\"mean\", \"median\"],\n",
    "#         \"classifier__n_estimators\": [10, 100, 1000],\n",
    "#         \"classifier\": [RandomForestClassifier()]\n",
    "#     }\n",
    "]\n",
    "\n",
    "# model = LogisticRegression()\n",
    "# grid_search = GridSearchCV(model, param_grid, cv=10, verbose=1,n_jobs=-1)\n",
    "# grid_search.fit(X_train, y_train)\n",
    "\n",
    "# model = RandomForestClassifier()\n",
    "# grid_search = GridSearchCV(model, param_grid, cv=10, verbose=1,n_jobs=-1)\n",
    "# grid_search.fit(X_train, y_train)\n",
    "\n",
    "#Grid Search \n",
    "grid_search = GridSearchCV(model, param_grid, scoring='accuracy', n_jobs=-1)\n",
    "grid_result = grid_search.fit(X_train, y_train) #Replace x and y with input output variables later \n",
    "\n",
    "print('Best Score: %s' % grid_result.best_score_)\n",
    "print('Best Hyperparameters: %s' % grid_result.best_params_)\n",
    "\n",
    "#Random Search \n",
    "random_search = RandomizedSearchCV(model, param_grid, scoring='accuracy', n_jobs=-1)\n",
    "random_result = random_search.fit(X_train, y_train) #Replace x and y with input output variables later \n",
    "\n",
    "print('Best Score: %s' % random_result.best_score_)\n",
    "print('Best Hyperparameters: %s' % random_result.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a55e7e71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6010362694300517"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ct = make_column_transformer(\n",
    "       (OneHotEncoder(handle_unknown='ignore', sparse=False), features),\n",
    "       remainder = 'passthrough'\n",
    ")\n",
    "\n",
    "# lgr = LogisticRegression(C=1e-05, penalty='none', solver='lbfgs')\n",
    "lgr = LogisticRegression()\n",
    "\n",
    "p1 = make_pipeline(ct, lgr)\n",
    "\n",
    "cross_val_score( p1, X_train , y_train, cv=2, scoring='accuracy').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf77d55a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}