classical.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/steven/Library/Python/3.9/lib/python/site-packages/urllib3/__init__.py:34: NotOpenSSLWarning: urllib3 v2.0 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of features in the first data point: 73030\n",
      "(799, 73030) (100, 73030) (100, 73030)\n"
     ]
    }
   ],
   "source": [
    "from util import load_numpy_data_from_split\n",
    "X_tr, y_tr, X_val, y_val, X_te, y_te = load_numpy_data_from_split(resize_images=False)\n",
    "num_features_in_first_data_point = X_tr[0].shape[0]\n",
    "print(\"Number of features in the first data point:\", num_features_in_first_data_point)\n",
    "print(X_tr.shape, X_val.shape, X_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Explained variance ratio:\n",
      " [2.05311984e-01 4.17890027e-02 2.47591455e-02 1.37391975e-02\n",
      " 1.05562033e-02 8.82426184e-03 7.28303427e-03 5.90902101e-03\n",
      " 5.42792445e-03 5.03229350e-03 4.79841791e-03 4.59604943e-03\n",
      " 4.24889009e-03 4.09477344e-03 3.99180967e-03 3.65215773e-03\n",
      " 3.61169269e-03 3.52519331e-03 3.41946958e-03 3.33110359e-03\n",
      " 3.25680850e-03 3.13063036e-03 3.06395674e-03 2.96723121e-03\n",
      " 2.95070210e-03 2.92282458e-03 2.89649726e-03 2.83088814e-03\n",
      " 2.77206488e-03 2.71362788e-03 2.65356433e-03 2.64333980e-03\n",
      " 2.63070664e-03 2.57565523e-03 2.55641225e-03 2.53908359e-03\n",
      " 2.47714343e-03 2.46588560e-03 2.45357724e-03 2.43100803e-03\n",
      " 2.41284468e-03 2.37898785e-03 2.34946096e-03 2.33810814e-03\n",
      " 2.31383601e-03 2.30114651e-03 2.29048287e-03 2.27103638e-03\n",
      " 2.24809023e-03 2.22032773e-03 2.17491342e-03 2.15789001e-03\n",
      " 2.14207382e-03 2.13187397e-03 2.11567921e-03 2.10133102e-03\n",
      " 2.09160196e-03 2.07458483e-03 2.05421285e-03 2.03801575e-03\n",
      " 2.02954444e-03 2.00637337e-03 1.98919978e-03 1.97551609e-03\n",
      " 1.96647551e-03 1.95444957e-03 1.94218161e-03 1.93183415e-03\n",
      " 1.90526864e-03 1.89762434e-03 1.88313762e-03 1.87686656e-03\n",
      " 1.86322618e-03 1.84938661e-03 1.84037746e-03 1.82184833e-03\n",
      " 1.82070001e-03 1.80562772e-03 1.78277737e-03 1.77498988e-03\n",
      " 1.76989613e-03 1.76213041e-03 1.75526936e-03 1.74381479e-03\n",
      " 1.73542451e-03 1.72440999e-03 1.72142661e-03 1.70722394e-03\n",
      " 1.69264793e-03 1.68664171e-03 1.66934263e-03 1.66869652e-03\n",
      " 1.66468660e-03 1.66347669e-03 1.64828822e-03 1.64323836e-03\n",
      " 1.63745845e-03 1.62259291e-03 1.61612430e-03 1.60693261e-03\n",
      " 1.59906282e-03 1.58919790e-03 1.58583175e-03 1.57446368e-03\n",
      " 1.56768144e-03 1.56144460e-03 1.55368785e-03 1.54458545e-03\n",
      " 1.54260569e-03 1.53672416e-03 1.53250643e-03 1.51566404e-03\n",
      " 1.50700146e-03 1.50325592e-03 1.50107965e-03 1.49652478e-03\n",
      " 1.48674019e-03 1.48453261e-03 1.47715106e-03 1.47111597e-03\n",
      " 1.46371208e-03 1.45395345e-03 1.44509075e-03 1.43701746e-03\n",
      " 1.43256306e-03 1.42843626e-03 1.42461678e-03 1.42127392e-03\n",
      " 1.41464104e-03 1.41260657e-03 1.40819815e-03 1.39467639e-03\n",
      " 1.39315694e-03 1.38888054e-03 1.38329109e-03 1.37630478e-03\n",
      " 1.37443887e-03 1.36866013e-03 1.35821407e-03 1.35653175e-03\n",
      " 1.35070842e-03 1.34569441e-03 1.33941066e-03 1.33510935e-03\n",
      " 1.33015600e-03 1.32686272e-03 1.32090354e-03 1.31278206e-03\n",
      " 1.30573800e-03 1.30327290e-03 1.30054494e-03 1.29300903e-03\n",
      " 1.29092287e-03 1.28639501e-03 1.28370035e-03 1.28100999e-03\n",
      " 1.27772777e-03 1.27158198e-03 1.26577087e-03 1.26319507e-03\n",
      " 1.25683181e-03 1.25373283e-03 1.24895340e-03 1.24813616e-03\n",
      " 1.24085194e-03 1.23669067e-03 1.23241346e-03 1.22980250e-03\n",
      " 1.22270710e-03 1.21900486e-03 1.21274858e-03 1.20937394e-03\n",
      " 1.20407529e-03 1.20311778e-03 1.19720213e-03 1.19699398e-03\n",
      " 1.19084166e-03 1.18620484e-03 1.18185254e-03 1.17778033e-03\n",
      " 1.17434922e-03 1.17216085e-03 1.16758165e-03 1.16682623e-03\n",
      " 1.16503751e-03 1.16046588e-03 1.15634350e-03 1.15316419e-03\n",
      " 1.15151948e-03 1.14957045e-03 1.14422839e-03 1.14347052e-03\n",
      " 1.14134408e-03 1.13719783e-03 1.12721801e-03 1.12447969e-03\n",
      " 1.12136896e-03 1.11776555e-03 1.11475796e-03 1.11263781e-03\n",
      " 1.11013139e-03 1.10431528e-03 1.10297650e-03 1.09717017e-03\n",
      " 1.09511614e-03 1.09419180e-03 1.09002797e-03 1.08660746e-03\n",
      " 1.08354166e-03 1.07933639e-03 1.07683660e-03 1.07462029e-03\n",
      " 1.07306626e-03 1.06743199e-03 1.06689113e-03 1.06372393e-03\n",
      " 1.05632714e-03 1.05440547e-03 1.05279498e-03 1.04884966e-03\n",
      " 1.04724406e-03 1.04667526e-03 1.04298862e-03 1.03843946e-03\n",
      " 1.03643094e-03 1.03337725e-03 1.03030063e-03 1.02565379e-03\n",
      " 1.02361804e-03 1.02037936e-03 1.02007657e-03 1.01604115e-03\n",
      " 1.01505476e-03 1.01235672e-03 1.01143878e-03 1.00541534e-03\n",
      " 1.00471522e-03 1.00240181e-03 9.98707139e-04 9.96831106e-04\n",
      " 9.94938426e-04 9.89449443e-04 9.88164567e-04 9.87727777e-04\n",
      " 9.81371617e-04 9.79010714e-04 9.76654585e-04 9.71473113e-04\n",
      " 9.68444627e-04 9.65742278e-04 9.64461127e-04 9.62966122e-04\n",
      " 9.61107784e-04 9.58386518e-04 9.55376192e-04 9.52835311e-04\n",
      " 9.50292801e-04 9.46393178e-04 9.44705564e-04 9.40702215e-04\n",
      " 9.39654652e-04 9.36593569e-04 9.35335760e-04 9.31218732e-04\n",
      " 9.28524765e-04 9.27090645e-04 9.24746681e-04 9.20241175e-04\n",
      " 9.19623126e-04 9.14240198e-04 9.13530646e-04 9.11127892e-04\n",
      " 9.08931892e-04 9.06796078e-04 9.05939669e-04 9.03102977e-04\n",
      " 9.01626772e-04 8.98058526e-04 8.96362006e-04 8.93966004e-04\n",
      " 8.92005803e-04 8.90061492e-04 8.88296927e-04 8.87267117e-04\n",
      " 8.83747591e-04 8.79933359e-04 8.78688588e-04 8.77936254e-04\n",
      " 8.76590959e-04 8.73869751e-04 8.72089178e-04 8.69588985e-04\n",
      " 8.67289840e-04 8.66102870e-04 8.63573048e-04 8.60028202e-04\n",
      " 8.56170664e-04 8.54098587e-04 8.52281286e-04 8.48880853e-04\n",
      " 8.47063959e-04 8.46402836e-04 8.42128182e-04 8.41386500e-04\n",
      " 8.37650965e-04 8.36080173e-04 8.34334234e-04 8.33068509e-04\n",
      " 8.32090387e-04 8.30054632e-04 8.26764270e-04 8.23340204e-04\n",
      " 8.22354632e-04 8.19569745e-04 8.19004083e-04 8.17156746e-04\n",
      " 8.15937063e-04 8.13643448e-04 8.11543490e-04 8.10110010e-04\n",
      " 8.07570992e-04 8.06364289e-04 8.04482843e-04 8.01378512e-04\n",
      " 7.98940833e-04 7.97560904e-04 7.95643777e-04 7.92951032e-04\n",
      " 7.92237464e-04 7.88925041e-04 7.86412042e-04 7.86131714e-04\n",
      " 7.84524658e-04 7.82525167e-04 7.80975330e-04 7.78300338e-04\n",
      " 7.75261957e-04 7.73179228e-04 7.70912971e-04 7.70145620e-04\n",
      " 7.69341481e-04 7.68150610e-04 7.65547389e-04 7.63358315e-04\n",
      " 7.62924436e-04 7.59823830e-04 7.58157577e-04 7.57750997e-04\n",
      " 7.54813664e-04 7.51355255e-04 7.50606589e-04 7.47976301e-04\n",
      " 7.46320176e-04 7.45234254e-04 7.44755904e-04 7.42167525e-04\n",
      " 7.40994699e-04 7.38694973e-04 7.38379138e-04 7.37132854e-04\n",
      " 7.35092675e-04 7.33493071e-04 7.31236360e-04 7.29941181e-04\n",
      " 7.29170744e-04 7.24923506e-04 7.22654222e-04 7.21952936e-04\n",
      " 7.20235577e-04 7.16391485e-04 7.15429371e-04 7.13599904e-04\n",
      " 7.11367175e-04 7.10216176e-04 7.08060339e-04 7.07273139e-04\n",
      " 7.05128128e-04 7.03153491e-04 7.01866520e-04 7.01521989e-04\n",
      " 7.01277575e-04 6.99543976e-04 6.98044430e-04 6.96234929e-04\n",
      " 6.92946604e-04 6.92164060e-04 6.90970803e-04 6.89582375e-04\n",
      " 6.88539469e-04 6.87271182e-04 6.85316161e-04 6.83647406e-04\n",
      " 6.82434998e-04 6.79700286e-04 6.78859651e-04 6.77388394e-04\n",
      " 6.75003219e-04 6.71384798e-04 6.71321934e-04 6.68549328e-04\n",
      " 6.67023531e-04 6.65459665e-04 6.65176718e-04 6.63801737e-04\n",
      " 6.61546306e-04 6.59336569e-04 6.58506120e-04 6.56014367e-04\n",
      " 6.54956035e-04 6.54569885e-04 6.52291055e-04 6.51648908e-04\n",
      " 6.49057678e-04 6.47478562e-04 6.47017499e-04 6.46070519e-04\n",
      " 6.44230691e-04 6.43425272e-04 6.42440224e-04 6.41910650e-04\n",
      " 6.39831065e-04 6.37497229e-04 6.36678364e-04 6.35406584e-04\n",
      " 6.34083932e-04 6.31912670e-04 6.30426686e-04 6.28658396e-04\n",
      " 6.27626257e-04 6.25922519e-04 6.25318033e-04 6.22588675e-04\n",
      " 6.20649429e-04 6.20260253e-04 6.18787948e-04 6.17446727e-04\n",
      " 6.14787627e-04 6.12011121e-04 6.11542433e-04 6.10129617e-04\n",
      " 6.09865529e-04 6.08081697e-04 6.06606016e-04 6.04019326e-04\n",
      " 6.03033695e-04 6.00984727e-04 5.99995547e-04 5.99048624e-04\n",
      " 5.97476435e-04 5.96838538e-04 5.95512101e-04 5.94452955e-04\n",
      " 5.93118602e-04 5.91600197e-04 5.88521711e-04 5.87414077e-04\n",
      " 5.85861388e-04 5.85336413e-04 5.83279005e-04 5.82704204e-04\n",
      " 5.81849017e-04 5.80256921e-04 5.79465006e-04 5.77441824e-04\n",
      " 5.75819402e-04 5.73757978e-04 5.72520541e-04 5.71655459e-04\n",
      " 5.70988981e-04 5.69490308e-04 5.68228425e-04 5.65988943e-04\n",
      " 5.65769325e-04 5.64364251e-04 5.63436595e-04 5.61158522e-04\n",
      " 5.60250191e-04 5.58621367e-04 5.57531253e-04 5.56237123e-04\n",
      " 5.54997823e-04 5.53527148e-04 5.52158803e-04 5.50139579e-04\n",
      " 5.49303251e-04 5.47768839e-04 5.45739720e-04 5.44637966e-04\n",
      " 5.42946800e-04 5.40967740e-04 5.39691420e-04 5.38684777e-04\n",
      " 5.37119398e-04 5.36007225e-04 5.34735213e-04 5.34256687e-04\n",
      " 5.32284612e-04 5.31046011e-04 5.29844256e-04 5.27973170e-04\n",
      " 5.26697957e-04 5.25760988e-04 5.25111274e-04 5.23091177e-04\n",
      " 5.22219401e-04 5.21354901e-04 5.19543013e-04 5.16485656e-04\n",
      " 5.16200555e-04 5.15525171e-04 5.13898791e-04 5.12818340e-04\n",
      " 5.12136379e-04 5.10149577e-04 5.09723439e-04 5.07842167e-04\n",
      " 5.05921082e-04 5.04719617e-04 5.03476418e-04 5.02703828e-04\n",
      " 5.01283736e-04 5.00082155e-04 4.97032714e-04 4.96196211e-04\n",
      " 4.95006680e-04 4.94506327e-04 4.91919927e-04 4.91582381e-04\n",
      " 4.90627543e-04 4.89341503e-04 4.87100595e-04 4.85666213e-04\n",
      " 4.85285826e-04 4.83560754e-04 4.82636940e-04 4.81966621e-04\n",
      " 4.80096962e-04 4.79029928e-04 4.78053029e-04 4.77035792e-04\n",
      " 4.75213950e-04 4.72644577e-04 4.71657433e-04 4.71055537e-04\n",
      " 4.68712009e-04 4.67297446e-04 4.66766942e-04 4.64248500e-04\n",
      " 4.63016127e-04 4.62547614e-04 4.61622170e-04 4.60743875e-04\n",
      " 4.59359493e-04 4.57851915e-04 4.57521266e-04 4.56971698e-04\n",
      " 4.53560642e-04 4.52275330e-04 4.50804626e-04 4.50626976e-04\n",
      " 4.49449348e-04 4.48929757e-04 4.47370519e-04 4.46411461e-04\n",
      " 4.45377344e-04 4.44320816e-04 4.42569290e-04 4.40556905e-04\n",
      " 4.39802592e-04 4.38733259e-04 4.37481882e-04 4.36427916e-04\n",
      " 4.35198832e-04 4.34008020e-04 4.33046516e-04 4.31318011e-04\n",
      " 4.30130982e-04 4.28822619e-04 4.28084662e-04 4.27859661e-04\n",
      " 4.25501814e-04 4.24416328e-04 4.22775251e-04 4.21047647e-04\n",
      " 4.20765864e-04 4.18569311e-04 4.17650415e-04 4.16841998e-04\n",
      " 4.15242743e-04 4.14971320e-04 4.12628491e-04 4.11423913e-04\n",
      " 4.09647211e-04 4.08530585e-04 4.07076441e-04 4.06186358e-04\n",
      " 4.04343009e-04 4.03382495e-04 4.01260564e-04 4.00513323e-04\n",
      " 3.99808894e-04 3.97576106e-04 3.97118885e-04 3.94787232e-04\n",
      " 3.94550618e-04 3.93556082e-04 3.93055117e-04 3.91897571e-04\n",
      " 3.90308211e-04 3.88322835e-04 3.87285923e-04 3.86513828e-04\n",
      " 3.84795800e-04 3.83437029e-04 3.82458646e-04 3.80972953e-04\n",
      " 3.80653422e-04 3.79547797e-04 3.78791825e-04 3.77613265e-04\n",
      " 3.75655742e-04 3.74061230e-04 3.73162562e-04 3.72765266e-04\n",
      " 3.70900030e-04 3.69001122e-04 3.68123990e-04 3.67372908e-04\n",
      " 3.66273947e-04 3.65393207e-04 3.64839885e-04 3.60855076e-04\n",
      " 3.60680366e-04 3.59258440e-04 3.57978948e-04 3.55989527e-04\n",
      " 3.54971591e-04 3.54583201e-04 3.53025505e-04 3.52142175e-04\n",
      " 3.49177833e-04 3.48496425e-04 3.47140100e-04 3.46352957e-04\n",
      " 3.44961853e-04 3.43735795e-04 3.42495914e-04 3.41850537e-04\n",
      " 3.40718922e-04 3.39464663e-04 3.38231737e-04 3.36459867e-04\n",
      " 3.34698881e-04 3.33449716e-04 3.31782474e-04 3.30839917e-04\n",
      " 3.28269351e-04 3.27760354e-04 3.26699141e-04 3.25789530e-04\n",
      " 3.25379311e-04 3.22907668e-04 3.22070176e-04 3.21141735e-04\n",
      " 3.19869141e-04 3.17819533e-04 3.17537750e-04 3.16119404e-04\n",
      " 3.14175675e-04 3.11926735e-04 3.09592928e-04 3.08522227e-04\n",
      " 3.07094160e-04 3.05659749e-04 3.04627581e-04 3.03628680e-04\n",
      " 3.02040309e-04 3.00389103e-04 2.99930136e-04 2.98188534e-04\n",
      " 2.96677259e-04 2.95284815e-04 2.93615274e-04 2.92679761e-04\n",
      " 2.91541626e-04 2.90989061e-04 2.88460287e-04 2.87469185e-04\n",
      " 2.86437018e-04 2.85864604e-04 2.83449161e-04 2.82321183e-04\n",
      " 2.81344954e-04 2.79460073e-04 2.77814310e-04 2.77321436e-04\n",
      " 2.75888073e-04 2.75458500e-04 2.71976198e-04 2.71757861e-04\n",
      " 2.70350632e-04 2.69677723e-04 2.68150703e-04 2.64391885e-04\n",
      " 2.63357477e-04 2.62604590e-04 2.61778710e-04 2.59641529e-04\n",
      " 2.58569460e-04 2.57882464e-04 2.56934669e-04 2.56026455e-04\n",
      " 2.55121966e-04 2.54007289e-04 2.52769882e-04 2.51280231e-04\n",
      " 2.50185607e-04 2.49955192e-04 2.47119897e-04 2.46059528e-04\n",
      " 2.44406459e-04 2.43393777e-04 2.42450886e-04 2.40875233e-04\n",
      " 2.38297114e-04 2.37504413e-04 2.35853920e-04 2.34305408e-04\n",
      " 2.33124345e-04 2.31429352e-04 2.30939579e-04 2.29593948e-04\n",
      " 2.28048026e-04 2.26185293e-04 2.25137075e-04 2.22688730e-04\n",
      " 2.21247188e-04 2.18919929e-04 2.17173132e-04 2.16124434e-04\n",
      " 2.14847241e-04 2.13678562e-04 2.13064443e-04 2.07883204e-04\n",
      " 2.06826400e-04 2.04409982e-04 2.01895615e-04 1.99306742e-04\n",
      " 1.98478490e-04 1.96122041e-04 1.94517954e-04 1.90404811e-04\n",
      " 1.90139312e-04 1.89198821e-04 1.86728212e-04 1.82508564e-04\n",
      " 1.81818614e-04 1.81480136e-04 1.79485476e-04 1.77979076e-04\n",
      " 1.76563568e-04 1.72665284e-04 1.70567509e-04 1.68280574e-04\n",
      " 1.66963451e-04 1.64752215e-04 1.62981494e-04 1.61202333e-04\n",
      " 1.58405819e-04 1.54007925e-04 1.49630228e-04 1.47670522e-04\n",
      " 1.43031328e-04 1.41619283e-04 1.38833202e-04 1.31983659e-04\n",
      " 1.30318105e-04 1.26632469e-04 1.18347445e-04 1.11692316e-04\n",
      " 1.00260666e-04 7.95008309e-05 7.23448247e-05 2.36923015e-07\n",
      " 5.07417859e-08 1.32952414e-08 1.26489015e-08 1.23561996e-08\n",
      " 9.98111016e-09 9.42208001e-09 6.29109476e-09 2.33946285e-09\n",
      " 2.67873153e-13 1.25189233e-13 6.54157311e-14 2.97415392e-14\n",
      " 2.41454517e-14 2.13338716e-14 3.31479224e-16]\n",
      "Cumulative explained variance:\n",
      " [0.20531198 0.24710098 0.27186012 0.28559932 0.2961555  0.30497977\n",
      " 0.3122628  0.31817183 0.32359976 0.32863206 0.33343047 0.33802652\n",
      " 0.3422754  0.3463702  0.350362   0.35401416 0.35762584 0.36115104\n",
      " 0.3645705  0.3679016  0.3711584  0.37428904 0.37735298 0.38032022\n",
      " 0.38327092 0.38619375 0.38909024 0.39192113 0.3946932  0.39740682\n",
      " 0.4000604  0.40270373 0.40533444 0.4079101  0.41046652 0.41300562\n",
      " 0.41548276 0.41794863 0.4204022  0.4228332  0.42524606 0.42762506\n",
      " 0.42997453 0.43231264 0.43462646 0.43692762 0.4392181  0.44148913\n",
      " 0.4437372  0.44595754 0.44813246 0.45029035 0.45243242 0.4545643\n",
      " 0.45667997 0.4587813  0.46087292 0.46294752 0.46500173 0.46703973\n",
      " 0.46906927 0.47107565 0.47306484 0.47504035 0.47700682 0.47896126\n",
      " 0.48090345 0.4828353  0.48474056 0.4866382  0.48852134 0.4903982\n",
      " 0.4922614  0.4941108  0.49595118 0.49777302 0.49959373 0.50139934\n",
      " 0.5031821  0.5049571  0.506727   0.50848913 0.5102444  0.5119882\n",
      " 0.5137237  0.5154481  0.51716954 0.51887673 0.5205694  0.522256\n",
      " 0.52392536 0.52559406 0.52725875 0.52892226 0.53057057 0.5322138\n",
      " 0.53385127 0.5354739  0.53709    0.53869694 0.540296   0.5418852\n",
      " 0.54347104 0.5450455  0.54661316 0.5481746  0.54972833 0.5512729\n",
      " 0.55281556 0.5543523  0.5558848  0.55740047 0.55890745 0.5604107\n",
      " 0.56191176 0.5634083  0.56489503 0.56637955 0.56785667 0.5693278\n",
      " 0.5707915  0.5722454  0.57369053 0.57512754 0.5765601  0.5779885\n",
      " 0.5794131  0.5808344  0.58224905 0.5836617  0.5850699  0.5864646\n",
      " 0.5878577  0.58924663 0.59062994 0.59200627 0.5933807  0.59474933\n",
      " 0.59610754 0.5974641  0.5988148  0.6001605  0.6014999  0.602835\n",
      " 0.60416514 0.605492   0.6068129  0.6081257  0.60943145 0.6107347\n",
      " 0.6120353  0.6133283  0.6146192  0.6159056  0.6171893  0.6184703\n",
      " 0.61974806 0.62101966 0.6222854  0.6235486  0.62480545 0.6260592\n",
      " 0.62730813 0.62855625 0.6297971  0.6310338  0.63226616 0.633496\n",
      " 0.6347187  0.63593775 0.6371505  0.6383599  0.639564   0.6407671\n",
      " 0.6419643  0.6431613  0.64435214 0.64553833 0.6467202  0.64789796\n",
      " 0.6490723  0.6502445  0.65141207 0.6525789  0.6537439  0.65490437\n",
      " 0.6560607  0.65721387 0.65836537 0.65951496 0.6606592  0.66180265\n",
      " 0.662944   0.6640812  0.66520846 0.66633296 0.6674543  0.66857207\n",
      " 0.66968685 0.6707995  0.67190963 0.6730139  0.6741169  0.67521405\n",
      " 0.67630917 0.67740333 0.6784934  0.67958    0.6806635  0.68174285\n",
      " 0.68281966 0.6838943  0.68496734 0.6860348  0.68710166 0.68816537\n",
      " 0.6892217  0.6902761  0.6913289  0.69237775 0.693425   0.69447166\n",
      " 0.6955146  0.69655305 0.69758946 0.6986228  0.69965315 0.7006788\n",
      " 0.7017024  0.7027228  0.70374286 0.7047589  0.70577395 0.70678633\n",
      " 0.70779777 0.7088032  0.7098079  0.7108103  0.71180904 0.71280587\n",
      " 0.7138008  0.7147902  0.7157784  0.7167661  0.7177475  0.7187265\n",
      " 0.7197032  0.7206747  0.72164315 0.72260886 0.7235733  0.7245363\n",
      " 0.7254974  0.7264558  0.7274112  0.72836405 0.7293143  0.7302607\n",
      " 0.73120546 0.73214614 0.7330858  0.7340224  0.7349577  0.7358889\n",
      " 0.7368174  0.7377445  0.7386693  0.7395895  0.74050915 0.74142337\n",
      " 0.7423369  0.74324805 0.74415696 0.7450638  0.7459697  0.74687284\n",
      " 0.7477745  0.74867254 0.7495689  0.75046283 0.7513548  0.7522449\n",
      " 0.7531332  0.75402045 0.7549042  0.75578415 0.75666285 0.75754076\n",
      " 0.75841737 0.75929123 0.7601633  0.7610329  0.7619002  0.7627663\n",
      " 0.76362985 0.7644899  0.76534605 0.7662001  0.7670524  0.7679013\n",
      " 0.76874834 0.7695947  0.7704369  0.77127826 0.7721159  0.77295196\n",
      " 0.7737863  0.7746194  0.7754515  0.77628154 0.7771083  0.77793163\n",
      " 0.778754   0.77957356 0.7803926  0.78120977 0.7820257  0.78283936\n",
      " 0.7836509  0.78446096 0.78526855 0.78607494 0.7868794  0.7876808\n",
      " 0.78847975 0.7892773  0.790073   0.79086596 0.7916582  0.79244715\n",
      " 0.7932336  0.7940197  0.7948042  0.79558676 0.79636776 0.7971461\n",
      " 0.79792136 0.79869455 0.7994655  0.8002356  0.80100495 0.8017731\n",
      " 0.80253863 0.803302   0.8040649  0.80482477 0.80558294 0.8063407\n",
      " 0.8070955  0.8078469  0.8085975  0.8093455  0.8100918  0.81083703\n",
      " 0.8115818  0.812324   0.813065   0.8138037  0.81454206 0.8152792\n",
      " 0.8160143  0.8167478  0.817479   0.81820893 0.8189381  0.819663\n",
      " 0.82038563 0.82110757 0.8218278  0.8225442  0.82325965 0.82397324\n",
      " 0.8246846  0.8253948  0.82610285 0.8268101  0.82751524 0.8282184\n",
      " 0.82892025 0.8296218  0.83032304 0.83102256 0.8317206  0.83241683\n",
      " 0.8331098  0.833802   0.834493   0.83518255 0.8358711  0.83655834\n",
      " 0.8372437  0.83792734 0.83860976 0.8392894  0.83996826 0.8406457\n",
      " 0.8413207  0.8419921  0.8426634  0.84333193 0.84399897 0.84466445\n",
      " 0.84532964 0.84599346 0.846655   0.84731436 0.84797287 0.8486289\n",
      " 0.8492838  0.8499384  0.8505907  0.85124236 0.8518914  0.8525389\n",
      " 0.8531859  0.85383195 0.85447615 0.8551196  0.855762   0.8564039\n",
      " 0.85704374 0.8576812  0.8583179  0.8589533  0.8595874  0.8602193\n",
      " 0.86084974 0.8614784  0.862106   0.86273193 0.86335725 0.8639798\n",
      " 0.8646005  0.8652207  0.86583954 0.866457   0.86707175 0.86768377\n",
      " 0.8682953  0.8689054  0.8695153  0.8701234  0.87073    0.871334\n",
      " 0.87193704 0.87253803 0.873138   0.87373704 0.8743345  0.87493134\n",
      " 0.87552685 0.8761213  0.8767144  0.877306   0.8778945  0.8784819\n",
      " 0.8790678  0.8796531  0.8802364  0.8808191  0.88140094 0.8819812\n",
      " 0.8825607  0.8831381  0.88371396 0.8842877  0.8848602  0.8854319\n",
      " 0.8860029  0.88657236 0.8871406  0.8877066  0.88827235 0.8888367\n",
      " 0.8894001  0.8899613  0.8905215  0.89108014 0.8916377  0.8921939\n",
      " 0.8927489  0.89330244 0.8938546  0.89440477 0.8949541  0.89550185\n",
      " 0.8960476  0.89659226 0.8971352  0.89767617 0.8982159  0.8987546\n",
      " 0.8992917  0.8998277  0.90036243 0.90089667 0.90142894 0.90195996\n",
      " 0.9024898  0.90301776 0.9035445  0.90407026 0.9045954  0.90511847\n",
      " 0.90564066 0.906162   0.90668154 0.907198   0.9077142  0.9082297\n",
      " 0.9087436  0.90925646 0.9097686  0.91027874 0.9107885  0.9112963\n",
      " 0.91180223 0.91230696 0.91281044 0.91331315 0.9138144  0.9143145\n",
      " 0.91481155 0.91530776 0.9158028  0.91629726 0.9167892  0.91728073\n",
      " 0.91777134 0.9182607  0.9187478  0.91923344 0.91971874 0.9202023\n",
      " 0.92068493 0.9211669  0.921647   0.92212605 0.9226041  0.9230811\n",
      " 0.9235563  0.924029   0.92450064 0.9249717  0.92544043 0.92590773\n",
      " 0.9263745  0.92683876 0.92730176 0.9277643  0.92822593 0.9286867\n",
      " 0.92914605 0.9296039  0.9300614  0.9305184  0.9309719  0.9314242\n",
      " 0.931875   0.9323256  0.9327751  0.933224   0.9336714  0.93411785\n",
      " 0.9345632  0.9350075  0.9354501  0.9358906  0.93633044 0.9367692\n",
      " 0.9372067  0.9376431  0.9380783  0.93851227 0.9389453  0.9393766\n",
      " 0.9398067  0.9402355  0.9406636  0.9410914  0.94151694 0.9419414\n",
      " 0.94236416 0.9427852  0.94320595 0.9436245  0.94404215 0.94445896\n",
      " 0.9448742  0.9452892  0.94570184 0.9461133  0.94652295 0.9469315\n",
      " 0.9473386  0.9477448  0.94814914 0.94855255 0.9489538  0.9493543\n",
      " 0.9497541  0.9501517  0.9505488  0.9509436  0.9513381  0.9517317\n",
      " 0.9521247  0.9525166  0.9529069  0.95329523 0.95368254 0.9540691\n",
      " 0.9544539  0.9548373  0.9552198  0.9556008  0.95598143 0.956361\n",
      " 0.9567398  0.9571174  0.957493   0.9578671  0.9582403  0.95861304\n",
      " 0.95898396 0.95935297 0.9597211  0.96008843 0.9604547  0.9608201\n",
      " 0.9611849  0.96154577 0.96190643 0.9622657  0.96262366 0.9629797\n",
      " 0.9633346  0.9636892  0.96404225 0.9643944  0.96474355 0.96509206\n",
      " 0.9654392  0.96578556 0.9661305  0.96647424 0.9668167  0.96715856\n",
      " 0.96749926 0.9678387  0.96817696 0.9685134  0.9688481  0.96918154\n",
      " 0.9695133  0.96984416 0.9701724  0.9705002  0.97082686 0.97115266\n",
      " 0.97147804 0.9718009  0.97212297 0.9724441  0.972764   0.9730818\n",
      " 0.97339934 0.9737155  0.97402966 0.9743416  0.97465116 0.9749597\n",
      " 0.97526675 0.9755724  0.97587705 0.9761807  0.9764827  0.9767831\n",
      " 0.977083   0.9773812  0.9776779  0.97797316 0.9782668  0.97855943\n",
      " 0.97885096 0.97914195 0.97943044 0.9797179  0.9800044  0.98029023\n",
      " 0.98057365 0.980856   0.98113734 0.9814168  0.98169464 0.981972\n",
      " 0.9822479  0.9825233  0.9827953  0.98306704 0.9833374  0.98360705\n",
      " 0.9838752  0.9841396  0.98440295 0.9846656  0.98492736 0.985187\n",
      " 0.98544556 0.98570347 0.9859604  0.9862164  0.98647153 0.98672557\n",
      " 0.98697835 0.98722965 0.9874798  0.9877298  0.9879769  0.98822296\n",
      " 0.98846734 0.9887107  0.9889532  0.98919404 0.98943233 0.98966986\n",
      " 0.9899057  0.99014    0.99037313 0.9906046  0.99083555 0.99106514\n",
      " 0.9912932  0.9915194  0.9917445  0.9919672  0.99218845 0.9924074\n",
      " 0.9926246  0.9928407  0.9930556  0.99326926 0.99348235 0.99369025\n",
      " 0.9938971  0.99410146 0.99430335 0.99450266 0.99470115 0.99489725\n",
      " 0.99509174 0.9952821  0.99547225 0.99566144 0.9958482  0.9960307\n",
      " 0.9962125  0.996394   0.99657345 0.9967514  0.996928   0.99710065\n",
      " 0.99727124 0.9974395  0.99760646 0.9977712  0.99793416 0.9980954\n",
      " 0.9982538  0.99840784 0.99855745 0.99870515 0.9988482  0.9989898\n",
      " 0.99912864 0.9992606  0.9993909  0.99951756 0.99963593 0.99974763\n",
      " 0.9998479  0.9999274  0.99999976 1.         1.         1.\n",
      " 1.         1.         1.         1.         1.         1.\n",
      " 1.         1.         1.         1.         1.         1.\n",
      " 1.        ]\n",
      "Number of components explaining 99.9% variance:  775\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "scaler = StandardScaler()\n",
    "# normalize by feature\n",
    "scaler.fit(X_tr)\n",
    "\n",
    "# apply to datasets\n",
    "X_tr_scaled = scaler.transform(X_tr)\n",
    "X_val_scaled = scaler.transform(X_val)\n",
    "X_te_scaled = scaler.transform(X_te)\n",
    "\n",
    "pca = PCA()\n",
    "principalComponents = pca.fit_transform(X_tr_scaled)\n",
    "\n",
    "print(\"Explained variance ratio:\\n\", pca.explained_variance_ratio_)\n",
    "print(\"Cumulative explained variance:\\n\", np.cumsum(pca.explained_variance_ratio_))\n",
    "\n",
    "n_components = np.argmax(np.cumsum(pca.explained_variance_ratio_) >= 0.999) + 1\n",
    "print(\"Number of components explaining 99.9% variance: \", n_components)\n",
    "\n",
    "pca = PCA(n_components=n_components)\n",
    "X_tr_p = pca.fit_transform(X_tr_scaled)\n",
    "X_val_p = pca.transform(X_val_scaled)\n",
    "X_te_p = pca.transform(X_te_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.35 0.35 0.36 0.33 0.32 0.3  0.31 0.3  0.3 ]\n",
      " [0.34 0.34 0.32 0.31 0.28 0.26 0.27 0.26 0.26]\n",
      " [0.43 0.43 0.41 0.36 0.37 0.36 0.35 0.35 0.33]\n",
      " [0.46 0.46 0.46 0.46 0.41 0.41 0.42 0.41 0.42]]\n",
      "Maximum validation accuracy: 0.46\n",
      "Optimal k: 1\n",
      "Optimal Distance Metric: correlation\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.neighbors import NearestNeighbors, KNeighborsClassifier\n",
    "from statistics import mode\n",
    "\n",
    "def do_knn(X_tr, y_tr, X_val, y_val, k, metric):\n",
    "    # regular knn for comparison\n",
    "    y_pred = np.zeros((k, len(y_val)))\n",
    "    nbrs = NearestNeighbors(n_neighbors=k, metric=metric, n_jobs=-1).fit(X_tr)\n",
    "    distances, neighbors_list = nbrs.kneighbors(X_val)\n",
    "    for (index, neighbors) in enumerate(neighbors_list):\n",
    "        for j in range(k):\n",
    "            predicted = mode(y_tr[neighbors[0:j+1]])\n",
    "            y_pred[j, index] = predicted\n",
    "                \n",
    "    return y_pred\n",
    "\n",
    "# KNN grid search\n",
    "metrics = [\"euclidean\", \"cityblock\", \"cosine\", \"correlation\"]\n",
    "accuracy_scores = np.zeros((4, 9))\n",
    "for (metric_idx, j) in enumerate(metrics):\n",
    "    y_pred = do_knn(X_tr_scaled, y_tr, X_val_scaled, y_val, 9, metric=j)\n",
    "    for (k_idx, pred) in enumerate(y_pred):\n",
    "        accuracy_scores[metric_idx, k_idx] = accuracy_score(y_val, pred)\n",
    "\n",
    "max_idx = np.unravel_index(np.argmax(accuracy_scores, axis=None), accuracy_scores.shape)\n",
    "max_metric = metrics[max_idx[0]]\n",
    "max_k = max_idx[1] + 1  # Adding 1 because k starts from 1, not 0\n",
    "max_accuracy_score = accuracy_scores[max_idx]\n",
    "\n",
    "print(accuracy_scores)\n",
    "print(\"Maximum validation accuracy:\", max_accuracy_score)\n",
    "print(\"Optimal k:\", max_k)\n",
    "print(\"Optimal Distance Metric:\", max_metric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "knn test accuracy: 0.41\n"
     ]
    }
   ],
   "source": [
    "knbrs = KNeighborsClassifier(n_neighbors=max_k, metric=max_metric)\n",
    "knbrs.fit(X_tr_scaled, y_tr)\n",
    "accuracy = knbrs.score(X_te_scaled, y_te)\n",
    "print(\"knn test accuracy:\", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation for kernel poly\n",
      "C=0.03125\n",
      "C=0.0625\n",
      "C=0.125\n",
      "C=0.25\n",
      "C=0.5\n",
      "C=1.0\n",
      "C=2.0\n",
      "C=4.0\n",
      "C=8.0\n",
      "C=16.0\n",
      "validation for kernel rbf\n",
      "C=0.03125\n",
      "C=0.0625\n",
      "C=0.125\n",
      "C=0.25\n",
      "C=0.5\n",
      "C=1.0\n",
      "C=2.0\n",
      "C=4.0\n",
      "C=8.0\n",
      "C=16.0\n",
      "validation for kernel sigmoid\n",
      "C=0.03125\n",
      "C=0.0625\n",
      "C=0.125\n",
      "C=0.25\n",
      "C=0.5\n",
      "C=1.0\n",
      "C=2.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/steven/Library/Python/3.9/lib/python/site-packages/sklearn/svm/_base.py:299: ConvergenceWarning: Solver terminated early (max_iter=1000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C=4.0\n",
      "C=8.0\n",
      "C=16.0\n",
      "{'poly': [0.1, 0.1, 0.12, 0.12, 0.18, 0.28, 0.32, 0.31, 0.29, 0.34], 'rbf': [0.32, 0.32, 0.32, 0.32, 0.35, 0.44, 0.49, 0.49, 0.49, 0.49], 'sigmoid': [0.21, 0.21, 0.26, 0.34, 0.4, 0.45, 0.47, 0.47, 0.47, 0.48]}\n",
      "validation for linear\n",
      "C=0.03125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/steven/Library/Python/3.9/lib/python/site-packages/sklearn/svm/_base.py:1244: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C=0.0625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/steven/Library/Python/3.9/lib/python/site-packages/sklearn/svm/_base.py:1244: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C=0.125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/steven/Library/Python/3.9/lib/python/site-packages/sklearn/svm/_base.py:1244: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C=0.25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/steven/Library/Python/3.9/lib/python/site-packages/sklearn/svm/_base.py:1244: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C=0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/steven/Library/Python/3.9/lib/python/site-packages/sklearn/svm/_base.py:1244: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C=1.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/steven/Library/Python/3.9/lib/python/site-packages/sklearn/svm/_base.py:1244: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C=2.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/steven/Library/Python/3.9/lib/python/site-packages/sklearn/svm/_base.py:1244: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C=4.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/steven/Library/Python/3.9/lib/python/site-packages/sklearn/svm/_base.py:1244: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C=8.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/steven/Library/Python/3.9/lib/python/site-packages/sklearn/svm/_base.py:1244: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C=16.0\n",
      "[0.48, 0.48, 0.48, 0.48, 0.47, 0.47, 0.47, 0.47, 0.48, 0.47]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/steven/Library/Python/3.9/lib/python/site-packages/sklearn/svm/_base.py:1244: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "C = [np.float_power(2,i) for i in range(-5,5)]\n",
    "kernels = ['poly', 'rbf', 'sigmoid'] \n",
    "kernel_score_dict = {\"poly\": [], \"rbf\": [], \"sigmoid\": []}\n",
    "\n",
    "for kern in kernels:\n",
    "    print(f'validation for kernel {kern}')\n",
    "    for reg in C: \n",
    "        print(f'C={reg}')\n",
    "        clf = SVC(C = reg, kernel = kern, max_iter = 1000).fit(X_tr_p, y_tr)\n",
    "        preds = clf.predict(X_val_p)\n",
    "        score = accuracy_score(y_val, preds) \n",
    "        kernel_score_dict[kern].append(score)\n",
    "\n",
    "print(kernel_score_dict)\n",
    "\n",
    "## SVM LINEAR\n",
    "\n",
    "svc_scores = []\n",
    "print(f'validation for linear')\n",
    "for rg in C: \n",
    "    print(f'C={rg}')\n",
    "    clf = LinearSVC(C = rg, max_iter=1000).fit(X_tr_p, y_tr)\n",
    "    preds = clf.predict(X_val_p)\n",
    "    score = accuracy_score(y_val, preds)\n",
    "    svc_scores.append(score)\n",
    "\n",
    "print(svc_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kernel accuracy 0.47\n",
      "linear accuracy 0.46\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/steven/Library/Python/3.9/lib/python/site-packages/sklearn/svm/_base.py:1244: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# test kernel\n",
    "clf = SVC(C = 2.0, kernel = 'rbf', max_iter = 1000).fit(X_tr_p, y_tr)\n",
    "preds = clf.predict(X_te_p)\n",
    "print('kernel accuracy', accuracy_score(y_te, preds))\n",
    "\n",
    "# test linear\n",
    "clf = LinearSVC(C = 0.03125, max_iter=1000).fit(X_tr_p, y_tr)\n",
    "preds = clf.predict(X_te_p)\n",
    "print('linear accuracy', accuracy_score(y_te, preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T = 100\n",
      "T = 200\n",
      "T = 300\n",
      "T = 400\n",
      "T = 500\n",
      "T = 600\n",
      "T = 700\n",
      "T = 800\n",
      "T = 900\n",
      "[0.3, 0.35, 0.35, 0.35, 0.31, 0.31, 0.36, 0.34, 0.35]\n",
      "[0.34, 0.39, 0.36, 0.38, 0.35, 0.39, 0.37, 0.37, 0.39]\n",
      "[0.55, 0.53, 0.5, 0.49, 0.49, 0.49, 0.49, 0.49, 0.49]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier, BaggingClassifier, GradientBoostingClassifier, HistGradientBoostingClassifier\n",
    "from sklearn.multiclass import OneVsOneClassifier\n",
    "\n",
    "# RF test\n",
    "Ts = range(100, 901, 100)\n",
    "rf_scores = []\n",
    "bag_scores = []\n",
    "grad_scores = []\n",
    "for T in Ts:\n",
    "    print(f\"T = {T}\")\n",
    "    best_rf = RandomForestClassifier(n_estimators = T, n_jobs = -1).fit(X_tr_p, y_tr) \n",
    "    best_rf_preds = best_rf.predict(X_val_p)\n",
    "    rf_score = accuracy_score(y_val, best_rf_preds)\n",
    "    rf_scores.append(rf_score)\n",
    "\n",
    "    bag = BaggingClassifier(n_estimators = T, n_jobs = -1).fit(X_tr_p, y_tr) \n",
    "    bag_preds = bag.predict(X_val_p)\n",
    "    bag_score = accuracy_score(y_val, bag_preds)\n",
    "    bag_scores.append(bag_score)\n",
    "\n",
    "    gb_clf = GradientBoostingClassifier(n_estimators=T, random_state=10)\n",
    "    ovo_clf = OneVsOneClassifier(gb_clf, n_jobs=-1)\n",
    "    ovo_clf.fit(X_tr_p, y_tr)\n",
    "    grad_preds = ovo_clf.predict(X_te_p) \n",
    "    grad_score = accuracy_score(y_te, grad_preds)\n",
    "    grad_scores.append(grad_score)\n",
    "\n",
    "print(rf_scores)\n",
    "print(bag_scores)\n",
    "print(grad_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rf accuracy 0.42\n",
      "bagging accuracy 0.49\n",
      "grad accuracy 0.55\n",
      "hist grad accuracy 0.44\n"
     ]
    }
   ],
   "source": [
    "best_rf = RandomForestClassifier(n_estimators = 700, n_jobs = -1).fit(X_tr_p, y_tr)\n",
    "best_rf_preds = best_rf.predict(X_te_p)\n",
    "print('rf accuracy', accuracy_score(y_te, best_rf_preds))\n",
    "\n",
    "bag = BaggingClassifier(n_estimators = 200, n_jobs = -1).fit(X_tr_p, y_tr) \n",
    "bag_preds = bag.predict(X_te_p)\n",
    "print('bagging accuracy', accuracy_score(y_te, bag_preds))\n",
    "\n",
    "gb_clf = GradientBoostingClassifier(n_estimators=100, random_state=10)\n",
    "ovo_clf = OneVsOneClassifier(gb_clf, n_jobs=-1)\n",
    "ovo_clf.fit(X_tr_p, y_tr)\n",
    "grad_preds = ovo_clf.predict(X_te_p) \n",
    "grad_score = accuracy_score(y_te, grad_preds)\n",
    "print('grad accuracy', grad_score)\n",
    "\n",
    "gb_clf = HistGradientBoostingClassifier(learning_rate=1, random_state=10)\n",
    "ovo_clf = OneVsOneClassifier(gb_clf, n_jobs=-1)\n",
    "ovo_clf.fit(X_tr_p, y_tr)\n",
    "hist_grad_preds = ovo_clf.predict(X_te_p) \n",
    "hist_grad_score = accuracy_score(y_te, hist_grad_preds)\n",
    "print('hist grad accuracy', hist_grad_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max_depth=10\n",
      "max_depth=11\n",
      "max_depth=12\n",
      "max_depth=13\n",
      "max_depth=14\n",
      "[0.35, 0.32, 0.31, 0.31, 0.28]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "T = 200\n",
    "adaboost_stats = []\n",
    "for i in list(range(10,15)):\n",
    "    print(f'max_depth={i}')\n",
    "    tree = DecisionTreeClassifier(max_depth=i)\n",
    "    a_clf = AdaBoostClassifier(\n",
    "        tree, n_estimators=T, learning_rate=1, random_state=10\n",
    "    )\n",
    "    a_clf.fit(X_tr_p, y_tr)\n",
    "    preds = a_clf.predict(X_val_p)\n",
    "    score = accuracy_score(y_val, preds)\n",
    "    adaboost_stats.append(score)\n",
    "print(adaboost_stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "adaboost accuracy 0.33\n"
     ]
    }
   ],
   "source": [
    "a_clf = AdaBoostClassifier(\n",
    "    DecisionTreeClassifier(max_depth=10), n_estimators=200, learning_rate=1, random_state=10\n",
    ")\n",
    "a_clf.fit(X_tr_p, y_tr)\n",
    "preds = a_clf.predict(X_te_p)\n",
    "print('adaboost accuracy', accuracy_score(y_te, preds))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}