notebooks/Optimizations/Two_Moons_DBSCAN_Hyperparameter_Search.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimizing DBSCAN's hyperparameters minPts, eps on the Two Moons dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To reproduce this experiment, first delete or remove the database DBSCAN_two_moons_HPO.db that is stored in the results/optimization folder. Otherwise, the TPE optimization will use the 100 trials stored in the existing DBSCAN_two_moons_HPO.db and continue with the 101th trial up to the 200th trial. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "UbvqVKulcAwD"
   },
   "outputs": [],
   "source": [
    "import sys  \n",
    "sys.path.insert(0, '../..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 766,
     "status": "ok",
     "timestamp": 1621161358988,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "lWMfMVRSeO88",
    "outputId": "8073093a-b4d8-4e52-9539-c073de516725"
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "EXvc2zQ-csmQ"
   },
   "outputs": [],
   "source": [
    "import optuna\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.datasets import make_blobs, make_moons\n",
    "from sklearn.cluster import KMeans, DBSCAN, SpectralClustering, MeanShift, estimate_bandwidth\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import adjusted_mutual_info_score as AMI\n",
    "from rock import ROCK\n",
    "\n",
    "import logging\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "eDgO4l1liEs5"
   },
   "outputs": [],
   "source": [
    "def objective(trial):\n",
    "    seed = 0\n",
    "\n",
    "    results = []\n",
    "\n",
    "    n_samples = 1000\n",
    "    jitter = 0.15\n",
    "\n",
    "    min_pts = trial.suggest_int('min_pts', 2, 100)\n",
    "    eps = trial.suggest_int('eps', 1, 10) * 0.1\n",
    "\n",
    "    datasets = []\n",
    "    n_centers = None\n",
    "    \n",
    "    for s in range(seed, seed+10):\n",
    "        datasets.append(make_moons(n_samples=n_samples, noise=jitter,\n",
    "            shuffle=True, random_state=s))\n",
    "        \n",
    "    step = 0\n",
    "    for dataset in datasets: \n",
    "        run = {}\n",
    "        run['step'] = step\n",
    "        run['n_samples'] = n_samples\n",
    "        run['jitter'] = jitter\n",
    "        scaler = StandardScaler()\n",
    "        data = scaler.fit_transform(dataset[0])\n",
    "\n",
    "        dbscan = DBSCAN(eps=eps, min_samples=min_pts).fit(data).labels_\n",
    "        run['eps'] = eps\n",
    "        run['min_pts'] = min_pts\n",
    "\n",
    "        gt = dataset[1]\n",
    "        results.append(AMI(dbscan, gt))\n",
    "        run['DBSCAN'] = AMI(dbscan, gt)\n",
    "        step += 1\n",
    "        experiment.append(run)\n",
    "\n",
    "    return np.mean(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "sfM20A23iHzF"
   },
   "outputs": [],
   "source": [
    "experiment = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "eMBDJdRPehpG"
   },
   "outputs": [],
   "source": [
    "study_name = 'DBSCAN_two_moons_HPO'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 926,
     "status": "ok",
     "timestamp": 1621161366492,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "R1jP84jFed9N",
    "outputId": "5cf7d6fa-56e9-4a9f-ef0b-d7ea2f2c1e9a"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-06-01 18:50:50,920]\u001b[0m A new study created in RDB with name: DBSCAN_two_moons_HPO\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "storage_name = f'sqlite:///../../results/optimization/{study_name}.db'\n",
    "sampler = optuna.samplers.TPESampler(seed=0)\n",
    "study = optuna.create_study(study_name=study_name, storage=storage_name, sampler=sampler, load_if_exists=True,  direction=\"maximize\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-06-01 18:50:58,442]\u001b[0m Trial 0 finished with value: 0.0 and parameters: {'min_pts': 56, 'eps': 8}. Best is trial 0 with value: 0.0.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:50:58,658]\u001b[0m Trial 1 finished with value: 0.0001820904250210742 and parameters: {'min_pts': 61, 'eps': 6}. Best is trial 1 with value: 0.0001820904250210742.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:50:58,859]\u001b[0m Trial 2 finished with value: 7.672985006701068e-16 and parameters: {'min_pts': 43, 'eps': 7}. Best is trial 1 with value: 0.0001820904250210742.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:50:59,090]\u001b[0m Trial 3 finished with value: 0.0 and parameters: {'min_pts': 45, 'eps': 9}. Best is trial 1 with value: 0.0001820904250210742.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:50:59,259]\u001b[0m Trial 4 finished with value: 0.0 and parameters: {'min_pts': 97, 'eps': 4}. Best is trial 1 with value: 0.0001820904250210742.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:50:59,444]\u001b[0m Trial 5 finished with value: 0.0009439327152330999 and parameters: {'min_pts': 80, 'eps': 6}. Best is trial 5 with value: 0.0009439327152330999.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:50:59,675]\u001b[0m Trial 6 finished with value: 0.0 and parameters: {'min_pts': 58, 'eps': 10}. Best is trial 5 with value: 0.0009439327152330999.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:50:59,838]\u001b[0m Trial 7 finished with value: 0.1747753794754873 and parameters: {'min_pts': 9, 'eps': 1}. Best is trial 7 with value: 0.1747753794754873.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:00,069]\u001b[0m Trial 8 finished with value: 0.0 and parameters: {'min_pts': 4, 'eps': 9}. Best is trial 7 with value: 0.1747753794754873.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:00,289]\u001b[0m Trial 9 finished with value: 0.0 and parameters: {'min_pts': 79, 'eps': 9}. Best is trial 7 with value: 0.1747753794754873.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:00,442]\u001b[0m Trial 10 finished with value: 0.37337308562260835 and parameters: {'min_pts': 3, 'eps': 1}. Best is trial 10 with value: 0.37337308562260835.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:00,608]\u001b[0m Trial 11 finished with value: 0.22661012736222963 and parameters: {'min_pts': 7, 'eps': 1}. Best is trial 10 with value: 0.37337308562260835.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:00,739]\u001b[0m Trial 12 finished with value: 0.0 and parameters: {'min_pts': 22, 'eps': 1}. Best is trial 10 with value: 0.37337308562260835.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:00,924]\u001b[0m Trial 13 finished with value: 0.7653743420598683 and parameters: {'min_pts': 23, 'eps': 3}. Best is trial 13 with value: 0.7653743420598683.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:01,093]\u001b[0m Trial 14 finished with value: 0.8223943608040875 and parameters: {'min_pts': 26, 'eps': 3}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:01,256]\u001b[0m Trial 15 finished with value: 0.8163653995203012 and parameters: {'min_pts': 27, 'eps': 3}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:01,442]\u001b[0m Trial 16 finished with value: 0.7774998172810004 and parameters: {'min_pts': 31, 'eps': 3}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:01,615]\u001b[0m Trial 17 finished with value: -0.0010272852621480322 and parameters: {'min_pts': 31, 'eps': 4}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:01,778]\u001b[0m Trial 18 finished with value: 0.17219692510907428 and parameters: {'min_pts': 18, 'eps': 3}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:01,956]\u001b[0m Trial 19 finished with value: 0.16670944502313895 and parameters: {'min_pts': 36, 'eps': 4}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:02,126]\u001b[0m Trial 20 finished with value: 0.7351977987676654 and parameters: {'min_pts': 13, 'eps': 2}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:02,295]\u001b[0m Trial 21 finished with value: 0.8108741691418432 and parameters: {'min_pts': 28, 'eps': 3}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:02,464]\u001b[0m Trial 22 finished with value: -0.00019639383427738437 and parameters: {'min_pts': 27, 'eps': 5}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:02,627]\u001b[0m Trial 23 finished with value: 0.0 and parameters: {'min_pts': 40, 'eps': 2}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:02,781]\u001b[0m Trial 24 finished with value: 0.0 and parameters: {'min_pts': 49, 'eps': 2}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:02,959]\u001b[0m Trial 25 finished with value: -0.00019639383427891895 and parameters: {'min_pts': 17, 'eps': 5}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:03,143]\u001b[0m Trial 26 finished with value: 0.7133010190216507 and parameters: {'min_pts': 34, 'eps': 3}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:03,303]\u001b[0m Trial 27 finished with value: -7.240515142824025e-05 and parameters: {'min_pts': 25, 'eps': 4}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:03,483]\u001b[0m Trial 28 finished with value: 0.6722140816055235 and parameters: {'min_pts': 15, 'eps': 2}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:03,667]\u001b[0m Trial 29 finished with value: 0.30418222929450817 and parameters: {'min_pts': 65, 'eps': 5}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:03,848]\u001b[0m Trial 30 finished with value: 0.2958999292589684 and parameters: {'min_pts': 49, 'eps': 3}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:04,030]\u001b[0m Trial 31 finished with value: 0.7881525424337006 and parameters: {'min_pts': 30, 'eps': 3}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:04,184]\u001b[0m Trial 32 finished with value: 0.0 and parameters: {'min_pts': 38, 'eps': 2}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:04,352]\u001b[0m Trial 33 finished with value: -0.0010514325476725368 and parameters: {'min_pts': 30, 'eps': 4}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:04,575]\u001b[0m Trial 34 finished with value: 7.672985006701068e-16 and parameters: {'min_pts': 43, 'eps': 7}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:04,748]\u001b[0m Trial 35 finished with value: 0.5931519273403391 and parameters: {'min_pts': 21, 'eps': 3}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:04,948]\u001b[0m Trial 36 finished with value: -0.0002678469785684112 and parameters: {'min_pts': 12, 'eps': 4}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:05,150]\u001b[0m Trial 37 finished with value: -0.0001963938342766171 and parameters: {'min_pts': 52, 'eps': 6}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:05,325]\u001b[0m Trial 38 finished with value: 0.0 and parameters: {'min_pts': 64, 'eps': 2}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:05,534]\u001b[0m Trial 39 finished with value: -0.00019639383427738437 and parameters: {'min_pts': 27, 'eps': 5}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:05,718]\u001b[0m Trial 40 finished with value: 0.45474141735903684 and parameters: {'min_pts': 43, 'eps': 3}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:05,903]\u001b[0m Trial 41 finished with value: 0.7344671391522274 and parameters: {'min_pts': 33, 'eps': 3}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:06,088]\u001b[0m Trial 42 finished with value: 0.7993511240643232 and parameters: {'min_pts': 29, 'eps': 3}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:06,266]\u001b[0m Trial 43 finished with value: -0.00019357014487705253 and parameters: {'min_pts': 20, 'eps': 4}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:06,437]\u001b[0m Trial 44 finished with value: 0.21971308065927736 and parameters: {'min_pts': 27, 'eps': 2}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:06,589]\u001b[0m Trial 45 finished with value: 0.20266814067312966 and parameters: {'min_pts': 8, 'eps': 1}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-06-01 18:51:06,767]\u001b[0m Trial 46 finished with value: 0.5842809606119299 and parameters: {'min_pts': 39, 'eps': 4}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:06,936]\u001b[0m Trial 47 finished with value: -0.00041245973435900183 and parameters: {'min_pts': 2, 'eps': 3}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:07,121]\u001b[0m Trial 48 finished with value: 0.2843963826403294 and parameters: {'min_pts': 24, 'eps': 2}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:07,274]\u001b[0m Trial 49 finished with value: 0.0 and parameters: {'min_pts': 100, 'eps': 1}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:07,472]\u001b[0m Trial 50 finished with value: 0.7039805622386408 and parameters: {'min_pts': 35, 'eps': 3}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:07,653]\u001b[0m Trial 51 finished with value: 0.7881525424337006 and parameters: {'min_pts': 30, 'eps': 3}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:07,853]\u001b[0m Trial 52 finished with value: 0.7774998172810004 and parameters: {'min_pts': 31, 'eps': 3}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:08,041]\u001b[0m Trial 53 finished with value: -0.0009728835516980697 and parameters: {'min_pts': 29, 'eps': 4}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:08,208]\u001b[0m Trial 54 finished with value: 0.45049641827192277 and parameters: {'min_pts': 19, 'eps': 2}. Best is trial 14 with value: 0.8223943608040875.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:08,370]\u001b[0m Trial 55 finished with value: 0.827642731199133 and parameters: {'min_pts': 25, 'eps': 3}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:08,555]\u001b[0m Trial 56 finished with value: 1.5345970013402136e-15 and parameters: {'min_pts': 13, 'eps': 5}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:08,724]\u001b[0m Trial 57 finished with value: 0.16670944502313895 and parameters: {'min_pts': 36, 'eps': 4}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:08,891]\u001b[0m Trial 58 finished with value: 0.0 and parameters: {'min_pts': 23, 'eps': 1}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:09,056]\u001b[0m Trial 59 finished with value: 0.5543027224549918 and parameters: {'min_pts': 17, 'eps': 2}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:09,226]\u001b[0m Trial 60 finished with value: 0.3752715393016814 and parameters: {'min_pts': 45, 'eps': 3}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:09,426]\u001b[0m Trial 61 finished with value: 0.8108741691418432 and parameters: {'min_pts': 28, 'eps': 3}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:09,616]\u001b[0m Trial 62 finished with value: -0.0005685365241752909 and parameters: {'min_pts': 26, 'eps': 4}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:09,809]\u001b[0m Trial 63 finished with value: 0.5934917508748019 and parameters: {'min_pts': 22, 'eps': 3}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:09,963]\u001b[0m Trial 64 finished with value: 0.0 and parameters: {'min_pts': 40, 'eps': 2}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:10,132]\u001b[0m Trial 65 finished with value: 0.7133010190216507 and parameters: {'min_pts': 34, 'eps': 3}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:10,316]\u001b[0m Trial 66 finished with value: 0.17296670402896694 and parameters: {'min_pts': 16, 'eps': 3}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:10,495]\u001b[0m Trial 67 finished with value: -0.0004562922115234594 and parameters: {'min_pts': 27, 'eps': 4}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:10,648]\u001b[0m Trial 68 finished with value: 0.7613412535732192 and parameters: {'min_pts': 12, 'eps': 2}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:10,838]\u001b[0m Trial 69 finished with value: -0.0005304563648340845 and parameters: {'min_pts': 32, 'eps': 4}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:11,005]\u001b[0m Trial 70 finished with value: 0.0 and parameters: {'min_pts': 88, 'eps': 3}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:11,170]\u001b[0m Trial 71 finished with value: 0.7881525424337006 and parameters: {'min_pts': 30, 'eps': 3}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:11,370]\u001b[0m Trial 72 finished with value: 0.7653743420598683 and parameters: {'min_pts': 23, 'eps': 3}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:11,540]\u001b[0m Trial 73 finished with value: 0.645713914985216 and parameters: {'min_pts': 37, 'eps': 3}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:11,718]\u001b[0m Trial 74 finished with value: 0.14690835695261506 and parameters: {'min_pts': 29, 'eps': 2}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:11,887]\u001b[0m Trial 75 finished with value: -0.00019357014487705253 and parameters: {'min_pts': 20, 'eps': 4}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:12,057]\u001b[0m Trial 76 finished with value: 0.827642731199133 and parameters: {'min_pts': 25, 'eps': 3}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:12,288]\u001b[0m Trial 77 finished with value: 0.0 and parameters: {'min_pts': 5, 'eps': 10}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:12,478]\u001b[0m Trial 78 finished with value: 0.25924213657179396 and parameters: {'min_pts': 25, 'eps': 2}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:12,658]\u001b[0m Trial 79 finished with value: 0.7344671391522274 and parameters: {'min_pts': 33, 'eps': 3}. Best is trial 55 with value: 0.827642731199133.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:12,843]\u001b[0m Trial 80 finished with value: 0.8300476666603241 and parameters: {'min_pts': 41, 'eps': 4}. Best is trial 80 with value: 0.8300476666603241.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:13,043]\u001b[0m Trial 81 finished with value: 0.8300476666603241 and parameters: {'min_pts': 41, 'eps': 4}. Best is trial 80 with value: 0.8300476666603241.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:13,237]\u001b[0m Trial 82 finished with value: -0.0001631473548651512 and parameters: {'min_pts': 49, 'eps': 5}. Best is trial 80 with value: 0.8300476666603241.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:13,406]\u001b[0m Trial 83 finished with value: 0.7555200348795387 and parameters: {'min_pts': 54, 'eps': 4}. Best is trial 80 with value: 0.8300476666603241.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:13,591]\u001b[0m Trial 84 finished with value: 0.829386229002961 and parameters: {'min_pts': 42, 'eps': 4}. Best is trial 80 with value: 0.8300476666603241.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:13,760]\u001b[0m Trial 85 finished with value: 0.001789533911998657 and parameters: {'min_pts': 57, 'eps': 5}. Best is trial 80 with value: 0.8300476666603241.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:13,945]\u001b[0m Trial 86 finished with value: 0.8160970772884617 and parameters: {'min_pts': 45, 'eps': 4}. Best is trial 80 with value: 0.8300476666603241.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:14,145]\u001b[0m Trial 87 finished with value: -0.0001128043867234138 and parameters: {'min_pts': 44, 'eps': 5}. Best is trial 80 with value: 0.8300476666603241.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:14,346]\u001b[0m Trial 88 finished with value: 3.0691940026804272e-15 and parameters: {'min_pts': 47, 'eps': 6}. Best is trial 80 with value: 0.8300476666603241.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:14,524]\u001b[0m Trial 89 finished with value: 0.8300476666603241 and parameters: {'min_pts': 41, 'eps': 4}. Best is trial 80 with value: 0.8300476666603241.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:14,723]\u001b[0m Trial 90 finished with value: 0.829386229002961 and parameters: {'min_pts': 42, 'eps': 4}. Best is trial 80 with value: 0.8300476666603241.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:14,900]\u001b[0m Trial 91 finished with value: 0.829386229002961 and parameters: {'min_pts': 42, 'eps': 4}. Best is trial 80 with value: 0.8300476666603241.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-06-01 18:51:15,078]\u001b[0m Trial 92 finished with value: 0.829386229002961 and parameters: {'min_pts': 42, 'eps': 4}. Best is trial 80 with value: 0.8300476666603241.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:15,263]\u001b[0m Trial 93 finished with value: 0.8300476666603241 and parameters: {'min_pts': 41, 'eps': 4}. Best is trial 80 with value: 0.8300476666603241.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:15,448]\u001b[0m Trial 94 finished with value: -0.0006429245958157786 and parameters: {'min_pts': 42, 'eps': 5}. Best is trial 80 with value: 0.8300476666603241.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:15,626]\u001b[0m Trial 95 finished with value: 0.8300476666603241 and parameters: {'min_pts': 41, 'eps': 4}. Best is trial 80 with value: 0.8300476666603241.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:15,795]\u001b[0m Trial 96 finished with value: 0.8298539801615614 and parameters: {'min_pts': 40, 'eps': 4}. Best is trial 80 with value: 0.8300476666603241.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:15,980]\u001b[0m Trial 97 finished with value: 0.8300476666603241 and parameters: {'min_pts': 41, 'eps': 4}. Best is trial 80 with value: 0.8300476666603241.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:16,187]\u001b[0m Trial 98 finished with value: 0.8298539801615614 and parameters: {'min_pts': 40, 'eps': 4}. Best is trial 80 with value: 0.8300476666603241.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 18:51:16,370]\u001b[0m Trial 99 finished with value: -0.0006250792279639105 and parameters: {'min_pts': 51, 'eps': 5}. Best is trial 80 with value: 0.8300476666603241.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "study.optimize(objective, n_trials=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "U3FnCVEum_Vi"
   },
   "outputs": [],
   "source": [
    "results = pd.DataFrame(experiment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "tVPJ0KOPnALz"
   },
   "outputs": [],
   "source": [
    "results.to_csv('../../results/optimization/DBSCAN_two_moons_HPO.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8300476666603241"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.iloc[80 * 10 : 81 * 10].mean()['DBSCAN']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 613,
     "status": "ok",
     "timestamp": 1621161432048,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "_Poggjy1hyMe",
    "outputId": "424386c8-24b8-45e4-f9f6-9a7cf53e3826"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.024435737741039746"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.iloc[80 * 10 : 81 * 10].std()['DBSCAN']"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyNFrvfaTTloHbT9oEK2zNNH",
   "collapsed_sections": [],
   "mount_file_id": "1yZZ4GfxaXLw7kQQ-ufKegMhT-sh93n-d",
   "name": "Two_Moons_DBSCAN_Hyperparameter_Search.ipynb",
   "provenance": [
    {
     "file_id": "1ovm-C9cOxmlH44OzWKrLSfVgtP3HqT18",
     "timestamp": 1620113188233
    }
   ]
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}