notebooks/Optimizations/Overoptimism_Two_Moons.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset Optimization - Two Moons"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To reproduce this experiment, first delete or remove the database two_moons.db that is stored in the results/optimization folder. Otherwise, the TPE optimization will use the 100 trials stored in the existing two_moons.db and continue with the 101st trial up to the 200th trial. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "executionInfo": {
     "elapsed": 468,
     "status": "ok",
     "timestamp": 1621076258424,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "UbvqVKulcAwD"
   },
   "outputs": [],
   "source": [
    "import sys  \n",
    "sys.path.insert(0, '../..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 460,
     "status": "ok",
     "timestamp": 1621076507730,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "lWMfMVRSeO88",
    "outputId": "198350a5-fdf1-4046-954a-d3a4a09e0180"
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "executionInfo": {
     "elapsed": 658,
     "status": "ok",
     "timestamp": 1621076509486,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "EXvc2zQ-csmQ"
   },
   "outputs": [],
   "source": [
    "import optuna\n",
    "from optuna.samplers import TPESampler\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.datasets import make_blobs, make_moons\n",
    "from sklearn.cluster import KMeans, DBSCAN, SpectralClustering, MeanShift, estimate_bandwidth\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import adjusted_mutual_info_score as AMI\n",
    "from rock import ROCK\n",
    "\n",
    "import logging\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "executionInfo": {
     "elapsed": 465,
     "status": "ok",
     "timestamp": 1621076510782,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "eDgO4l1liEs5"
   },
   "outputs": [],
   "source": [
    "def objective(trial):\n",
    "    seed = 0\n",
    "\n",
    "    rock_results = []\n",
    "    other_results = []\n",
    "\n",
    "    n_samples = trial.suggest_int('n_samples', 1, 16) * 100\n",
    "    jitter = trial.suggest_int('jitter', 1, 20) * 0.01\n",
    "\n",
    "\n",
    "    datasets = []\n",
    "    n_centers = None\n",
    "    \n",
    "    for s in range(seed, seed+10):\n",
    "        datasets.append(make_moons(n_samples=n_samples, noise=jitter,\n",
    "            shuffle=True, random_state=s))\n",
    "        \n",
    "    step = 0\n",
    "    for dataset in datasets: \n",
    "        run = {}\n",
    "        run['step'] = step\n",
    "        run['n_samples'] = n_samples\n",
    "        run['jitter'] = jitter\n",
    "        scaler = StandardScaler()\n",
    "        data = scaler.fit_transform(dataset[0])\n",
    "        kmeans = KMeans(n_clusters=2, random_state=seed).fit(data).labels_\n",
    "        run['k'] = 2\n",
    "        eps, min_pts = (0.2, 4)\n",
    "        dbscan = DBSCAN(eps=eps, min_samples=min_pts).fit(data).labels_\n",
    "        run['eps'] = eps\n",
    "        run['min_pts'] = min_pts\n",
    "        spectral = SpectralClustering(n_clusters=2, random_state=seed).fit(data).labels_\n",
    "        bandwidth = estimate_bandwidth(data)\n",
    "        run['bandwidth'] = bandwidth\n",
    "        mean_shift = MeanShift(bandwidth=bandwidth).fit(data).labels_\n",
    "\n",
    "        rock = ROCK(tmax=15).fit(data).labels_\n",
    "\n",
    "        gt = dataset[1]\n",
    "        rock_results.append(AMI(rock, gt))\n",
    "        other_results.append(np.max([AMI(kmeans, gt), AMI(dbscan, gt), AMI(spectral, gt), AMI(mean_shift, gt)]))\n",
    "\n",
    "        run['ROCK'] = AMI(rock, gt)\n",
    "        run['K_MEANS'] = AMI(kmeans, gt)\n",
    "        run['DBSCAN'] = AMI(dbscan, gt)\n",
    "        run['SPECTRAL'] = AMI(spectral, gt)\n",
    "        run['MEAN_SHIFT'] = AMI(mean_shift, gt)\n",
    "\n",
    "        step += 1\n",
    "\n",
    "        experiment.append(run)\n",
    "\n",
    "    return np.mean(rock_results) - np.mean(other_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "executionInfo": {
     "elapsed": 690,
     "status": "ok",
     "timestamp": 1621086157755,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "sfM20A23iHzF"
   },
   "outputs": [],
   "source": [
    "experiment = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "executionInfo": {
     "elapsed": 460,
     "status": "ok",
     "timestamp": 1621086156763,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "eMBDJdRPehpG"
   },
   "outputs": [],
   "source": [
    "study_name = 'two_moons'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 734,
     "status": "ok",
     "timestamp": 1621086152378,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "R1jP84jFed9N",
    "outputId": "6a6b0c45-f87b-4961-b3bc-3f5e8519ac42"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-06-01 13:04:20,341]\u001b[0m A new study created in RDB with name: two_moons\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "storage_name = f'sqlite:///../../results/optimization/{study_name}.db'\n",
    "sampler = TPESampler(seed=42)\n",
    "study = optuna.create_study(study_name=study_name, storage=storage_name, sampler=sampler, load_if_exists=True,  direction=\"maximize\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-06-01 13:04:54,058]\u001b[0m Trial 0 finished with value: 0.17165110355273971 and parameters: {'n_samples': 6, 'jitter': 20}. Best is trial 0 with value: 0.17165110355273971.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:05:53,324]\u001b[0m Trial 1 finished with value: 0.1707221500535816 and parameters: {'n_samples': 12, 'jitter': 12}. Best is trial 0 with value: 0.17165110355273971.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:06:04,871]\u001b[0m Trial 2 finished with value: -0.1777327860237462 and parameters: {'n_samples': 3, 'jitter': 4}. Best is trial 0 with value: 0.17165110355273971.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:06:09,031]\u001b[0m Trial 3 finished with value: 0.100519992181483 and parameters: {'n_samples': 1, 'jitter': 18}. Best is trial 0 with value: 0.17165110355273971.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:06:59,006]\u001b[0m Trial 4 finished with value: 0.3580777415846451 and parameters: {'n_samples': 10, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:07:03,262]\u001b[0m Trial 5 finished with value: 0.09615812279639446 and parameters: {'n_samples': 1, 'jitter': 20}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:08:10,815]\u001b[0m Trial 6 finished with value: -0.6452263296287215 and parameters: {'n_samples': 14, 'jitter': 5}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:08:25,681]\u001b[0m Trial 7 finished with value: -0.1777327860237462 and parameters: {'n_samples': 3, 'jitter': 4}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:08:48,943]\u001b[0m Trial 8 finished with value: 0.04028581588278013 and parameters: {'n_samples': 5, 'jitter': 11}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:09:22,318]\u001b[0m Trial 9 finished with value: -0.2895366020629693 and parameters: {'n_samples': 7, 'jitter': 6}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:10:17,579]\u001b[0m Trial 10 finished with value: 0.3580777415846451 and parameters: {'n_samples': 10, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:11:14,515]\u001b[0m Trial 11 finished with value: 0.3580777415846451 and parameters: {'n_samples': 10, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:12:08,689]\u001b[0m Trial 12 finished with value: 0.3580777415846451 and parameters: {'n_samples': 10, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:13:43,706]\u001b[0m Trial 13 finished with value: 0.14706713670728033 and parameters: {'n_samples': 16, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:14:28,316]\u001b[0m Trial 14 finished with value: -0.22937890472335865 and parameters: {'n_samples': 9, 'jitter': 8}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:15:40,419]\u001b[0m Trial 15 finished with value: 0.24642902178190923 and parameters: {'n_samples': 12, 'jitter': 17}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:16:34,677]\u001b[0m Trial 16 finished with value: -0.7692835738675992 and parameters: {'n_samples': 12, 'jitter': 1}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:17:17,288]\u001b[0m Trial 17 finished with value: 0.17382676865840374 and parameters: {'n_samples': 8, 'jitter': 13}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:18:36,480]\u001b[0m Trial 18 finished with value: -0.26503836035852146 and parameters: {'n_samples': 15, 'jitter': 9}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:19:56,048]\u001b[0m Trial 19 finished with value: 0.30895642882388913 and parameters: {'n_samples': 13, 'jitter': 13}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:20:56,443]\u001b[0m Trial 20 finished with value: 0.2451708372967551 and parameters: {'n_samples': 10, 'jitter': 18}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:21:48,975]\u001b[0m Trial 21 finished with value: 0.3580777415846451 and parameters: {'n_samples': 10, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:22:31,844]\u001b[0m Trial 22 finished with value: 0.22437843526006518 and parameters: {'n_samples': 8, 'jitter': 16}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:23:34,431]\u001b[0m Trial 23 finished with value: 0.2564067667209675 and parameters: {'n_samples': 11, 'jitter': 13}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:24:24,162]\u001b[0m Trial 24 finished with value: 0.18671781195876136 and parameters: {'n_samples': 9, 'jitter': 19}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:25:24,046]\u001b[0m Trial 25 finished with value: 0.24303520985894067 and parameters: {'n_samples': 11, 'jitter': 14}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:25:52,293]\u001b[0m Trial 26 finished with value: -0.2224824848304291 and parameters: {'n_samples': 6, 'jitter': 10}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:27:19,276]\u001b[0m Trial 27 finished with value: 0.20336157681701972 and parameters: {'n_samples': 14, 'jitter': 17}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:27:50,744]\u001b[0m Trial 28 finished with value: -0.25922406126908526 and parameters: {'n_samples': 7, 'jitter': 11}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:28:55,843]\u001b[0m Trial 29 finished with value: 0.17732066213339082 and parameters: {'n_samples': 11, 'jitter': 20}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:29:17,003]\u001b[0m Trial 30 finished with value: 0.13355353933130798 and parameters: {'n_samples': 5, 'jitter': 16}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:30:08,293]\u001b[0m Trial 31 finished with value: 0.3580777415846451 and parameters: {'n_samples': 10, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:30:50,704]\u001b[0m Trial 32 finished with value: 0.2917480978699463 and parameters: {'n_samples': 9, 'jitter': 14}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:31:50,849]\u001b[0m Trial 33 finished with value: 0.1707221500535816 and parameters: {'n_samples': 12, 'jitter': 12}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:32:49,344]\u001b[0m Trial 34 finished with value: 0.2857950185115127 and parameters: {'n_samples': 10, 'jitter': 17}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:33:28,867]\u001b[0m Trial 35 finished with value: 0.32198910475527726 and parameters: {'n_samples': 8, 'jitter': 14}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:34:44,238]\u001b[0m Trial 36 finished with value: 0.21453842847454235 and parameters: {'n_samples': 13, 'jitter': 19}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:35:48,460]\u001b[0m Trial 37 finished with value: 0.26614851507529974 and parameters: {'n_samples': 11, 'jitter': 16}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:36:19,974]\u001b[0m Trial 38 finished with value: -0.026053134604077433 and parameters: {'n_samples': 7, 'jitter': 12}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:37:32,570]\u001b[0m Trial 39 finished with value: 0.24066649345377633 and parameters: {'n_samples': 13, 'jitter': 18}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 13:38:17,969]\u001b[0m Trial 40 finished with value: 0.31115339290221 and parameters: {'n_samples': 9, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:07:12,506]\u001b[0m Trial 41 finished with value: 0.3580777415846451 and parameters: {'n_samples': 10, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:08:05,501]\u001b[0m Trial 42 finished with value: 0.2841854437682288 and parameters: {'n_samples': 10, 'jitter': 16}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:08:56,658]\u001b[0m Trial 43 finished with value: 0.2564067667209675 and parameters: {'n_samples': 11, 'jitter': 13}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-06-01 14:09:44,001]\u001b[0m Trial 44 finished with value: 0.3196228543529997 and parameters: {'n_samples': 10, 'jitter': 14}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:10:25,271]\u001b[0m Trial 45 finished with value: 0.3462561084025223 and parameters: {'n_samples': 9, 'jitter': 12}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:11:02,361]\u001b[0m Trial 46 finished with value: 0.2418578202112659 and parameters: {'n_samples': 8, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:12:05,025]\u001b[0m Trial 47 finished with value: 0.24642902178190923 and parameters: {'n_samples': 12, 'jitter': 17}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:12:37,795]\u001b[0m Trial 48 finished with value: 0.1671843432233151 and parameters: {'n_samples': 7, 'jitter': 19}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:13:27,554]\u001b[0m Trial 49 finished with value: -0.24286667062275336 and parameters: {'n_samples': 11, 'jitter': 10}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:14:14,282]\u001b[0m Trial 50 finished with value: 0.3196228543529997 and parameters: {'n_samples': 10, 'jitter': 14}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:15:03,153]\u001b[0m Trial 51 finished with value: 0.3580777415846451 and parameters: {'n_samples': 10, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:15:52,024]\u001b[0m Trial 52 finished with value: 0.3580777415846451 and parameters: {'n_samples': 10, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:16:38,166]\u001b[0m Trial 53 finished with value: 0.2655614416957731 and parameters: {'n_samples': 9, 'jitter': 16}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:17:17,106]\u001b[0m Trial 54 finished with value: 0.2869335258517937 and parameters: {'n_samples': 8, 'jitter': 17}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:18:19,876]\u001b[0m Trial 55 finished with value: 0.17936057627260427 and parameters: {'n_samples': 11, 'jitter': 18}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:19:24,479]\u001b[0m Trial 56 finished with value: 0.328785310283695 and parameters: {'n_samples': 12, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:22:44,068]\u001b[0m Trial 57 finished with value: 0.292791568933074 and parameters: {'n_samples': 10, 'jitter': 13}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:23:27,885]\u001b[0m Trial 58 finished with value: 0.2655614416957731 and parameters: {'n_samples': 9, 'jitter': 16}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:24:24,311]\u001b[0m Trial 59 finished with value: -0.7792480639734296 and parameters: {'n_samples': 13, 'jitter': 1}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:25:12,649]\u001b[0m Trial 60 finished with value: 0.3580777415846451 and parameters: {'n_samples': 10, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:26:15,828]\u001b[0m Trial 61 finished with value: 0.3580777415846451 and parameters: {'n_samples': 10, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:27:49,601]\u001b[0m Trial 62 finished with value: 0.2917480978699463 and parameters: {'n_samples': 9, 'jitter': 14}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:28:56,207]\u001b[0m Trial 63 finished with value: 0.17601762758483475 and parameters: {'n_samples': 11, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:30:00,459]\u001b[0m Trial 64 finished with value: 0.23968111188644925 and parameters: {'n_samples': 12, 'jitter': 13}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:30:37,920]\u001b[0m Trial 65 finished with value: 0.09483551295096393 and parameters: {'n_samples': 8, 'jitter': 11}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:31:38,632]\u001b[0m Trial 66 finished with value: 0.26614851507529974 and parameters: {'n_samples': 11, 'jitter': 16}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:32:24,074]\u001b[0m Trial 67 finished with value: 0.2917480978699463 and parameters: {'n_samples': 9, 'jitter': 14}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:33:17,687]\u001b[0m Trial 68 finished with value: 0.2857950185115127 and parameters: {'n_samples': 10, 'jitter': 17}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:33:42,358]\u001b[0m Trial 69 finished with value: -0.16286755877486947 and parameters: {'n_samples': 6, 'jitter': 8}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:34:36,924]\u001b[0m Trial 70 finished with value: 0.2564067667209675 and parameters: {'n_samples': 11, 'jitter': 13}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:35:27,083]\u001b[0m Trial 71 finished with value: 0.3580777415846451 and parameters: {'n_samples': 10, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:36:20,309]\u001b[0m Trial 72 finished with value: 0.3580777415846451 and parameters: {'n_samples': 10, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:37:08,508]\u001b[0m Trial 73 finished with value: 0.3196228543529997 and parameters: {'n_samples': 10, 'jitter': 14}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:37:52,011]\u001b[0m Trial 74 finished with value: 0.2655614416957731 and parameters: {'n_samples': 9, 'jitter': 16}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:38:43,634]\u001b[0m Trial 75 finished with value: 0.3196228543529997 and parameters: {'n_samples': 10, 'jitter': 14}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:39:48,469]\u001b[0m Trial 76 finished with value: 0.328785310283695 and parameters: {'n_samples': 12, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:40:41,244]\u001b[0m Trial 77 finished with value: 0.2451708372967551 and parameters: {'n_samples': 10, 'jitter': 18}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:41:40,166]\u001b[0m Trial 78 finished with value: 0.21485171773125555 and parameters: {'n_samples': 11, 'jitter': 17}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:42:20,785]\u001b[0m Trial 79 finished with value: 0.16003428146581766 and parameters: {'n_samples': 8, 'jitter': 12}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:43:04,377]\u001b[0m Trial 80 finished with value: 0.2655614416957731 and parameters: {'n_samples': 9, 'jitter': 16}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:44:09,495]\u001b[0m Trial 81 finished with value: 0.17601762758483475 and parameters: {'n_samples': 11, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:45:12,255]\u001b[0m Trial 82 finished with value: 0.3196228543529997 and parameters: {'n_samples': 10, 'jitter': 14}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:46:08,853]\u001b[0m Trial 83 finished with value: 0.31115339290221 and parameters: {'n_samples': 9, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:46:17,647]\u001b[0m Trial 84 finished with value: 0.266609453370082 and parameters: {'n_samples': 2, 'jitter': 13}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:47:27,340]\u001b[0m Trial 85 finished with value: 0.2857950185115127 and parameters: {'n_samples': 10, 'jitter': 17}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:48:34,389]\u001b[0m Trial 86 finished with value: 0.26614851507529974 and parameters: {'n_samples': 11, 'jitter': 16}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:49:35,088]\u001b[0m Trial 87 finished with value: 0.3580777415846451 and parameters: {'n_samples': 10, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-06-01 14:50:29,948]\u001b[0m Trial 88 finished with value: 0.3580777415846451 and parameters: {'n_samples': 10, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:51:50,215]\u001b[0m Trial 89 finished with value: 0.27018874201047866 and parameters: {'n_samples': 12, 'jitter': 14}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:52:43,856]\u001b[0m Trial 90 finished with value: 0.2852639926243666 and parameters: {'n_samples': 9, 'jitter': 13}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:53:41,660]\u001b[0m Trial 91 finished with value: 0.3580777415846451 and parameters: {'n_samples': 10, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:54:40,393]\u001b[0m Trial 92 finished with value: 0.3580777415846451 and parameters: {'n_samples': 10, 'jitter': 15}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:55:56,543]\u001b[0m Trial 93 finished with value: 0.26614851507529974 and parameters: {'n_samples': 11, 'jitter': 16}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:56:48,648]\u001b[0m Trial 94 finished with value: 0.2655614416957731 and parameters: {'n_samples': 9, 'jitter': 16}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:57:41,084]\u001b[0m Trial 95 finished with value: 0.24458288107763604 and parameters: {'n_samples': 8, 'jitter': 18}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:58:38,434]\u001b[0m Trial 96 finished with value: 0.24303520985894067 and parameters: {'n_samples': 11, 'jitter': 14}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 14:59:21,465]\u001b[0m Trial 97 finished with value: 0.3462561084025223 and parameters: {'n_samples': 9, 'jitter': 12}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:00:33,039]\u001b[0m Trial 98 finished with value: 0.21485171773125555 and parameters: {'n_samples': 11, 'jitter': 17}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:01:48,205]\u001b[0m Trial 99 finished with value: 0.32406850885661703 and parameters: {'n_samples': 12, 'jitter': 16}. Best is trial 4 with value: 0.3580777415846451.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "study.optimize(objective, n_trials=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "executionInfo": {
     "elapsed": 841,
     "status": "ok",
     "timestamp": 1621085740394,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "U3FnCVEum_Vi"
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(experiment).to_csv('../../results/optimization/two_moons.csv')"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyOzwpQPVVT/6iODl9VQ5VRB",
   "collapsed_sections": [],
   "mount_file_id": "1ovm-C9cOxmlH44OzWKrLSfVgtP3HqT18",
   "name": "OveropTwo_Moons.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}