notebooks/Optimizations/Overoptimism_Rings.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset Optimization - Rings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To reproduce this experiment, first delete or remove the database rings.db that is stored in the results/optimization folder. Otherwise, the TPE optimization will use the 100 trials stored in the existing rings.db and continue with the 101st trial up to the 200th trial. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "executionInfo": {
     "elapsed": 676,
     "status": "ok",
     "timestamp": 1621077674116,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "D98-cMVM08N2"
   },
   "outputs": [],
   "source": [
    "import sys  \n",
    "sys.path.insert(0, '../..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1421,
     "status": "ok",
     "timestamp": 1621077678202,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "UVecTsHK1AcP",
    "outputId": "cc29dec0-a8b5-4226-f54f-bf7d1e63b908"
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "executionInfo": {
     "elapsed": 628,
     "status": "ok",
     "timestamp": 1621077678203,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "lP-i52Au8kmy"
   },
   "outputs": [],
   "source": [
    "import optuna\n",
    "from optuna.samplers import TPESampler\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.datasets import make_blobs, make_moons, make_circles\n",
    "from sklearn.cluster import KMeans, DBSCAN, SpectralClustering, MeanShift, estimate_bandwidth\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import adjusted_mutual_info_score as AMI\n",
    "from rock import ROCK\n",
    "\n",
    "import logging\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "executionInfo": {
     "elapsed": 696,
     "status": "ok",
     "timestamp": 1621077680997,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "sALbXlA-1Bnq"
   },
   "outputs": [],
   "source": [
    "def objective(trial):\n",
    "    seed = 0\n",
    "\n",
    "    rock_results = []\n",
    "    other_results = []\n",
    "\n",
    "    n_samples = trial.suggest_int('n_samples', 1, 16) * 100\n",
    "    jitter = trial.suggest_int('jitter', 1, 20) * 0.01\n",
    "\n",
    "    datasets = []\n",
    "\n",
    "    for s in range(seed, seed+10):\n",
    "        datasets.append(make_circles(n_samples=n_samples,\n",
    "                                     noise=jitter, \n",
    "                                     random_state=s))\n",
    "        \n",
    "    step = 0\n",
    "    for dataset in datasets: \n",
    "        run = {}\n",
    "        run['step'] = step\n",
    "        run['n_samples'] = n_samples\n",
    "        run['jitter'] = jitter\n",
    "        scaler = StandardScaler()\n",
    "        data = scaler.fit_transform(dataset[0])\n",
    "        kmeans = KMeans(n_clusters=2, random_state=seed).fit(data).labels_\n",
    "        run['k'] = 2\n",
    "        eps, min_pts = (0.2, 4)\n",
    "        dbscan = DBSCAN(eps=eps, min_samples=min_pts).fit(data).labels_\n",
    "        run['eps'] = eps\n",
    "        run['min_pts'] = min_pts\n",
    "        spectral = SpectralClustering(n_clusters=2, random_state=seed).fit(data).labels_\n",
    "        bandwidth = estimate_bandwidth(data)\n",
    "        run['bandwidth'] = bandwidth\n",
    "        mean_shift = MeanShift(bandwidth=bandwidth).fit(data).labels_\n",
    "\n",
    "        rock = ROCK(tmax=15).fit(data).labels_\n",
    "\n",
    "        gt = dataset[1]\n",
    "        rock_results.append(AMI(rock, gt))\n",
    "        other_results.append(np.max([AMI(kmeans, gt), AMI(dbscan, gt), AMI(spectral, gt), AMI(mean_shift, gt)]))\n",
    "\n",
    "        run['ROCK'] = AMI(rock, gt)\n",
    "        run['K_MEANS'] = AMI(kmeans, gt)\n",
    "        run['DBSCAN'] = AMI(dbscan, gt)\n",
    "        run['SPECTRAL'] = AMI(spectral, gt)\n",
    "        run['MEAN_SHIFT'] = AMI(mean_shift, gt)\n",
    "\n",
    "        step += 1\n",
    "\n",
    "        experiment.append(run)\n",
    "\n",
    "    return np.mean(rock_results) - np.mean(other_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "executionInfo": {
     "elapsed": 895,
     "status": "ok",
     "timestamp": 1621077679754,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "sVmdS5nR8ic7"
   },
   "outputs": [],
   "source": [
    "experiment = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "executionInfo": {
     "elapsed": 517,
     "status": "ok",
     "timestamp": 1621077683566,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "d9mAiC1J8rMz"
   },
   "outputs": [],
   "source": [
    "study_name = 'rings'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 792,
     "status": "ok",
     "timestamp": 1621077687006,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "oZfpmiPh8nDk",
    "outputId": "e3553afb-b73a-4237-a9bc-17a436ab7337"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-06-01 16:17:45,623]\u001b[0m A new study created in RDB with name: rings\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "storage_name = f'sqlite:///../../results/optimization/{study_name}.db'\n",
    "sampler = TPESampler(seed=42)\n",
    "study = optuna.create_study(study_name=study_name, storage=storage_name, sampler=sampler, load_if_exists=True,  direction=\"maximize\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-06-01 16:18:36,069]\u001b[0m Trial 0 finished with value: -0.02142656584239766 and parameters: {'n_samples': 6, 'jitter': 20}. Best is trial 0 with value: -0.02142656584239766.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:19:46,897]\u001b[0m Trial 1 finished with value: 0.0019477457941735627 and parameters: {'n_samples': 12, 'jitter': 12}. Best is trial 1 with value: 0.0019477457941735627.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:19:54,637]\u001b[0m Trial 2 finished with value: -0.03451547094700948 and parameters: {'n_samples': 3, 'jitter': 4}. Best is trial 1 with value: 0.0019477457941735627.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:19:58,094]\u001b[0m Trial 3 finished with value: -0.006095386668716002 and parameters: {'n_samples': 1, 'jitter': 18}. Best is trial 1 with value: 0.0019477457941735627.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:20:55,043]\u001b[0m Trial 4 finished with value: -0.010851512210408872 and parameters: {'n_samples': 10, 'jitter': 15}. Best is trial 1 with value: 0.0019477457941735627.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:20:58,590]\u001b[0m Trial 5 finished with value: -0.007955213696754998 and parameters: {'n_samples': 1, 'jitter': 20}. Best is trial 1 with value: 0.0019477457941735627.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:21:58,965]\u001b[0m Trial 6 finished with value: 0.007022758619026565 and parameters: {'n_samples': 14, 'jitter': 5}. Best is trial 6 with value: 0.007022758619026565.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:22:07,112]\u001b[0m Trial 7 finished with value: -0.03451547094700948 and parameters: {'n_samples': 3, 'jitter': 4}. Best is trial 6 with value: 0.007022758619026565.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:22:30,712]\u001b[0m Trial 8 finished with value: -0.010469730406427565 and parameters: {'n_samples': 5, 'jitter': 11}. Best is trial 6 with value: 0.007022758619026565.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:23:00,946]\u001b[0m Trial 9 finished with value: 0.003921525180912526 and parameters: {'n_samples': 7, 'jitter': 6}. Best is trial 6 with value: 0.007022758619026565.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:23:51,610]\u001b[0m Trial 10 finished with value: -0.8413867837651066 and parameters: {'n_samples': 16, 'jitter': 1}. Best is trial 6 with value: 0.007022758619026565.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:25:16,062]\u001b[0m Trial 11 finished with value: 0.005881701915558263 and parameters: {'n_samples': 16, 'jitter': 7}. Best is trial 6 with value: 0.007022758619026565.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:26:38,131]\u001b[0m Trial 12 finished with value: 0.00549560383290051 and parameters: {'n_samples': 15, 'jitter': 8}. Best is trial 6 with value: 0.007022758619026565.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:27:16,393]\u001b[0m Trial 13 finished with value: -0.8391450638722207 and parameters: {'n_samples': 13, 'jitter': 1}. Best is trial 6 with value: 0.007022758619026565.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:28:39,877]\u001b[0m Trial 14 finished with value: 0.0060723688166045885 and parameters: {'n_samples': 14, 'jitter': 8}. Best is trial 6 with value: 0.007022758619026565.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:29:54,711]\u001b[0m Trial 15 finished with value: 0.002859097632205054 and parameters: {'n_samples': 13, 'jitter': 10}. Best is trial 6 with value: 0.007022758619026565.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:30:33,759]\u001b[0m Trial 16 finished with value: 0.015381630723519255 and parameters: {'n_samples': 10, 'jitter': 4}. Best is trial 16 with value: 0.015381630723519255.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:31:08,120]\u001b[0m Trial 17 finished with value: 0.037617259937406944 and parameters: {'n_samples': 10, 'jitter': 3}. Best is trial 17 with value: 0.037617259937406944.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:31:36,446]\u001b[0m Trial 18 finished with value: 0.17597702663426662 and parameters: {'n_samples': 10, 'jitter': 2}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:32:03,046]\u001b[0m Trial 19 finished with value: -0.83642090197778 and parameters: {'n_samples': 10, 'jitter': 1}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:32:25,575]\u001b[0m Trial 20 finished with value: 0.1479836478374353 and parameters: {'n_samples': 8, 'jitter': 2}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:32:50,803]\u001b[0m Trial 21 finished with value: 0.1479836478374353 and parameters: {'n_samples': 8, 'jitter': 2}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:33:16,699]\u001b[0m Trial 22 finished with value: 0.1479836478374353 and parameters: {'n_samples': 8, 'jitter': 2}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:33:40,994]\u001b[0m Trial 23 finished with value: 0.1479836478374353 and parameters: {'n_samples': 8, 'jitter': 2}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:33:57,440]\u001b[0m Trial 24 finished with value: 0.0039019895826933842 and parameters: {'n_samples': 5, 'jitter': 6}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:34:36,043]\u001b[0m Trial 25 finished with value: 0.04893359520426764 and parameters: {'n_samples': 11, 'jitter': 3}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:35:00,023]\u001b[0m Trial 26 finished with value: -0.8382428061354813 and parameters: {'n_samples': 9, 'jitter': 1}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:35:33,640]\u001b[0m Trial 27 finished with value: -0.0022903145201720555 and parameters: {'n_samples': 7, 'jitter': 9}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:36:27,849]\u001b[0m Trial 28 finished with value: -0.010015527537412562 and parameters: {'n_samples': 9, 'jitter': 14}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:36:44,815]\u001b[0m Trial 29 finished with value: 0.02532098656375508 and parameters: {'n_samples': 6, 'jitter': 3}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:37:08,696]\u001b[0m Trial 30 finished with value: 0.00916373480376921 and parameters: {'n_samples': 7, 'jitter': 5}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:37:31,339]\u001b[0m Trial 31 finished with value: 0.1479836478374353 and parameters: {'n_samples': 8, 'jitter': 2}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:38:03,775]\u001b[0m Trial 32 finished with value: -0.8368852839506938 and parameters: {'n_samples': 11, 'jitter': 1}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:38:20,754]\u001b[0m Trial 33 finished with value: 0.12879023903282436 and parameters: {'n_samples': 6, 'jitter': 2}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:38:52,277]\u001b[0m Trial 34 finished with value: 0.017349970788047474 and parameters: {'n_samples': 8, 'jitter': 5}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:39:05,916]\u001b[0m Trial 35 finished with value: 0.011028363116559439 and parameters: {'n_samples': 4, 'jitter': 3}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:39:40,683]\u001b[0m Trial 36 finished with value: 0.16292888501735076 and parameters: {'n_samples': 11, 'jitter': 2}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:40:32,763]\u001b[0m Trial 37 finished with value: 0.010344697157425337 and parameters: {'n_samples': 12, 'jitter': 4}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:41:23,420]\u001b[0m Trial 38 finished with value: 0.011199049568729354 and parameters: {'n_samples': 11, 'jitter': 6}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:42:46,264]\u001b[0m Trial 39 finished with value: -0.001919099829834462 and parameters: {'n_samples': 12, 'jitter': 13}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:43:36,195]\u001b[0m Trial 40 finished with value: -0.00909303750678471 and parameters: {'n_samples': 9, 'jitter': 16}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:44:02,534]\u001b[0m Trial 41 finished with value: 0.15936768962926667 and parameters: {'n_samples': 9, 'jitter': 2}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:44:37,819]\u001b[0m Trial 42 finished with value: 0.015381630723519255 and parameters: {'n_samples': 10, 'jitter': 4}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:45:06,527]\u001b[0m Trial 43 finished with value: 0.15936768962926667 and parameters: {'n_samples': 9, 'jitter': 2}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-06-01 16:45:45,192]\u001b[0m Trial 44 finished with value: 0.04893359520426764 and parameters: {'n_samples': 11, 'jitter': 3}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:46:09,521]\u001b[0m Trial 45 finished with value: -0.8382428061354813 and parameters: {'n_samples': 9, 'jitter': 1}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:46:51,500]\u001b[0m Trial 46 finished with value: 0.014908427039648557 and parameters: {'n_samples': 10, 'jitter': 5}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:48:28,118]\u001b[0m Trial 47 finished with value: -0.011764404984306665 and parameters: {'n_samples': 13, 'jitter': 19}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:48:47,205]\u001b[0m Trial 48 finished with value: 0.011689196725315465 and parameters: {'n_samples': 6, 'jitter': 4}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:49:16,268]\u001b[0m Trial 49 finished with value: 0.0036874326375312474 and parameters: {'n_samples': 7, 'jitter': 7}. Best is trial 18 with value: 0.17597702663426662.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:50:01,645]\u001b[0m Trial 50 finished with value: 0.1775234037892933 and parameters: {'n_samples': 12, 'jitter': 2}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:50:40,371]\u001b[0m Trial 51 finished with value: 0.1775234037892933 and parameters: {'n_samples': 12, 'jitter': 2}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:51:23,136]\u001b[0m Trial 52 finished with value: -0.8402494518463727 and parameters: {'n_samples': 14, 'jitter': 1}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:52:14,914]\u001b[0m Trial 53 finished with value: 0.04913213673453489 and parameters: {'n_samples': 12, 'jitter': 3}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:52:55,054]\u001b[0m Trial 54 finished with value: 0.1775234037892933 and parameters: {'n_samples': 12, 'jitter': 2}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:53:31,914]\u001b[0m Trial 55 finished with value: -0.8391450638722207 and parameters: {'n_samples': 13, 'jitter': 1}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:54:34,895]\u001b[0m Trial 56 finished with value: 0.007022758619026565 and parameters: {'n_samples': 14, 'jitter': 5}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:55:40,352]\u001b[0m Trial 57 finished with value: 0.008229325404744595 and parameters: {'n_samples': 15, 'jitter': 4}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:56:22,668]\u001b[0m Trial 58 finished with value: 0.1775234037892933 and parameters: {'n_samples': 12, 'jitter': 2}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:57:12,340]\u001b[0m Trial 59 finished with value: 0.04913213673453489 and parameters: {'n_samples': 12, 'jitter': 3}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:57:48,979]\u001b[0m Trial 60 finished with value: -0.8391450638722207 and parameters: {'n_samples': 13, 'jitter': 1}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:58:27,167]\u001b[0m Trial 61 finished with value: 0.16292888501735076 and parameters: {'n_samples': 11, 'jitter': 2}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:59:03,842]\u001b[0m Trial 62 finished with value: 0.16292888501735076 and parameters: {'n_samples': 11, 'jitter': 2}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:59:45,164]\u001b[0m Trial 63 finished with value: 0.04913213673453489 and parameters: {'n_samples': 12, 'jitter': 3}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:00:18,232]\u001b[0m Trial 64 finished with value: 0.16292888501735076 and parameters: {'n_samples': 11, 'jitter': 2}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:01:19,247]\u001b[0m Trial 65 finished with value: 0.012817247046428601 and parameters: {'n_samples': 13, 'jitter': 4}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:01:44,934]\u001b[0m Trial 66 finished with value: -0.83642090197778 and parameters: {'n_samples': 10, 'jitter': 1}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:02:23,978]\u001b[0m Trial 67 finished with value: 0.1775234037892933 and parameters: {'n_samples': 12, 'jitter': 2}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:03:24,755]\u001b[0m Trial 68 finished with value: 0.008550923642589327 and parameters: {'n_samples': 12, 'jitter': 6}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:04:25,727]\u001b[0m Trial 69 finished with value: 0.04431947882268261 and parameters: {'n_samples': 15, 'jitter': 3}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:05:05,536]\u001b[0m Trial 70 finished with value: -0.8391450638722207 and parameters: {'n_samples': 13, 'jitter': 1}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:05:43,917]\u001b[0m Trial 71 finished with value: 0.16292888501735076 and parameters: {'n_samples': 11, 'jitter': 2}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:06:21,652]\u001b[0m Trial 72 finished with value: 0.1775234037892933 and parameters: {'n_samples': 12, 'jitter': 2}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:07:05,756]\u001b[0m Trial 73 finished with value: 0.04913213673453489 and parameters: {'n_samples': 12, 'jitter': 3}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:08:00,742]\u001b[0m Trial 74 finished with value: 0.025613185962392277 and parameters: {'n_samples': 14, 'jitter': 3}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:08:39,366]\u001b[0m Trial 75 finished with value: -0.8377256182050731 and parameters: {'n_samples': 12, 'jitter': 1}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:09:15,873]\u001b[0m Trial 76 finished with value: 0.015381630723519255 and parameters: {'n_samples': 10, 'jitter': 4}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:09:51,188]\u001b[0m Trial 77 finished with value: 0.16292888501735076 and parameters: {'n_samples': 11, 'jitter': 2}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:10:49,424]\u001b[0m Trial 78 finished with value: 0.012817247046428601 and parameters: {'n_samples': 13, 'jitter': 4}. Best is trial 50 with value: 0.1775234037892933.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:11:37,275]\u001b[0m Trial 79 finished with value: 0.1776092548696203 and parameters: {'n_samples': 14, 'jitter': 2}. Best is trial 79 with value: 0.1776092548696203.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:12:38,196]\u001b[0m Trial 80 finished with value: 0.06224135093179387 and parameters: {'n_samples': 16, 'jitter': 3}. Best is trial 79 with value: 0.1776092548696203.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:13:28,611]\u001b[0m Trial 81 finished with value: 0.17886297832745685 and parameters: {'n_samples': 15, 'jitter': 2}. Best is trial 81 with value: 0.17886297832745685.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:14:22,135]\u001b[0m Trial 82 finished with value: 0.17886297832745685 and parameters: {'n_samples': 15, 'jitter': 2}. Best is trial 81 with value: 0.17886297832745685.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:15:12,369]\u001b[0m Trial 83 finished with value: -0.8398635684066963 and parameters: {'n_samples': 15, 'jitter': 1}. Best is trial 81 with value: 0.17886297832745685.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:16:03,727]\u001b[0m Trial 84 finished with value: 0.17886297832745685 and parameters: {'n_samples': 15, 'jitter': 2}. Best is trial 81 with value: 0.17886297832745685.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:16:53,366]\u001b[0m Trial 85 finished with value: -0.8398635684066963 and parameters: {'n_samples': 15, 'jitter': 1}. Best is trial 81 with value: 0.17886297832745685.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:17:58,331]\u001b[0m Trial 86 finished with value: 0.1789493922434452 and parameters: {'n_samples': 16, 'jitter': 2}. Best is trial 86 with value: 0.1789493922434452.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:19:04,222]\u001b[0m Trial 87 finished with value: 0.06224135093179387 and parameters: {'n_samples': 16, 'jitter': 3}. Best is trial 86 with value: 0.1789493922434452.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-06-01 17:20:18,042]\u001b[0m Trial 88 finished with value: 0.0076502401050678554 and parameters: {'n_samples': 15, 'jitter': 5}. Best is trial 86 with value: 0.1789493922434452.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:22:05,478]\u001b[0m Trial 89 finished with value: 0.0018143812211078294 and parameters: {'n_samples': 16, 'jitter': 11}. Best is trial 86 with value: 0.1789493922434452.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:23:09,213]\u001b[0m Trial 90 finished with value: 0.008229325404744595 and parameters: {'n_samples': 15, 'jitter': 4}. Best is trial 86 with value: 0.1789493922434452.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:23:59,835]\u001b[0m Trial 91 finished with value: 0.1776092548696203 and parameters: {'n_samples': 14, 'jitter': 2}. Best is trial 86 with value: 0.1789493922434452.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:24:49,725]\u001b[0m Trial 92 finished with value: 0.1776092548696203 and parameters: {'n_samples': 14, 'jitter': 2}. Best is trial 86 with value: 0.1789493922434452.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:25:31,512]\u001b[0m Trial 93 finished with value: -0.8402494518463727 and parameters: {'n_samples': 14, 'jitter': 1}. Best is trial 86 with value: 0.1789493922434452.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:26:25,881]\u001b[0m Trial 94 finished with value: 0.025613185962392277 and parameters: {'n_samples': 14, 'jitter': 3}. Best is trial 86 with value: 0.1789493922434452.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:27:15,999]\u001b[0m Trial 95 finished with value: 0.1776092548696203 and parameters: {'n_samples': 14, 'jitter': 2}. Best is trial 86 with value: 0.1789493922434452.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:27:58,972]\u001b[0m Trial 96 finished with value: -0.8402494518463727 and parameters: {'n_samples': 14, 'jitter': 1}. Best is trial 86 with value: 0.1789493922434452.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:28:56,030]\u001b[0m Trial 97 finished with value: 0.1789493922434452 and parameters: {'n_samples': 16, 'jitter': 2}. Best is trial 86 with value: 0.1789493922434452.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:30:00,809]\u001b[0m Trial 98 finished with value: 0.06224135093179387 and parameters: {'n_samples': 16, 'jitter': 3}. Best is trial 86 with value: 0.1789493922434452.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 17:30:50,477]\u001b[0m Trial 99 finished with value: 0.17886297832745685 and parameters: {'n_samples': 15, 'jitter': 2}. Best is trial 86 with value: 0.1789493922434452.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "study.optimize(objective, n_trials=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "executionInfo": {
     "elapsed": 1152,
     "status": "ok",
     "timestamp": 1621084553902,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "_WtdsrSipQ3l"
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(experiment).to_csv('../../results/optimization/rings.csv')"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Overoptimism_Rings.ipynb",
   "provenance": [
    {
     "file_id": "1gZI4K7U86AGO3WSmrSjCR8j5RZd1CWOS",
     "timestamp": 1619594777428
    },
    {
     "file_id": "1WM_hnis-71gEi8mFVu7xhXRp0tULbqsc",
     "timestamp": 1619538520754
    }
   ]
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}