notebooks/Optimizations/Overoptimism_Den_Blobs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset Optimization - Blobs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To reproduce this experiment, first delete or remove the database den_blobs.db that is stored in the results/optimization folder. Otherwise, the TPE optimization will use the 100 trials stored in the existing den_blobs.db and continue with the 101st trial up to the 200th trial. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "executionInfo": {
     "elapsed": 992,
     "status": "ok",
     "timestamp": 1621077732816,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "D98-cMVM08N2"
   },
   "outputs": [],
   "source": [
    "import sys  \n",
    "sys.path.insert(0, '../..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "executionInfo": {
     "elapsed": 26690,
     "status": "ok",
     "timestamp": 1621077758539,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "UVecTsHK1AcP"
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "executionInfo": {
     "elapsed": 1994,
     "status": "ok",
     "timestamp": 1621077863918,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "lP-i52Au8kmy",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import optuna\n",
    "from optuna.samplers import TPESampler\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.datasets import make_blobs, make_moons\n",
    "from sklearn.cluster import KMeans, DBSCAN, SpectralClustering, MeanShift, estimate_bandwidth\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import adjusted_mutual_info_score as AMI\n",
    "from rock import ROCK\n",
    "\n",
    "import logging\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "executionInfo": {
     "elapsed": 804,
     "status": "ok",
     "timestamp": 1621077916332,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "CibAFzXOB1Js"
   },
   "outputs": [],
   "source": [
    "def getBlobDensities(n_centers):\n",
    "  return np.round([3/n_centers * x for x in range(1, n_centers + 1)], decimals=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "executionInfo": {
     "elapsed": 685,
     "status": "ok",
     "timestamp": 1621077915068,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "sALbXlA-1Bnq"
   },
   "outputs": [],
   "source": [
    "def objective(trial):\n",
    "    seed = 0\n",
    "\n",
    "    rock_results = []\n",
    "    other_results = []\n",
    "\n",
    "    n_samples = trial.suggest_int('n_samples', 1, 16) * 100\n",
    "    n_features = trial.suggest_int('n_features', 2, 20)\n",
    "    n_centers = trial.suggest_int('n_centers', 2, 10)\n",
    "\n",
    "    datasets = []\n",
    "\n",
    "    for s in range(seed, seed+10):\n",
    "        datasets.append(make_blobs(n_samples=n_samples, centers=n_centers, n_features=n_features, cluster_std=getBlobDensities(n_centers), random_state=s))\n",
    "        \n",
    "    step = 0\n",
    "    for dataset in datasets: \n",
    "        run = {}\n",
    "        run['step'] = step\n",
    "        run['n_samples'] = n_samples\n",
    "        run['n_centers'] = n_centers\n",
    "        run['n_features'] = n_features\n",
    "        scaler = StandardScaler()\n",
    "        data = scaler.fit_transform(dataset[0])\n",
    "        kmeans = KMeans(n_clusters=n_centers, random_state=seed).fit(data).labels_\n",
    "        run['k'] = n_centers\n",
    "        eps, min_pts = (0.2, 2 * n_features)\n",
    "        dbscan = DBSCAN(eps=eps, min_samples=min_pts).fit(data).labels_\n",
    "        run['eps'] = eps\n",
    "        run['min_pts'] = 2 * n_features\n",
    "        spectral = SpectralClustering(n_clusters=n_centers, random_state=seed).fit(data).labels_\n",
    "        bandwidth = estimate_bandwidth(data)\n",
    "        run['bandwidth'] = bandwidth\n",
    "        mean_shift = MeanShift(bandwidth=bandwidth).fit(data).labels_\n",
    "\n",
    "        rock = ROCK(tmax=15).fit(data).labels_\n",
    "\n",
    "        gt = dataset[1]\n",
    "        rock_results.append(AMI(rock, gt))\n",
    "        other_results.append(np.max([AMI(kmeans, gt), AMI(dbscan, gt), AMI(spectral, gt), AMI(mean_shift, gt)]))\n",
    "\n",
    "        run['ROCK'] = AMI(rock, gt)\n",
    "        run['K_MEANS'] = AMI(kmeans, gt)\n",
    "        run['DBSCAN'] = AMI(dbscan, gt)\n",
    "        run['SPECTRAL'] = AMI(spectral, gt)\n",
    "        run['MEAN_SHIFT'] = AMI(mean_shift, gt)\n",
    "\n",
    "        step += 1\n",
    "\n",
    "        experiment.append(run)\n",
    "\n",
    "    return np.mean(rock_results) - np.mean(other_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "executionInfo": {
     "elapsed": 841,
     "status": "ok",
     "timestamp": 1621077918244,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "sVmdS5nR8ic7"
   },
   "outputs": [],
   "source": [
    "experiment = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "executionInfo": {
     "elapsed": 559,
     "status": "ok",
     "timestamp": 1621077919192,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "d9mAiC1J8rMz"
   },
   "outputs": [],
   "source": [
    "study_name = 'den_blobs'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1017,
     "status": "ok",
     "timestamp": 1621077925295,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "oZfpmiPh8nDk",
    "outputId": "0ca29735-3c64-4371-a96f-d68ea8760f66"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-06-01 15:13:55,777]\u001b[0m A new study created in RDB with name: den_blobs\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "storage_name = f'sqlite:///../../results/optimization/{study_name}.db'\n",
    "sampler = TPESampler(seed=42)\n",
    "study = optuna.create_study(study_name=study_name, storage=storage_name, sampler=sampler, load_if_exists=True,  direction=\"maximize\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-06-01 15:14:40,726]\u001b[0m Trial 0 finished with value: -0.6082313190701353 and parameters: {'n_samples': 6, 'n_features': 20, 'n_centers': 8}. Best is trial 0 with value: -0.6082313190701353.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:15:21,394]\u001b[0m Trial 1 finished with value: -0.03773364939895252 and parameters: {'n_samples': 10, 'n_features': 4, 'n_centers': 3}. Best is trial 1 with value: -0.03773364939895252.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:15:24,552]\u001b[0m Trial 2 finished with value: -0.8526323451498112 and parameters: {'n_samples': 1, 'n_features': 18, 'n_centers': 7}. Best is trial 1 with value: -0.03773364939895252.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:16:35,672]\u001b[0m Trial 3 finished with value: -0.2194917520058074 and parameters: {'n_samples': 12, 'n_features': 2, 'n_centers': 10}. Best is trial 1 with value: -0.03773364939895252.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:17:31,696]\u001b[0m Trial 4 finished with value: -0.16328197707720904 and parameters: {'n_samples': 14, 'n_features': 6, 'n_centers': 3}. Best is trial 1 with value: -0.03773364939895252.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:17:44,258]\u001b[0m Trial 5 finished with value: -0.6119243447523015 and parameters: {'n_samples': 3, 'n_features': 7, 'n_centers': 6}. Best is trial 1 with value: -0.03773364939895252.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:18:19,699]\u001b[0m Trial 6 finished with value: -0.6333363129255779 and parameters: {'n_samples': 7, 'n_features': 7, 'n_centers': 7}. Best is trial 1 with value: -0.03773364939895252.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:18:33,425]\u001b[0m Trial 7 finished with value: -0.6938265487316795 and parameters: {'n_samples': 3, 'n_features': 7, 'n_centers': 5}. Best is trial 1 with value: -0.03773364939895252.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:19:00,369]\u001b[0m Trial 8 finished with value: -0.2674430842474804 and parameters: {'n_samples': 8, 'n_features': 16, 'n_centers': 3}. Best is trial 1 with value: -0.03773364939895252.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:19:37,929]\u001b[0m Trial 9 finished with value: -0.0015189343250560094 and parameters: {'n_samples': 9, 'n_features': 13, 'n_centers': 2}. Best is trial 9 with value: -0.0015189343250560094.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:20:51,589]\u001b[0m Trial 10 finished with value: -0.0006932226464533287 and parameters: {'n_samples': 16, 'n_features': 12, 'n_centers': 2}. Best is trial 10 with value: -0.0006932226464533287.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:22:02,112]\u001b[0m Trial 11 finished with value: 0.0003828093128325216 and parameters: {'n_samples': 16, 'n_features': 13, 'n_centers': 2}. Best is trial 11 with value: 0.0003828093128325216.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:23:14,448]\u001b[0m Trial 12 finished with value: -0.0006932226464533287 and parameters: {'n_samples': 16, 'n_features': 12, 'n_centers': 2}. Best is trial 11 with value: 0.0003828093128325216.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:24:46,664]\u001b[0m Trial 13 finished with value: -0.7414023607963146 and parameters: {'n_samples': 16, 'n_features': 14, 'n_centers': 4}. Best is trial 11 with value: 0.0003828093128325216.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:25:48,702]\u001b[0m Trial 14 finished with value: 0.0009214663206327245 and parameters: {'n_samples': 14, 'n_features': 10, 'n_centers': 2}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:27:03,653]\u001b[0m Trial 15 finished with value: -0.7553668461532926 and parameters: {'n_samples': 13, 'n_features': 10, 'n_centers': 4}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:28:03,674]\u001b[0m Trial 16 finished with value: -0.8818258366917826 and parameters: {'n_samples': 11, 'n_features': 10, 'n_centers': 4}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:29:08,405]\u001b[0m Trial 17 finished with value: -0.0007785382625751236 and parameters: {'n_samples': 14, 'n_features': 15, 'n_centers': 2}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:30:27,104]\u001b[0m Trial 18 finished with value: -0.4851258370491838 and parameters: {'n_samples': 15, 'n_features': 18, 'n_centers': 10}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:31:42,891]\u001b[0m Trial 19 finished with value: -0.47942217698024425 and parameters: {'n_samples': 12, 'n_features': 9, 'n_centers': 5}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:32:38,030]\u001b[0m Trial 20 finished with value: -0.26602593334133584 and parameters: {'n_samples': 14, 'n_features': 16, 'n_centers': 3}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:33:46,558]\u001b[0m Trial 21 finished with value: -0.0006932226464533287 and parameters: {'n_samples': 16, 'n_features': 12, 'n_centers': 2}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:34:54,946]\u001b[0m Trial 22 finished with value: 0.0005681681349104517 and parameters: {'n_samples': 16, 'n_features': 11, 'n_centers': 2}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:35:52,542]\u001b[0m Trial 23 finished with value: -0.26135660346435596 and parameters: {'n_samples': 15, 'n_features': 9, 'n_centers': 3}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:36:48,524]\u001b[0m Trial 24 finished with value: 0.0008302304122690396 and parameters: {'n_samples': 13, 'n_features': 11, 'n_centers': 2}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:37:54,485]\u001b[0m Trial 25 finished with value: -0.6911984817279127 and parameters: {'n_samples': 12, 'n_features': 11, 'n_centers': 4}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:39:15,045]\u001b[0m Trial 26 finished with value: -0.5551860635392618 and parameters: {'n_samples': 13, 'n_features': 9, 'n_centers': 5}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:39:52,395]\u001b[0m Trial 27 finished with value: -0.2631916258023559 and parameters: {'n_samples': 10, 'n_features': 10, 'n_centers': 3}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:41:11,381]\u001b[0m Trial 28 finished with value: -0.31438280238797545 and parameters: {'n_samples': 13, 'n_features': 8, 'n_centers': 9}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:42:19,075]\u001b[0m Trial 29 finished with value: 0.0007332495349391399 and parameters: {'n_samples': 15, 'n_features': 11, 'n_centers': 2}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:43:10,282]\u001b[0m Trial 30 finished with value: -0.40989907403383274 and parameters: {'n_samples': 11, 'n_features': 5, 'n_centers': 4}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:44:15,024]\u001b[0m Trial 31 finished with value: 0.0007332495349391399 and parameters: {'n_samples': 15, 'n_features': 11, 'n_centers': 2}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:45:25,326]\u001b[0m Trial 32 finished with value: 0.0 and parameters: {'n_samples': 15, 'n_features': 14, 'n_centers': 2}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:46:16,346]\u001b[0m Trial 33 finished with value: -0.2624210607177453 and parameters: {'n_samples': 14, 'n_features': 11, 'n_centers': 3}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:47:22,059]\u001b[0m Trial 34 finished with value: -0.2617949379108675 and parameters: {'n_samples': 15, 'n_features': 13, 'n_centers': 3}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:48:29,365]\u001b[0m Trial 35 finished with value: -0.025034609246004313 and parameters: {'n_samples': 13, 'n_features': 2, 'n_centers': 2}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:49:06,618]\u001b[0m Trial 36 finished with value: -0.2333152343599486 and parameters: {'n_samples': 11, 'n_features': 8, 'n_centers': 3}. Best is trial 14 with value: 0.0009214663206327245.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:50:04,651]\u001b[0m Trial 37 finished with value: 0.0017796584262947945 and parameters: {'n_samples': 12, 'n_features': 10, 'n_centers': 2}. Best is trial 37 with value: 0.0017796584262947945.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:50:58,493]\u001b[0m Trial 38 finished with value: -0.22152488938699655 and parameters: {'n_samples': 10, 'n_features': 4, 'n_centers': 6}. Best is trial 37 with value: 0.0017796584262947945.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-06-01 15:51:11,604]\u001b[0m Trial 39 finished with value: -0.2601294109595216 and parameters: {'n_samples': 5, 'n_features': 8, 'n_centers': 3}. Best is trial 37 with value: 0.0017796584262947945.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:52:25,744]\u001b[0m Trial 40 finished with value: -0.3380548439321752 and parameters: {'n_samples': 12, 'n_features': 6, 'n_centers': 7}. Best is trial 37 with value: 0.0017796584262947945.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:53:24,201]\u001b[0m Trial 41 finished with value: 0.0009214663206327245 and parameters: {'n_samples': 14, 'n_features': 10, 'n_centers': 2}. Best is trial 37 with value: 0.0017796584262947945.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:54:28,472]\u001b[0m Trial 42 finished with value: 0.0009214663206327245 and parameters: {'n_samples': 14, 'n_features': 10, 'n_centers': 2}. Best is trial 37 with value: 0.0017796584262947945.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:55:18,111]\u001b[0m Trial 43 finished with value: -0.2620591450294322 and parameters: {'n_samples': 14, 'n_features': 9, 'n_centers': 3}. Best is trial 37 with value: 0.0017796584262947945.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:56:31,741]\u001b[0m Trial 44 finished with value: -0.7126152736653144 and parameters: {'n_samples': 12, 'n_features': 10, 'n_centers': 8}. Best is trial 37 with value: 0.0017796584262947945.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:57:06,412]\u001b[0m Trial 45 finished with value: 0.0 and parameters: {'n_samples': 9, 'n_features': 12, 'n_centers': 2}. Best is trial 37 with value: 0.0017796584262947945.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:58:01,733]\u001b[0m Trial 46 finished with value: 0.001032293311968524 and parameters: {'n_samples': 13, 'n_features': 8, 'n_centers': 2}. Best is trial 37 with value: 0.0017796584262947945.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:59:08,411]\u001b[0m Trial 47 finished with value: -0.19935014100300164 and parameters: {'n_samples': 14, 'n_features': 7, 'n_centers': 3}. Best is trial 37 with value: 0.0017796584262947945.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 15:59:54,261]\u001b[0m Trial 48 finished with value: 0.013041317933929486 and parameters: {'n_samples': 11, 'n_features': 6, 'n_centers': 2}. Best is trial 48 with value: 0.013041317933929486.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:00:54,258]\u001b[0m Trial 49 finished with value: -0.45421086790574916 and parameters: {'n_samples': 11, 'n_features': 6, 'n_centers': 4}. Best is trial 48 with value: 0.013041317933929486.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:01:25,549]\u001b[0m Trial 50 finished with value: -0.11837903226178292 and parameters: {'n_samples': 9, 'n_features': 5, 'n_centers': 3}. Best is trial 48 with value: 0.013041317933929486.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:02:20,101]\u001b[0m Trial 51 finished with value: 0.001032293311968524 and parameters: {'n_samples': 13, 'n_features': 8, 'n_centers': 2}. Best is trial 48 with value: 0.013041317933929486.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:03:12,695]\u001b[0m Trial 52 finished with value: 0.00049565156189324 and parameters: {'n_samples': 12, 'n_features': 8, 'n_centers': 2}. Best is trial 48 with value: 0.013041317933929486.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:04:10,658]\u001b[0m Trial 53 finished with value: 0.009084189107050444 and parameters: {'n_samples': 13, 'n_features': 7, 'n_centers': 2}. Best is trial 48 with value: 0.013041317933929486.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:05:03,413]\u001b[0m Trial 54 finished with value: 0.009084189107050444 and parameters: {'n_samples': 13, 'n_features': 7, 'n_centers': 2}. Best is trial 48 with value: 0.013041317933929486.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:05:46,148]\u001b[0m Trial 55 finished with value: 0.00872613907248232 and parameters: {'n_samples': 11, 'n_features': 7, 'n_centers': 2}. Best is trial 48 with value: 0.013041317933929486.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:06:21,190]\u001b[0m Trial 56 finished with value: -0.19422187976911431 and parameters: {'n_samples': 10, 'n_features': 6, 'n_centers': 3}. Best is trial 48 with value: 0.013041317933929486.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:06:49,246]\u001b[0m Trial 57 finished with value: 0.008376913278588805 and parameters: {'n_samples': 7, 'n_features': 7, 'n_centers': 2}. Best is trial 48 with value: 0.013041317933929486.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:07:15,442]\u001b[0m Trial 58 finished with value: -0.028965433399896212 and parameters: {'n_samples': 7, 'n_features': 3, 'n_centers': 3}. Best is trial 48 with value: 0.013041317933929486.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:07:31,772]\u001b[0m Trial 59 finished with value: 0.013895963844576031 and parameters: {'n_samples': 5, 'n_features': 5, 'n_centers': 2}. Best is trial 59 with value: 0.013895963844576031.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:07:48,933]\u001b[0m Trial 60 finished with value: -0.5366820185511454 and parameters: {'n_samples': 5, 'n_features': 5, 'n_centers': 4}. Best is trial 59 with value: 0.013895963844576031.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:07:58,495]\u001b[0m Trial 61 finished with value: 0.017482991838864703 and parameters: {'n_samples': 3, 'n_features': 4, 'n_centers': 2}. Best is trial 61 with value: 0.017482991838864703.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:08:04,754]\u001b[0m Trial 62 finished with value: 0.02753040092321135 and parameters: {'n_samples': 2, 'n_features': 4, 'n_centers': 2}. Best is trial 62 with value: 0.02753040092321135.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:08:08,219]\u001b[0m Trial 63 finished with value: 0.00926120984861356 and parameters: {'n_samples': 1, 'n_features': 4, 'n_centers': 2}. Best is trial 62 with value: 0.02753040092321135.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:08:11,665]\u001b[0m Trial 64 finished with value: 0.01605522138489346 and parameters: {'n_samples': 1, 'n_features': 3, 'n_centers': 2}. Best is trial 62 with value: 0.02753040092321135.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:08:14,912]\u001b[0m Trial 65 finished with value: -0.040117229613198746 and parameters: {'n_samples': 1, 'n_features': 3, 'n_centers': 3}. Best is trial 62 with value: 0.02753040092321135.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:08:21,073]\u001b[0m Trial 66 finished with value: 0.02753040092321135 and parameters: {'n_samples': 2, 'n_features': 4, 'n_centers': 2}. Best is trial 62 with value: 0.02753040092321135.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:08:27,054]\u001b[0m Trial 67 finished with value: -0.02902165612731966 and parameters: {'n_samples': 2, 'n_features': 3, 'n_centers': 3}. Best is trial 62 with value: 0.02753040092321135.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:08:38,967]\u001b[0m Trial 68 finished with value: -0.053495409232081514 and parameters: {'n_samples': 3, 'n_features': 2, 'n_centers': 2}. Best is trial 62 with value: 0.02753040092321135.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:08:45,427]\u001b[0m Trial 69 finished with value: 0.02753040092321135 and parameters: {'n_samples': 2, 'n_features': 4, 'n_centers': 2}. Best is trial 62 with value: 0.02753040092321135.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:08:52,167]\u001b[0m Trial 70 finished with value: -0.1744218370341184 and parameters: {'n_samples': 2, 'n_features': 4, 'n_centers': 3}. Best is trial 62 with value: 0.02753040092321135.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:08:55,856]\u001b[0m Trial 71 finished with value: 0.00926120984861356 and parameters: {'n_samples': 1, 'n_features': 4, 'n_centers': 2}. Best is trial 62 with value: 0.02753040092321135.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:09:02,345]\u001b[0m Trial 72 finished with value: 0.02753040092321135 and parameters: {'n_samples': 2, 'n_features': 4, 'n_centers': 2}. Best is trial 62 with value: 0.02753040092321135.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:09:09,514]\u001b[0m Trial 73 finished with value: 0.033407548327088366 and parameters: {'n_samples': 2, 'n_features': 3, 'n_centers': 2}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:09:16,129]\u001b[0m Trial 74 finished with value: 0.033407548327088366 and parameters: {'n_samples': 2, 'n_features': 3, 'n_centers': 2}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:09:22,951]\u001b[0m Trial 75 finished with value: 0.022570992101012877 and parameters: {'n_samples': 2, 'n_features': 2, 'n_centers': 3}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:09:29,637]\u001b[0m Trial 76 finished with value: 0.022570992101012877 and parameters: {'n_samples': 2, 'n_features': 2, 'n_centers': 3}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:09:37,119]\u001b[0m Trial 77 finished with value: -0.2743697907983458 and parameters: {'n_samples': 2, 'n_features': 2, 'n_centers': 5}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:09:47,765]\u001b[0m Trial 78 finished with value: 0.008104212717944592 and parameters: {'n_samples': 3, 'n_features': 2, 'n_centers': 3}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-06-01 16:10:01,996]\u001b[0m Trial 79 finished with value: -0.2653784512690106 and parameters: {'n_samples': 4, 'n_features': 3, 'n_centers': 4}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:10:08,589]\u001b[0m Trial 80 finished with value: 0.022570992101012877 and parameters: {'n_samples': 2, 'n_features': 2, 'n_centers': 3}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:10:15,156]\u001b[0m Trial 81 finished with value: 0.022570992101012877 and parameters: {'n_samples': 2, 'n_features': 2, 'n_centers': 3}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:10:28,277]\u001b[0m Trial 82 finished with value: -0.0312123413451616 and parameters: {'n_samples': 4, 'n_features': 3, 'n_centers': 3}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:10:36,154]\u001b[0m Trial 83 finished with value: -0.5126161090141399 and parameters: {'n_samples': 2, 'n_features': 4, 'n_centers': 4}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:10:54,241]\u001b[0m Trial 84 finished with value: 0.011749848001169405 and parameters: {'n_samples': 4, 'n_features': 2, 'n_centers': 3}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:11:01,120]\u001b[0m Trial 85 finished with value: -0.02902165612731966 and parameters: {'n_samples': 2, 'n_features': 3, 'n_centers': 3}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:11:10,816]\u001b[0m Trial 86 finished with value: -0.007555175001586867 and parameters: {'n_samples': 3, 'n_features': 5, 'n_centers': 2}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:11:15,417]\u001b[0m Trial 87 finished with value: -0.16644095357307442 and parameters: {'n_samples': 1, 'n_features': 4, 'n_centers': 3}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:11:23,065]\u001b[0m Trial 88 finished with value: -0.432012912310147 and parameters: {'n_samples': 2, 'n_features': 3, 'n_centers': 4}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:11:36,746]\u001b[0m Trial 89 finished with value: -0.053495409232081514 and parameters: {'n_samples': 3, 'n_features': 2, 'n_centers': 2}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:11:40,068]\u001b[0m Trial 90 finished with value: -0.2147940598815703 and parameters: {'n_samples': 1, 'n_features': 5, 'n_centers': 3}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:11:46,966]\u001b[0m Trial 91 finished with value: 0.022570992101012877 and parameters: {'n_samples': 2, 'n_features': 2, 'n_centers': 3}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:11:53,629]\u001b[0m Trial 92 finished with value: 0.033407548327088366 and parameters: {'n_samples': 2, 'n_features': 3, 'n_centers': 2}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:12:03,274]\u001b[0m Trial 93 finished with value: 0.017482991838864703 and parameters: {'n_samples': 3, 'n_features': 4, 'n_centers': 2}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:12:10,665]\u001b[0m Trial 94 finished with value: 0.033407548327088366 and parameters: {'n_samples': 2, 'n_features': 3, 'n_centers': 2}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:12:25,812]\u001b[0m Trial 95 finished with value: -0.0040176146991171 and parameters: {'n_samples': 4, 'n_features': 3, 'n_centers': 2}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:12:29,504]\u001b[0m Trial 96 finished with value: -0.05654973273476127 and parameters: {'n_samples': 1, 'n_features': 5, 'n_centers': 2}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:12:39,479]\u001b[0m Trial 97 finished with value: -0.5216037487249454 and parameters: {'n_samples': 2, 'n_features': 4, 'n_centers': 6}. Best is trial 73 with value: 0.033407548327088366.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:12:52,998]\u001b[0m Trial 98 finished with value: 0.04754203356431452 and parameters: {'n_samples': 3, 'n_features': 3, 'n_centers': 2}. Best is trial 98 with value: 0.04754203356431452.\u001b[0m\n",
      "\u001b[32m[I 2021-06-01 16:13:06,429]\u001b[0m Trial 99 finished with value: 0.04754203356431452 and parameters: {'n_samples': 3, 'n_features': 3, 'n_centers': 2}. Best is trial 98 with value: 0.04754203356431452.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "study.optimize(objective, n_trials=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "executionInfo": {
     "elapsed": 671,
     "status": "ok",
     "timestamp": 1621085803125,
     "user": {
      "displayName": "Isdm Hueni",
      "photoUrl": "",
      "userId": "05072038424951783895"
     },
     "user_tz": -120
    },
    "id": "_WtdsrSipQ3l"
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(experiment).to_csv('../../results/optimization/den_blobs.csv')"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Overoptimism_Den_Blobs.ipynb",
   "provenance": [
    {
     "file_id": "1WM_hnis-71gEi8mFVu7xhXRp0tULbqsc",
     "timestamp": 1619538520754
    }
   ]
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}