Caching.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Speeding things up with caching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.56 s, sys: 11.8 ms, total: 2.57 s\n",
      "Wall time: 2.57 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "from creme import datasets\n",
    "\n",
    "X_y = datasets.ToulouseBikes()\n",
    "\n",
    "for x, y in X_y:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.71 s, sys: 129 ms, total: 3.83 s\n",
      "Wall time: 4.07 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "from creme import stream\n",
    "\n",
    "cache = stream.Cache(directory='.')\n",
    "\n",
    "for x, y in cache(X_y, key='bikes'):\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 586 ms, sys: 36.1 ms, total: 622 ms\n",
      "Wall time: 621 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "from creme import stream\n",
    "\n",
    "cache = stream.Cache(directory='.')\n",
    "\n",
    "for x, y in cache(X_y, key='bikes'):\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       ".\n",
       "bikes - 24.5MiB"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "."
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cache.clear('bikes')\n",
    "cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime as dt\n",
    "from creme import compose\n",
    "from creme import datasets\n",
    "from creme import feature_extraction\n",
    "from creme import linear_model\n",
    "from creme import metrics\n",
    "from creme import preprocessing\n",
    "from creme import stats\n",
    "\n",
    "X_y = datasets.ToulouseBikes()\n",
    "\n",
    "def add_hour(x):\n",
    "    x['hour'] = x['moment'].hour\n",
    "    return x\n",
    "\n",
    "def make_model():\n",
    "    model = compose.Whitelister('clouds', 'humidity', 'pressure', 'temperature', 'wind')\n",
    "    model += (\n",
    "        add_hour |\n",
    "        feature_extraction.TargetAgg(by=['station', 'hour'], how=stats.Mean())\n",
    "    )\n",
    "    model += feature_extraction.TargetAgg(by='station', how=stats.EWMean(0.5))\n",
    "    model |= preprocessing.StandardScaler()\n",
    "    model |= linear_model.LinearRegression()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[30,000] MAE: 2.230049\n",
      "[60,000] MAE: 2.290409\n",
      "[90,000] MAE: 2.334638\n",
      "[120,000] MAE: 2.315149\n",
      "[150,000] MAE: 2.319982\n",
      "[180,000] MAE: 2.335385\n",
      "CPU times: user 22.6 s, sys: 59.7 ms, total: 22.6 s\n",
      "Wall time: 22.7 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "MAE: 2.338837"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "from creme import model_selection\n",
    "\n",
    "model_selection.progressive_val_score(\n",
    "    X_y=X_y,\n",
    "    model=make_model(),\n",
    "    metric=metrics.MAE(),\n",
    "    on='moment',\n",
    "    delay=dt.timedelta(minutes=30),\n",
    "    print_every=30_000\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[30,000] MAE: 2.230049\n",
      "[60,000] MAE: 2.290409\n",
      "[90,000] MAE: 2.334638\n",
      "[120,000] MAE: 2.315149\n",
      "[150,000] MAE: 2.319982\n",
      "[180,000] MAE: 2.335385\n",
      "CPU times: user 24.8 s, sys: 273 ms, total: 25.1 s\n",
      "Wall time: 25.1 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "MAE: 2.338837"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "from creme import stream\n",
    "\n",
    "cache = stream.Cache()\n",
    "\n",
    "model_selection.progressive_val_score(\n",
    "    X_y=cache(X_y, key='bikes'),\n",
    "    model=make_model(),\n",
    "    metric=metrics.MAE(),\n",
    "    on='moment',\n",
    "    delay=dt.timedelta(minutes=30),\n",
    "    print_every=30_000\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[30,000] MAE: 2.230049\n",
      "[60,000] MAE: 2.290409\n",
      "[90,000] MAE: 2.334638\n",
      "[120,000] MAE: 2.315149\n",
      "[150,000] MAE: 2.319982\n",
      "[180,000] MAE: 2.335385\n",
      "CPU times: user 18 s, sys: 86.4 ms, total: 18.1 s\n",
      "Wall time: 18.1 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "MAE: 2.338837"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "model_selection.progressive_val_score(\n",
    "    X_y=cache(X_y, key='bikes'),\n",
    "    model=make_model(),\n",
    "    metric=metrics.MAE(),\n",
    "    on='moment',\n",
    "    delay=dt.timedelta(minutes=30),\n",
    "    print_every=30_000\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}