create_features-fastai.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import datetime\n",
    "import warnings\n",
    "import numpy as np\n",
    "import sklearn\n",
    "from sklearn_pandas import DataFrameMapper\n",
    "from pandas.api.types import is_string_dtype, is_numeric_dtype, is_categorical_dtype\n",
    "from sklearn.preprocessing import LabelEncoder, Imputer, StandardScaler\n",
    "PATH='dataset/rossmann/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "def numericalize(df, col, name, max_n_cat):\n",
    "    \"\"\" Changes the column col from a categorical type to it's integer codes.\n",
    "    Parameters:\n",
    "    -----------\n",
    "    df: A pandas dataframe. df[name] will be filled with the integer codes from\n",
    "        col.\n",
    "    col: The column you wish to change into the categories.\n",
    "    name: The column name you wish to insert into df. This column will hold the\n",
    "        integer codes.\n",
    "    max_n_cat: If col has more categories than max_n_cat it will not change the\n",
    "        it to its integer codes. If max_n_cat is None, then col will always be\n",
    "        converted.\n",
    "    Examples:\n",
    "    ---------\n",
    "    >>> df = pd.DataFrame({'col1' : [1, 2, 3], 'col2' : ['a', 'b', 'a']})\n",
    "    >>> df\n",
    "       col1 col2\n",
    "    0     1    a\n",
    "    1     2    b\n",
    "    2     3    a\n",
    "    note the type of col2 is string\n",
    "    >>> train_cats(df)\n",
    "    >>> df\n",
    "       col1 col2\n",
    "    0     1    a\n",
    "    1     2    b\n",
    "    2     3    a\n",
    "    now the type of col2 is category { a : 1, b : 2}\n",
    "    >>> numericalize(df, df['col2'], 'col3', None)\n",
    "       col1 col2 col3\n",
    "    0     1    a    1\n",
    "    1     2    b    2\n",
    "    2     3    a    1\n",
    "    \"\"\"\n",
    "    if not is_numeric_dtype(col) and ( max_n_cat is None or len(col.cat.categories)>max_n_cat):\n",
    "        df[name] = pd.Categorical(col).codes+1\n",
    "        \n",
    "def scale_vars(df, mapper):\n",
    "    warnings.filterwarnings('ignore', category=sklearn.exceptions.DataConversionWarning)\n",
    "    if mapper is None:\n",
    "        map_f = [([n],StandardScaler()) for n in df.columns if is_numeric_dtype(df[n])]\n",
    "        mapper = DataFrameMapper(map_f).fit(df)\n",
    "    df[mapper.transformed_names_] = mapper.transform(df)\n",
    "    return mapper\n",
    "\n",
    "def fix_missing(df, col, name, na_dict):\n",
    "    \"\"\" Fill missing data in a column of df with the median, and add a {name}_na column\n",
    "    which specifies if the data was missing.\n",
    "    Parameters:\n",
    "    -----------\n",
    "    df: The data frame that will be changed.\n",
    "    col: The column of data to fix by filling in missing data.\n",
    "    name: The name of the new filled column in df.\n",
    "    na_dict: A dictionary of values to create na's of and the value to insert. If\n",
    "        name is not a key of na_dict the median will fill any missing data. Also\n",
    "        if name is not a key of na_dict and there is no missing data in col, then\n",
    "        no {name}_na column is not created.\n",
    "    Examples:\n",
    "    ---------\n",
    "    >>> df = pd.DataFrame({'col1' : [1, np.NaN, 3], 'col2' : [5, 2, 2]})\n",
    "    >>> df\n",
    "       col1 col2\n",
    "    0     1    5\n",
    "    1   nan    2\n",
    "    2     3    2\n",
    "    >>> fix_missing(df, df['col1'], 'col1', {})\n",
    "    >>> df\n",
    "       col1 col2 col1_na\n",
    "    0     1    5   False\n",
    "    1     2    2    True\n",
    "    2     3    2   False\n",
    "    >>> df = pd.DataFrame({'col1' : [1, np.NaN, 3], 'col2' : [5, 2, 2]})\n",
    "    >>> df\n",
    "       col1 col2\n",
    "    0     1    5\n",
    "    1   nan    2\n",
    "    2     3    2\n",
    "    >>> fix_missing(df, df['col2'], 'col2', {})\n",
    "    >>> df\n",
    "       col1 col2\n",
    "    0     1    5\n",
    "    1   nan    2\n",
    "    2     3    2\n",
    "    >>> df = pd.DataFrame({'col1' : [1, np.NaN, 3], 'col2' : [5, 2, 2]})\n",
    "    >>> df\n",
    "       col1 col2\n",
    "    0     1    5\n",
    "    1   nan    2\n",
    "    2     3    2\n",
    "    >>> fix_missing(df, df['col1'], 'col1', {'col1' : 500})\n",
    "    >>> df\n",
    "       col1 col2 col1_na\n",
    "    0     1    5   False\n",
    "    1   500    2    True\n",
    "    2     3    2   False\n",
    "    \"\"\"\n",
    "    if is_numeric_dtype(col):\n",
    "        if pd.isnull(col).sum() or (name in na_dict):\n",
    "            df[name+'_na'] = pd.isnull(col)\n",
    "            filler = na_dict[name] if name in na_dict else col.median()\n",
    "            df[name] = col.fillna(filler)\n",
    "            na_dict[name] = filler\n",
    "    return na_dict\n",
    "\n",
    "def get_cv_idxs(n, cv_idx=0, val_pct=0.2, seed=42):\n",
    "    \"\"\" Get a list of index values for Validation set from a dataset\n",
    "    \n",
    "    Arguments:\n",
    "        n : int, Total number of elements in the data set.\n",
    "        cv_idx : int, starting index [idx_start = cv_idx*int(val_pct*n)] \n",
    "        val_pct : (int, float), validation set percentage \n",
    "        seed : seed value for RandomState\n",
    "        \n",
    "    Returns:\n",
    "        list of indexes \n",
    "    \"\"\"\n",
    "    np.random.seed(seed)\n",
    "    n_val = int(val_pct*n)\n",
    "    idx_start = cv_idx*n_val\n",
    "    idxs = np.random.permutation(n)\n",
    "    return idxs[idx_start:idx_start+n_val]\n",
    "\n",
    "def apply_cats(df, trn):\n",
    "    \"\"\"Changes any columns of strings in df into categorical variables using trn as\n",
    "    a template for the category codes.\n",
    "    Parameters:\n",
    "    -----------\n",
    "    df: A pandas dataframe. Any columns of strings will be changed to\n",
    "        categorical values. The category codes are determined by trn.\n",
    "    trn: A pandas dataframe. When creating a category for df, it looks up the\n",
    "        what the category's code were in trn and makes those the category codes\n",
    "        for df.\n",
    "    Examples:\n",
    "    ---------\n",
    "    >>> df = pd.DataFrame({'col1' : [1, 2, 3], 'col2' : ['a', 'b', 'a']})\n",
    "    >>> df\n",
    "       col1 col2\n",
    "    0     1    a\n",
    "    1     2    b\n",
    "    2     3    a\n",
    "    note the type of col2 is string\n",
    "    >>> train_cats(df)\n",
    "    >>> df\n",
    "       col1 col2\n",
    "    0     1    a\n",
    "    1     2    b\n",
    "    2     3    a\n",
    "    now the type of col2 is category {a : 1, b : 2}\n",
    "    >>> df2 = pd.DataFrame({'col1' : [1, 2, 3], 'col2' : ['b', 'a', 'a']})\n",
    "    >>> apply_cats(df2, df)\n",
    "           col1 col2\n",
    "        0     1    b\n",
    "        1     2    a\n",
    "        2     3    a\n",
    "    now the type of col is category {a : 1, b : 2}\n",
    "    \"\"\"\n",
    "    for n,c in df.items():\n",
    "        if (n in trn.columns) and (trn[n].dtype.name=='category'):\n",
    "            df[n] = c.astype('category').cat.as_ordered()\n",
    "            df[n].cat.set_categories(trn[n].cat.categories, ordered=True, inplace=True)\n",
    "            \n",
    "def proc_df(df, y_fld=None, skip_flds=None, ignore_flds=None, do_scale=False, na_dict=None,\n",
    "            preproc_fn=None, max_n_cat=None, subset=None, mapper=None):\n",
    "    \"\"\" proc_df takes a data frame df and splits off the response variable, and\n",
    "    changes the df into an entirely numeric dataframe. For each column of df \n",
    "    which is not in skip_flds nor in ignore_flds, na values are replaced by the\n",
    "    median value of the column.\n",
    "    Parameters:\n",
    "    -----------\n",
    "    df: The data frame you wish to process.\n",
    "    y_fld: The name of the response variable\n",
    "    skip_flds: A list of fields that dropped from df.\n",
    "    ignore_flds: A list of fields that are ignored during processing.\n",
    "    do_scale: Standardizes each column in df. Takes Boolean Values(True,False)\n",
    "    na_dict: a dictionary of na columns to add. Na columns are also added if there\n",
    "        are any missing values.\n",
    "    preproc_fn: A function that gets applied to df.\n",
    "    max_n_cat: The maximum number of categories to break into dummy values, instead\n",
    "        of integer codes.\n",
    "    subset: Takes a random subset of size subset from df.\n",
    "    mapper: If do_scale is set as True, the mapper variable\n",
    "        calculates the values used for scaling of variables during training time (mean and standard deviation).\n",
    "    Returns:\n",
    "    --------\n",
    "    [x, y, nas, mapper(optional)]:\n",
    "        x: x is the transformed version of df. x will not have the response variable\n",
    "            and is entirely numeric.\n",
    "        y: y is the response variable\n",
    "        nas: returns a dictionary of which nas it created, and the associated median.\n",
    "        mapper: A DataFrameMapper which stores the mean and standard deviation of the corresponding continuous\n",
    "        variables which is then used for scaling of during test-time.\n",
    "    Examples:\n",
    "    ---------\n",
    "    >>> df = pd.DataFrame({'col1' : [1, 2, 3], 'col2' : ['a', 'b', 'a']})\n",
    "    >>> df\n",
    "       col1 col2\n",
    "    0     1    a\n",
    "    1     2    b\n",
    "    2     3    a\n",
    "    note the type of col2 is string\n",
    "    >>> train_cats(df)\n",
    "    >>> df\n",
    "       col1 col2\n",
    "    0     1    a\n",
    "    1     2    b\n",
    "    2     3    a\n",
    "    now the type of col2 is category { a : 1, b : 2}\n",
    "    >>> x, y, nas = proc_df(df, 'col1')\n",
    "    >>> x\n",
    "       col2\n",
    "    0     1\n",
    "    1     2\n",
    "    2     1\n",
    "    >>> data = DataFrame(pet=[\"cat\", \"dog\", \"dog\", \"fish\", \"cat\", \"dog\", \"cat\", \"fish\"],\n",
    "                 children=[4., 6, 3, 3, 2, 3, 5, 4],\n",
    "                 salary=[90, 24, 44, 27, 32, 59, 36, 27])\n",
    "    >>> mapper = DataFrameMapper([(:pet, LabelBinarizer()),\n",
    "                          ([:children], StandardScaler())])\n",
    "    >>>round(fit_transform!(mapper, copy(data)), 2)\n",
    "    8x4 Array{Float64,2}:\n",
    "    1.0  0.0  0.0   0.21\n",
    "    0.0  1.0  0.0   1.88\n",
    "    0.0  1.0  0.0  -0.63\n",
    "    0.0  0.0  1.0  -0.63\n",
    "    1.0  0.0  0.0  -1.46\n",
    "    0.0  1.0  0.0  -0.63\n",
    "    1.0  0.0  0.0   1.04\n",
    "    0.0  0.0  1.0   0.21\n",
    "    \"\"\"\n",
    "    if not ignore_flds: ignore_flds=[]\n",
    "    if not skip_flds: skip_flds=[]\n",
    "    if subset: df = get_sample(df,subset)\n",
    "    else: df = df.copy()\n",
    "    ignored_flds = df.loc[:, ignore_flds]\n",
    "    df.drop(ignore_flds, axis=1, inplace=True)\n",
    "    if preproc_fn: preproc_fn(df)\n",
    "    if y_fld is None: y = None\n",
    "    else:\n",
    "        if not is_numeric_dtype(df[y_fld]): df[y_fld] = pd.Categorical(df[y_fld]).codes\n",
    "        y = df[y_fld].values\n",
    "        skip_flds += [y_fld]\n",
    "    df.drop(skip_flds, axis=1, inplace=True)\n",
    "\n",
    "    if na_dict is None: na_dict = {}\n",
    "    else: na_dict = na_dict.copy()\n",
    "    na_dict_initial = na_dict.copy()\n",
    "    for n,c in df.items(): na_dict = fix_missing(df, c, n, na_dict)\n",
    "    if len(na_dict_initial.keys()) > 0:\n",
    "        df.drop([a + '_na' for a in list(set(na_dict.keys()) - set(na_dict_initial.keys()))], axis=1, inplace=True)\n",
    "    if do_scale: mapper = scale_vars(df, mapper)\n",
    "    for n,c in df.items(): numericalize(df, c, n, max_n_cat)\n",
    "    df = pd.get_dummies(df, dummy_na=True)\n",
    "    df = pd.concat([ignored_flds, df], axis=1)\n",
    "    res = [df, y, na_dict]\n",
    "    if do_scale: res = res + [mapper]\n",
    "    return res"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "joined = pd.read_feather(f'{PATH}joined')\n",
    "joined_test = pd.read_feather(f'{PATH}joined_test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Store</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DayOfWeek</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <td>2015-07-31 00:00:00</td>\n",
       "      <td>2015-07-31 00:00:00</td>\n",
       "      <td>2015-07-31 00:00:00</td>\n",
       "      <td>2015-07-31 00:00:00</td>\n",
       "      <td>2015-07-31 00:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sales</th>\n",
       "      <td>5263</td>\n",
       "      <td>6064</td>\n",
       "      <td>8314</td>\n",
       "      <td>13995</td>\n",
       "      <td>4822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Customers</th>\n",
       "      <td>555</td>\n",
       "      <td>625</td>\n",
       "      <td>821</td>\n",
       "      <td>1498</td>\n",
       "      <td>559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Open</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Promo</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>StateHoliday</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SchoolHoliday</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Year</th>\n",
       "      <td>2015</td>\n",
       "      <td>2015</td>\n",
       "      <td>2015</td>\n",
       "      <td>2015</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Month</th>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Week</th>\n",
       "      <td>31</td>\n",
       "      <td>31</td>\n",
       "      <td>31</td>\n",
       "      <td>31</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Day</th>\n",
       "      <td>31</td>\n",
       "      <td>31</td>\n",
       "      <td>31</td>\n",
       "      <td>31</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Dayofweek</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Dayofyear</th>\n",
       "      <td>212</td>\n",
       "      <td>212</td>\n",
       "      <td>212</td>\n",
       "      <td>212</td>\n",
       "      <td>212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Is_month_end</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Is_month_start</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Is_quarter_end</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Is_quarter_start</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Is_year_end</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Is_year_start</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Elapsed</th>\n",
       "      <td>1438300800</td>\n",
       "      <td>1438300800</td>\n",
       "      <td>1438300800</td>\n",
       "      <td>1438300800</td>\n",
       "      <td>1438300800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>StoreType</th>\n",
       "      <td>c</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>c</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Assortment</th>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>c</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompetitionDistance</th>\n",
       "      <td>1270</td>\n",
       "      <td>570</td>\n",
       "      <td>14130</td>\n",
       "      <td>620</td>\n",
       "      <td>29910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompetitionOpenSinceMonth</th>\n",
       "      <td>9</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompetitionOpenSinceYear</th>\n",
       "      <td>2008</td>\n",
       "      <td>2007</td>\n",
       "      <td>2006</td>\n",
       "      <td>2009</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Promo2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Promo2SinceWeek</th>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Promo2SinceYear</th>\n",
       "      <td>1900</td>\n",
       "      <td>2010</td>\n",
       "      <td>2011</td>\n",
       "      <td>1900</td>\n",
       "      <td>1900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PromoInterval</th>\n",
       "      <td>None</td>\n",
       "      <td>Jan,Apr,Jul,Oct</td>\n",
       "      <td>Jan,Apr,Jul,Oct</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>State</th>\n",
       "      <td>HE</td>\n",
       "      <td>TH</td>\n",
       "      <td>NW</td>\n",
       "      <td>BE</td>\n",
       "      <td>SN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>file</th>\n",
       "      <td>Rossmann_DE_HE</td>\n",
       "      <td>Rossmann_DE_TH</td>\n",
       "      <td>Rossmann_DE_NW</td>\n",
       "      <td>Rossmann_DE_BE</td>\n",
       "      <td>Rossmann_DE_SN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>week</th>\n",
       "      <td>2015-08-02 - 2015-08-08</td>\n",
       "      <td>2015-08-02 - 2015-08-08</td>\n",
       "      <td>2015-08-02 - 2015-08-08</td>\n",
       "      <td>2015-08-02 - 2015-08-08</td>\n",
       "      <td>2015-08-02 - 2015-08-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>trend</th>\n",
       "      <td>85</td>\n",
       "      <td>80</td>\n",
       "      <td>86</td>\n",
       "      <td>74</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>file_DE</th>\n",
       "      <td>Rossmann_DE</td>\n",
       "      <td>Rossmann_DE</td>\n",
       "      <td>Rossmann_DE</td>\n",
       "      <td>Rossmann_DE</td>\n",
       "      <td>Rossmann_DE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>week_DE</th>\n",
       "      <td>2015-08-02 - 2015-08-08</td>\n",
       "      <td>2015-08-02 - 2015-08-08</td>\n",
       "      <td>2015-08-02 - 2015-08-08</td>\n",
       "      <td>2015-08-02 - 2015-08-08</td>\n",
       "      <td>2015-08-02 - 2015-08-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>trend_DE</th>\n",
       "      <td>83</td>\n",
       "      <td>83</td>\n",
       "      <td>83</td>\n",
       "      <td>83</td>\n",
       "      <td>83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date_DE</th>\n",
       "      <td>2015-08-02 00:00:00</td>\n",
       "      <td>2015-08-02 00:00:00</td>\n",
       "      <td>2015-08-02 00:00:00</td>\n",
       "      <td>2015-08-02 00:00:00</td>\n",
       "      <td>2015-08-02 00:00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 0                        1  \\\n",
       "index                                            0                        1   \n",
       "Store                                            1                        2   \n",
       "DayOfWeek                                        5                        5   \n",
       "Date                           2015-07-31 00:00:00      2015-07-31 00:00:00   \n",
       "Sales                                         5263                     6064   \n",
       "Customers                                      555                      625   \n",
       "Open                                             1                        1   \n",
       "Promo                                            1                        1   \n",
       "StateHoliday                                 False                    False   \n",
       "SchoolHoliday                                    1                        1   \n",
       "Year                                          2015                     2015   \n",
       "Month                                            7                        7   \n",
       "Week                                            31                       31   \n",
       "Day                                             31                       31   \n",
       "Dayofweek                                        4                        4   \n",
       "Dayofyear                                      212                      212   \n",
       "Is_month_end                                  True                     True   \n",
       "Is_month_start                               False                    False   \n",
       "Is_quarter_end                               False                    False   \n",
       "Is_quarter_start                             False                    False   \n",
       "Is_year_end                                  False                    False   \n",
       "Is_year_start                                False                    False   \n",
       "Elapsed                                 1438300800               1438300800   \n",
       "StoreType                                        c                        a   \n",
       "Assortment                                       a                        a   \n",
       "CompetitionDistance                           1270                      570   \n",
       "CompetitionOpenSinceMonth                        9                       11   \n",
       "CompetitionOpenSinceYear                      2008                     2007   \n",
       "Promo2                                           0                        1   \n",
       "Promo2SinceWeek                                  1                       13   \n",
       "Promo2SinceYear                               1900                     2010   \n",
       "PromoInterval                                 None          Jan,Apr,Jul,Oct   \n",
       "State                                           HE                       TH   \n",
       "file                                Rossmann_DE_HE           Rossmann_DE_TH   \n",
       "week                       2015-08-02 - 2015-08-08  2015-08-02 - 2015-08-08   \n",
       "trend                                           85                       80   \n",
       "file_DE                                Rossmann_DE              Rossmann_DE   \n",
       "week_DE                    2015-08-02 - 2015-08-08  2015-08-02 - 2015-08-08   \n",
       "trend_DE                                        83                       83   \n",
       "Date_DE                        2015-08-02 00:00:00      2015-08-02 00:00:00   \n",
       "\n",
       "                                                 2                        3  \\\n",
       "index                                            2                        3   \n",
       "Store                                            3                        4   \n",
       "DayOfWeek                                        5                        5   \n",
       "Date                           2015-07-31 00:00:00      2015-07-31 00:00:00   \n",
       "Sales                                         8314                    13995   \n",
       "Customers                                      821                     1498   \n",
       "Open                                             1                        1   \n",
       "Promo                                            1                        1   \n",
       "StateHoliday                                 False                    False   \n",
       "SchoolHoliday                                    1                        1   \n",
       "Year                                          2015                     2015   \n",
       "Month                                            7                        7   \n",
       "Week                                            31                       31   \n",
       "Day                                             31                       31   \n",
       "Dayofweek                                        4                        4   \n",
       "Dayofyear                                      212                      212   \n",
       "Is_month_end                                  True                     True   \n",
       "Is_month_start                               False                    False   \n",
       "Is_quarter_end                               False                    False   \n",
       "Is_quarter_start                             False                    False   \n",
       "Is_year_end                                  False                    False   \n",
       "Is_year_start                                False                    False   \n",
       "Elapsed                                 1438300800               1438300800   \n",
       "StoreType                                        a                        c   \n",
       "Assortment                                       a                        c   \n",
       "CompetitionDistance                          14130                      620   \n",
       "CompetitionOpenSinceMonth                       12                        9   \n",
       "CompetitionOpenSinceYear                      2006                     2009   \n",
       "Promo2                                           1                        0   \n",
       "Promo2SinceWeek                                 14                        1   \n",
       "Promo2SinceYear                               2011                     1900   \n",
       "PromoInterval                      Jan,Apr,Jul,Oct                     None   \n",
       "State                                           NW                       BE   \n",
       "file                                Rossmann_DE_NW           Rossmann_DE_BE   \n",
       "week                       2015-08-02 - 2015-08-08  2015-08-02 - 2015-08-08   \n",
       "trend                                           86                       74   \n",
       "file_DE                                Rossmann_DE              Rossmann_DE   \n",
       "week_DE                    2015-08-02 - 2015-08-08  2015-08-02 - 2015-08-08   \n",
       "trend_DE                                        83                       83   \n",
       "Date_DE                        2015-08-02 00:00:00      2015-08-02 00:00:00   \n",
       "\n",
       "                                                 4  \n",
       "index                                            4  \n",
       "Store                                            5  \n",
       "DayOfWeek                                        5  \n",
       "Date                           2015-07-31 00:00:00  \n",
       "Sales                                         4822  \n",
       "Customers                                      559  \n",
       "Open                                             1  \n",
       "Promo                                            1  \n",
       "StateHoliday                                 False  \n",
       "SchoolHoliday                                    1  \n",
       "Year                                          2015  \n",
       "Month                                            7  \n",
       "Week                                            31  \n",
       "Day                                             31  \n",
       "Dayofweek                                        4  \n",
       "Dayofyear                                      212  \n",
       "Is_month_end                                  True  \n",
       "Is_month_start                               False  \n",
       "Is_quarter_end                               False  \n",
       "Is_quarter_start                             False  \n",
       "Is_year_end                                  False  \n",
       "Is_year_start                                False  \n",
       "Elapsed                                 1438300800  \n",
       "StoreType                                        a  \n",
       "Assortment                                       a  \n",
       "CompetitionDistance                          29910  \n",
       "CompetitionOpenSinceMonth                        4  \n",
       "CompetitionOpenSinceYear                      2015  \n",
       "Promo2                                           0  \n",
       "Promo2SinceWeek                                  1  \n",
       "Promo2SinceYear                               1900  \n",
       "PromoInterval                                 None  \n",
       "State                                           SN  \n",
       "file                                Rossmann_DE_SN  \n",
       "week                       2015-08-02 - 2015-08-08  \n",
       "trend                                           82  \n",
       "file_DE                                Rossmann_DE  \n",
       "week_DE                    2015-08-02 - 2015-08-08  \n",
       "trend_DE                                        83  \n",
       "Date_DE                        2015-08-02 00:00:00  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joined.head().T.head(40)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that we've engineered all our features, we need to convert to input compatible with a neural network.\n",
    "\n",
    "This includes converting categorical variables into contiguous integers or one-hot encodings, normalizing continuous features to standard normal, etc..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "844338"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_vars = ['Store', 'DayOfWeek', 'Year', 'Month', 'Day', 'StateHoliday', 'CompetitionMonthsOpen',\n",
    "    'Promo2Weeks', 'StoreType', 'Assortment', 'PromoInterval', 'CompetitionOpenSinceYear', 'Promo2SinceYear',\n",
    "    'State', 'Week', 'Events', 'Promo_fw', 'Promo_bw', 'StateHoliday_fw', 'StateHoliday_bw',\n",
    "    'SchoolHoliday_fw', 'SchoolHoliday_bw']\n",
    "\n",
    "contin_vars = ['CompetitionDistance', 'Max_TemperatureC', 'Mean_TemperatureC', 'Min_TemperatureC',\n",
    "   'Max_Humidity', 'Mean_Humidity', 'Min_Humidity', 'Max_Wind_SpeedKm_h', \n",
    "   'Mean_Wind_SpeedKm_h', 'CloudCover', 'trend', 'trend_DE',\n",
    "   'AfterStateHoliday', 'BeforeStateHoliday', 'Promo', 'SchoolHoliday']\n",
    "\n",
    "n = len(joined); n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "dep = 'Sales'\n",
    "joined = joined[cat_vars+contin_vars+[dep, 'Date']].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_test[dep] = 0\n",
    "joined_test = joined_test[cat_vars+contin_vars+[dep, 'Date', 'Id']].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "for v in cat_vars: joined[v] = joined[v].astype('category').cat.as_ordered()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "apply_cats(joined_test, joined)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "for v in contin_vars:\n",
    "    joined[v] = joined[v].fillna(0).astype('float32')\n",
    "    joined_test[v] = joined_test[v].fillna(0).astype('float32')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We're going to run on a sample."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "150000"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idxs = get_cv_idxs(n, val_pct=150000/n)\n",
    "joined_samp = joined.iloc[idxs].set_index(\"Date\")\n",
    "samp_size = len(joined_samp); samp_size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To run on the full dataset, use this instead:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "samp_size = n\n",
    "joined_samp = joined.set_index(\"Date\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can now process our data..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Store</th>\n",
       "      <th>DayOfWeek</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>StateHoliday</th>\n",
       "      <th>CompetitionMonthsOpen</th>\n",
       "      <th>Promo2Weeks</th>\n",
       "      <th>StoreType</th>\n",
       "      <th>Assortment</th>\n",
       "      <th>...</th>\n",
       "      <th>Max_Wind_SpeedKm_h</th>\n",
       "      <th>Mean_Wind_SpeedKm_h</th>\n",
       "      <th>CloudCover</th>\n",
       "      <th>trend</th>\n",
       "      <th>trend_DE</th>\n",
       "      <th>AfterStateHoliday</th>\n",
       "      <th>BeforeStateHoliday</th>\n",
       "      <th>Promo</th>\n",
       "      <th>SchoolHoliday</th>\n",
       "      <th>Sales</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2015-07-31</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2015</td>\n",
       "      <td>7</td>\n",
       "      <td>31</td>\n",
       "      <td>False</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "      <td>c</td>\n",
       "      <td>a</td>\n",
       "      <td>...</td>\n",
       "      <td>24.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>85.0</td>\n",
       "      <td>83.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-31</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>2015</td>\n",
       "      <td>7</td>\n",
       "      <td>31</td>\n",
       "      <td>False</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>...</td>\n",
       "      <td>14.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>83.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6064</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 39 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           Store DayOfWeek  Year Month Day StateHoliday CompetitionMonthsOpen  \\\n",
       "Date                                                                            \n",
       "2015-07-31     1         5  2015     7  31        False                    24   \n",
       "2015-07-31     2         5  2015     7  31        False                    24   \n",
       "\n",
       "           Promo2Weeks StoreType Assortment  ... Max_Wind_SpeedKm_h  \\\n",
       "Date                                         ...                      \n",
       "2015-07-31           0         c          a  ...               24.0   \n",
       "2015-07-31          25         a          a  ...               14.0   \n",
       "\n",
       "           Mean_Wind_SpeedKm_h CloudCover trend trend_DE AfterStateHoliday  \\\n",
       "Date                                                                         \n",
       "2015-07-31                11.0        1.0  85.0     83.0              57.0   \n",
       "2015-07-31                11.0        4.0  80.0     83.0              67.0   \n",
       "\n",
       "           BeforeStateHoliday Promo SchoolHoliday Sales  \n",
       "Date                                                     \n",
       "2015-07-31                0.0   1.0           1.0  5263  \n",
       "2015-07-31                0.0   1.0           1.0  6064  \n",
       "\n",
       "[2 rows x 39 columns]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joined_samp.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "df, y, nas, mapper = proc_df(joined_samp, 'Sales', do_scale=True)\n",
    "yl = np.log(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_test = joined_test.set_index(\"Date\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test, _, nas, mapper = proc_df(joined_test, 'Sales', do_scale=True, skip_flds=['Id'],\n",
    "                                  mapper=mapper, na_dict=nas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Store</th>\n",
       "      <th>DayOfWeek</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>StateHoliday</th>\n",
       "      <th>CompetitionMonthsOpen</th>\n",
       "      <th>Promo2Weeks</th>\n",
       "      <th>StoreType</th>\n",
       "      <th>Assortment</th>\n",
       "      <th>...</th>\n",
       "      <th>Min_Humidity</th>\n",
       "      <th>Max_Wind_SpeedKm_h</th>\n",
       "      <th>Mean_Wind_SpeedKm_h</th>\n",
       "      <th>CloudCover</th>\n",
       "      <th>trend</th>\n",
       "      <th>trend_DE</th>\n",
       "      <th>AfterStateHoliday</th>\n",
       "      <th>BeforeStateHoliday</th>\n",
       "      <th>Promo</th>\n",
       "      <th>SchoolHoliday</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2015-07-31</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>31</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.620066</td>\n",
       "      <td>0.149027</td>\n",
       "      <td>-0.142774</td>\n",
       "      <td>-1.844823</td>\n",
       "      <td>1.732493</td>\n",
       "      <td>1.724336</td>\n",
       "      <td>0.604460</td>\n",
       "      <td>1.13112</td>\n",
       "      <td>1.113717</td>\n",
       "      <td>2.04105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-31</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>31</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.264031</td>\n",
       "      <td>-0.960613</td>\n",
       "      <td>-0.142774</td>\n",
       "      <td>-0.488722</td>\n",
       "      <td>1.294579</td>\n",
       "      <td>1.724336</td>\n",
       "      <td>0.926957</td>\n",
       "      <td>1.13112</td>\n",
       "      <td>1.113717</td>\n",
       "      <td>2.04105</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 38 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Store  DayOfWeek  Year  Month  Day  StateHoliday  \\\n",
       "Date                                                           \n",
       "2015-07-31      1          5     3      7   31             1   \n",
       "2015-07-31      2          5     3      7   31             1   \n",
       "\n",
       "            CompetitionMonthsOpen  Promo2Weeks  StoreType  Assortment  ...  \\\n",
       "Date                                                                   ...   \n",
       "2015-07-31                     25            1          3           1  ...   \n",
       "2015-07-31                     25           26          1           1  ...   \n",
       "\n",
       "            Min_Humidity  Max_Wind_SpeedKm_h  Mean_Wind_SpeedKm_h  CloudCover  \\\n",
       "Date                                                                            \n",
       "2015-07-31     -1.620066            0.149027            -0.142774   -1.844823   \n",
       "2015-07-31     -1.264031           -0.960613            -0.142774   -0.488722   \n",
       "\n",
       "               trend  trend_DE  AfterStateHoliday  BeforeStateHoliday  \\\n",
       "Date                                                                    \n",
       "2015-07-31  1.732493  1.724336           0.604460             1.13112   \n",
       "2015-07-31  1.294579  1.724336           0.926957             1.13112   \n",
       "\n",
       "               Promo  SchoolHoliday  \n",
       "Date                                 \n",
       "2015-07-31  1.113717        2.04105  \n",
       "2015-07-31  1.113717        2.04105  \n",
       "\n",
       "[2 rows x 38 columns]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In time series data, cross-validation is not random. Instead, our holdout data is generally the most recent data, as it would be in real application. This issue is discussed in detail in [this post](http://www.fast.ai/2017/11/13/validation-sets/) on our web site.\n",
    "\n",
    "One approach is to take the last 25% of rows (sorted by date) as our validation set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ratio = 0.75\n",
    "# train_ratio = 0.9\n",
    "train_size = int(samp_size * train_ratio); train_size\n",
    "val_idx = list(range(train_size, len(df)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "An even better option for picking a validation set is using the exact same length of time period as the test set uses - this is implemented here:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_idx = np.flatnonzero(\n",
    "    (df.index<=datetime.datetime(2014,9,17)) & (df.index>=datetime.datetime(2014,8,1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_idx=[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DL"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We're ready to put together our models.\n",
    "\n",
    "Root-mean-squared percent error is the metric Kaggle used for this competition."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "def inv_y(a): return np.exp(a)\n",
    "\n",
    "def exp_rmspe(y_pred, targ):\n",
    "    targ = inv_y(targ)\n",
    "    pct_var = (targ - inv_y(y_pred))/targ\n",
    "    return math.sqrt((pct_var**2).mean())\n",
    "\n",
    "max_log_y = np.max(yl)\n",
    "y_range = (0, max_log_y*1.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can create a ModelData object directly from out data frame."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'ColumnarModelData' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-89-9df421910b1c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m md = ColumnarModelData.from_data_frame(PATH, val_idx, df, yl.astype(np.float32), cat_flds=cat_vars, bs=128,\n\u001b[0m\u001b[1;32m      2\u001b[0m                                        test_df=df_test)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'ColumnarModelData' is not defined"
     ]
    }
   ],
   "source": [
    "md = ColumnarModelData.from_data_frame(PATH, val_idx, df, yl.astype(np.float32), cat_flds=cat_vars, bs=128,\n",
    "                                       test_df=df_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some categorical variables have a lot more levels than others. Store, in particular, has over a thousand!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_sz = [(c, len(joined_samp[c].cat.categories)+1) for c in cat_vars]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Store', 1116, 50),\n",
       " ('DayOfWeek', 8, 4),\n",
       " ('Year', 4, 2),\n",
       " ('Month', 13, 7),\n",
       " ('Day', 32, 16),\n",
       " ('StateHoliday', 3, 2),\n",
       " ('CompetitionMonthsOpen', 26, 13),\n",
       " ('Promo2Weeks', 27, 14),\n",
       " ('StoreType', 5, 3),\n",
       " ('Assortment', 4, 2),\n",
       " ('PromoInterval', 4, 2),\n",
       " ('CompetitionOpenSinceYear', 24, 12),\n",
       " ('Promo2SinceYear', 9, 5),\n",
       " ('State', 13, 7),\n",
       " ('Week', 53, 27),\n",
       " ('Events', 22, 11),\n",
       " ('Promo_fw', 7, 4),\n",
       " ('Promo_bw', 7, 4),\n",
       " ('StateHoliday_fw', 4, 2),\n",
       " ('StateHoliday_bw', 4, 2),\n",
       " ('SchoolHoliday_fw', 9, 5),\n",
       " ('SchoolHoliday_bw', 9, 5)]"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_sz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use the *cardinality* of each variable (that is, its number of unique values) to decide how large to make its *embeddings*. Each level will be associated with a vector with length defined as below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = md.get_learner(emb_szs, len(df.columns)-len(cat_vars),\n",
    "                   0.04, 1, [1000,500], [0.001,0.01], y_range=y_range)\n",
    "m.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0dcee5fad2f84107975671f29d455619",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 70%|██████▉   | 611/879 [00:06<00:02, 110.07it/s, loss=0.234] \n"
     ]
    }
   ],
   "source": [
    "lr = 1e-3\n",
    "m.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEOCAYAAABIESrBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8XXWd//HX52ZPszZJm+4tpXSDtrShpUXZN9GhjCsV\nEURFHZcZZ36O+JuZnw7ODI6z/EZxwYoI6k8QQbAisslugTYtLbSULpTuS5KmS5o0283n98c9rZc0\ny81tbk6W9/PxuI+ce7b76SHcd77ne873mLsjIiKSjEjYBYiIyMClEBERkaQpREREJGkKERERSZpC\nREREkqYQERGRpClEREQkaQoRERFJmkJERESSphAREZGkpYddQG8qLS31iRMnhl2GiMiAsWrVqhp3\nL0t2+0EVIhMnTqSysjLsMkREBgwz234q2+t0loiIJC1lLREzuwt4H1Dl7md2sPwrwHVxdUwHyty9\n1sy2AXVAFGh194pU1SkiIslLZUvkbuDKzha6+3+4+xx3nwN8DXjO3WvjVrkoWK4AERHpp1IWIu7+\nPFDb7YoxS4B7U1WLiIikRuh9ImaWS6zF8mDcbAeeMLNVZnZzOJWJiEh3+sPVWX8B/Kndqazz3H2P\nmY0AnjSzN4OWzUmCkLkZYPz48amvVkRETgi9JQJcS7tTWe6+J/hZBTwEzO9sY3df6u4V7l5RVtbz\nS52jbc4Lm6t5c9+RHm8rIjLUhRoiZlYIXAD8Nm7eMDPLPz4NXA6sS1kNwGd+vor7VuxM1UeIiAxa\nqbzE917gQqDUzHYBXwcyANz9jmC1vwSecPf6uE1HAg+Z2fH6funuj6WqzkjEmDIyn4376lL1ESIi\ng1bKQsTdlySwzt3ELgWOn7cVmJ2aqjo2bWQ+T27Yj7sThJeIiCSgP/SJhG5qeT619c3UHG0OuxQR\nkQFFIQJMK88H0CktEZEeUogQa4kAukJLRKSHFCJASV4WpXlZaomIiPSQQiQwrTyfjfsVIiIiPaEQ\nCUwtz2fT/jqibR52KSIiA4ZCJDC1PJ/GljZ21DaEXYqIyIChEAn8+Qotda6LiCRKIRKYMiIfM3hT\nnesiIglTiARyMtOYWDJMV2iJiPSAQiTOVI2hJSLSIwqROFPL89l2oJ7GlmjYpYiIDAgKkTjTyvNp\nc9i8/2jYpYiIDAgKkTga/kREpGcUInEmlAwjKz2ifhERkQQpROKkRYwpI/M0/ImISIIUIu1MLy9g\nw94juGv4ExGR7ihE2pk5uoCao81U1TWFXYqISL+nEGln5phCANbvORxyJSIi/Z9CpJ3jY2it360r\ntEREuqMQaSc/O4OJJbms36MQERHpTspCxMzuMrMqM1vXyfILzeywma0JXv8nbtmVZrbRzLaY2S2p\nqrEzM0cXsn6vTmeJiHQnlS2Ru4Eru1nnBXefE7xuBTCzNOD7wHuAGcASM5uRwjpPMmN0ATtrj3H4\nWEtffqyIyICTshBx9+eB2iQ2nQ9scfet7t4M3Acs7tXiujFzdAEAG/bqlJaISFfC7hNZaGZrzewP\nZjYzmDcG2Bm3zq5gXp+ZEYSI+kVERLqWHuJnrwYmuPtRM7sKeBiYAlgH63Z655+Z3QzcDDB+/Phe\nKWxEfjZl+Vm6zFdEpBuhtUTc/Yi7Hw2mHwUyzKyUWMtjXNyqY4E9XexnqbtXuHtFWVlZr9U3c3QB\nb6glIiLSpdBCxMzKzcyC6flBLQeAlcAUM5tkZpnAtcCyvq5v5ugCNlcd1bNFRES6kLLTWWZ2L3Ah\nUGpmu4CvAxkA7n4H8EHgc2bWChwDrvXYgFWtZvYF4HEgDbjL3denqs7OzBxdSLTN2bS/jllji/r6\n40VEBoSUhYi7L+lm+feA73Wy7FHg0VTUlagZo2Kd6+t2H1GIiIh0Iuyrs/qtCSW5FGSn8/ruQ2GX\nIiLSbylEOmFmzB5XxJqdukJLRKQzCpEuzB5bxKb9dRxrVue6iEhHFCJdmD2uiGib634REZFOKES6\nMHts7Nkia3aqX0REpCMKkS6MKMhmVGE2a3epJSIi0hGFSDdmjy3itV1qiYiIdEQh0o3Z44rYfqCB\ng/XNYZciItLvKES6caJfRK0REZGTKES6MWtcERGDV3coRERE2lOIdCMvK52p5QWs3n4w7FJERPod\nhUgC5k0o4tUdB4m2dfpYExGRIUkhkoB5E4qpb46ycV9d2KWIiPQrCpEEVEwYDsCqHTqlJSISTyGS\ngLHFOZTlZ6lfRESkHYVIAsyMeeOLWaUQERF5B4VIguZNKGZHbQNVdY1hlyIi0m8oRBI0d0IxAKu3\n634REZHjFCIJOnNMAZlpEVZtrw27FBGRfkMhkqCs9DTOGluofhERkTgKkR6omFDM67sP09iiJx2K\niEAKQ8TM7jKzKjNb18ny68zsteC13Mxmxy3bZmavm9kaM6tMVY09NX/ScFqizmrdLyIiAqS2JXI3\ncGUXy98GLnD3WcA3gaXtll/k7nPcvSJF9fVYxcThmMErW9UvIiICkJ6qHbv782Y2sYvly+PevgyM\nTVUtvaUwJ4MZowp45e0DYZciItIv9Jc+kU8Cf4h778ATZrbKzG7uakMzu9nMKs2ssrq6OqVFAiyY\nVMKrOw7R1Kp+ERGR0EPEzC4iFiJfjZt9nrvPBd4DfN7Mzu9se3df6u4V7l5RVlaW4mphwWnDaWpt\nY+1OPXddRCTUEDGzWcCdwGJ3P3GOyN33BD+rgIeA+eFUeLIFk473i+iUlohIaCFiZuOB3wDXu/um\nuPnDzCz/+DRwOdDhFV5hKMrNZOrIfF55W53rIiIp61g3s3uBC4FSM9sFfB3IAHD3O4D/A5QAPzAz\ngNbgSqyRwEPBvHTgl+7+WKrqTMa5p5Xwq5U7aYm2kZEW+hlBEZHQpPLqrCXdLP8U8KkO5m8FZp+8\nRf+xYNJw7l6+jdd2HWZeMKaWiMhQpD+jkzB/UuwhVbrUV0SGOoVIEkryspgyIk83HYrIkKcQSdKC\n04ZTua2W1mhb2KWIiIRGIZKkc08rob45yuu7db+IiAxdCpEkLZpcihm8sLkm7FJEREKjEEnS8GGZ\nnDWmkBc2p36oFRGR/kohcgrePaWU1TsOcaSxJexSRERCoRA5BedPKSPa5izfokt9RWRoUoicgrPH\nFzMsM02ntERkyFKInILM9AgLJ5fy/OZq3D3sckRE+pxC5BRdcEYpO2uP8XZNfdiliIj0OYXIKbpw\n6ggA/rihKuRKRET6nkLkFI0bnsu08nye3LA/7FJERPqcQqQXXDZjJJXbajlY3xx2KSIifUoh0gsu\nmzGSNodnNuqUlogMLQqRXnDm6EJGFmTxlE5picgQoxDpBZGIccn0kTy3sZqm1mjY5YiI9BmFSC+5\nbPpI6pujvPSW7l4XkaFDIdJLFk4uIScjTae0RGRIUYj0kuyMNM4/o5Sn3qjS3esiMmSkNETM7C4z\nqzKzdZ0sNzP7rpltMbPXzGxu3LIbzGxz8LohlXX2lkunj2TfkUbW7zkSdikiIn0i1S2Ru4Eru1j+\nHmBK8LoZ+CGAmQ0Hvg4sAOYDXzez4pRW2gsunjaCtIjx6Ot7wy5FRKRPpDRE3P15oLaLVRYDP/OY\nl4EiMxsFXAE86e617n4QeJKuw6hfKMnL4rzTS/ntmj20temUlogMfmH3iYwBdsa93xXM62x+v3fN\nnNHsPnSM1TsOhl2KiEjKhR0i1sE872L+yTswu9nMKs2ssro6/Od6XD6znOyMCA+v2R12KSIiKRd2\niOwCxsW9Hwvs6WL+Sdx9qbtXuHtFWVlZygpNVF5WOpdOH8nvX9tLS7Qt7HJERFIq7BBZBnw8uErr\nXOCwu+8FHgcuN7PioEP98mDegHDNnDEcbGjREw9FZNBLT+XOzexe4EKg1Mx2EbviKgPA3e8AHgWu\nArYADcAngmW1ZvZNYGWwq1vdvasO+n7l/DPKKMrN4OFX93DxtJFhlyMikjIpDRF3X9LNcgc+38my\nu4C7UlFXqmWmR7jqrFE8tHo39U2tDMtK6WEWEQlNQqezzOyvzawgOO30EzNbbWaXp7q4geyaOWM4\n1hLlyTc0DIqIDF6J9onc5O5HiPVNlBE77fStlFU1CFRMKGZMUY6u0hKRQS3REDl+ye1VwE/dfS0d\nX4YrgUjEuHrOaF7YXEPN0aawyxERSYlEQ2SVmT1BLEQeN7N8QNevdmPxnNFE25zfv6ZhUERkcEo0\nRD4J3AKc4+4NxK6w+kTKqhokppUXMK08n9/qlJaIDFKJhshCYKO7HzKzjwH/CBxOXVmDx+I5Y1i9\n4xA7DjSEXYqISK9LNER+CDSY2Wzg74HtwM9SVtUg8hezRwGoNSIig1KiIdIa3NOxGPiOu38HyE9d\nWYPH2OJc5k8czsNrduthVSIy6CQaInVm9jXgeuD3ZpZGcOe5dG/x2aN5q7peD6sSkUEn0RD5CNBE\n7H6RfcSGZf+PlFU1yFx15igy0kyntERk0EkoRILg+H9AoZm9D2h0d/WJJKh4WCYXnFHGsrV7aNXI\nviIyiCQ67MmHgRXAh4APA6+Y2QdTWdhg88F549h/pInnNbKviAwiiY4M+A/E7hGpAjCzMuAp4IFU\nFTbYXDJ9BKV5mdy3YqdG9hWRQSPRPpHI8QAJHOjBtgJkpEX4wLyx/PHNKqrqGsMuR0SkVyQaBI+Z\n2eNmdqOZ3Qj8ntizQKQHPlIxjmib8+AqdbCLyOCQaMf6V4ClwCxgNrDU3b+aysIGo9PK8lgwaTj3\nrdxBW5vuGRGRgS/hU1Lu/qC7/627f9ndH0plUYPZ9QsnsP1AA89uqup+ZRGRfq7LEDGzOjM70sGr\nzsx051wSrphZTnlBNj/907awSxEROWVdhoi757t7QQevfHcv6KsiB5OMtAjXL5zAC5tr2FJVF3Y5\nIiKnRFdYheDac8aRmR7hnuXbwy5FROSUKERCUJKXxeLZo3lw9S4OH2sJuxwRkaSlNETM7Eoz22hm\nW8zslg6W/18zWxO8NpnZobhl0bhly1JZZxhuWDSRhuYov67cGXYpIiJJS/SO9R4LRvr9PnAZsAtY\naWbL3P2N4+u4+5fj1v8icHbcLo65+5xU1Re2M8cUMn/icO55aRufOG8SaRE9sl5EBp5UtkTmA1vc\nfau7NwP3EXseSWeWAPemsJ5+56Z3TWRn7TEeeW1P2KWIiCQllSEyBog/V7MrmHcSM5sATAKejpud\nbWaVZvaymV2TujLDc/mMcqaV5/OdpzZrdF8RGZBSGSIdnZ/p7Dbta4EH3D0aN2+8u1cAHwX+x8wm\nd/ghZjcHYVNZXT2wRsiNRIy/uXQKW2vqWbZWrRERGXhSGSK7gHFx78cCnX1TXku7U1nuvif4uRV4\nlnf2l8Svt9TdK9y9oqys7FRr7nOXzyhnxqgCvvtHtUZEZOBJZYisBKaY2SQzyyQWFCddZWVmU4Fi\n4KW4ecVmlhVMlwLnAW+033YwON4a2XaggYde1cCMIjKwpCxE3L0V+ALwOLABuN/d15vZrWZ2ddyq\nS4D73D3+VNd0oNLM1gLPAN+Kv6prsLlsxkjOHFPA7U9voUWtEREZQOyd390DW0VFhVdWVoZdRlL+\nuGE/n7ynktvefxZL5o8PuxwRGSLMbFXQ/5wU3bHeT1w8bQRnjy/iu3/cTGNLtPsNRET6AYVIP2Fm\nfOWKqew93MgvXtaYWiIyMChE+pFFk0t595RSvv/MFuoaNaaWiPR/CpF+5u+vmMbBhhbufOHtsEsR\nEemWQqSfOWtsIVedVc6dL2zlwNGmsMsREemSQqQf+tvLpnKsJcrtT28JuxQRkS4pRPqh00fk8dEF\n4/nZS9t4Y4+eQiwi/ZdCpJ/6yuXTKM7N5J9+u462tsFzL4+IDC4KkX6qMDeDW94zjVXbD/LAql1h\nlyMi0iGFSD/2gbljqZhQzLcee5NDDc1hlyMichKFSD8WiRjfvOZMDh9r4duPbwy7HBGRkyhE+rnp\nowq4cdFE7l2xgzU7D3W/gYhIH1KIDAB/c+kURuZn85Vfr9W4WiLSryhEBoD87Az+40Oz2Fx1lG8/\nptNaItJ/KEQGiHdPKePGRRO5609v86ctNWGXIyICKEQGlK9eOY3Tyobxd/evpbZeV2uJSPgUIgNI\nTmYa3/nI2dQ2NPOFX67WM9lFJHQKkQHmrLGF/Os1Z7L8rQN86w9vhl2OiAxx6WEXID33oYpxrNt9\nmDtffJuzxhayeM6YsEsSkSFKLZEB6h/fN4P5E4fz1QdfY93uw2GXIyJDlEJkgMpIi/D96+ZSnJvJ\nZ36+Sh3tIhKKlIaImV1pZhvNbIuZ3dLB8hvNrNrM1gSvT8Utu8HMNgevG1JZ50BVlp/FHR+bR/XR\nJnW0i0goUhYiZpYGfB94DzADWGJmMzpY9VfuPid43RlsOxz4OrAAmA983cyKU1XrQDZ7XNGJjvZ/\n+u163DVsvIj0nVS2ROYDW9x9q7s3A/cBixPc9grgSXevdfeDwJPAlSmqc8D7UMU4PnfhZO5dsYPv\n6WmIItKHUhkiY4Cdce93BfPa+4CZvWZmD5jZuB5uK4G/v2Iq7587hv96chP3V+7sfgMRkV6QyhCx\nDua1P9fyO2Ciu88CngLu6cG2sRXNbjazSjOrrK6uTrrYgc7M+PcPzOLdU0r52m9e55mNVWGXJCJD\nQCpDZBcwLu79WGBP/ArufsDdm4K3PwbmJbpt3D6WunuFu1eUlZX1SuEDVUZahB9+bB7TyvP5q1+s\nZq2GjheRFEtliKwEppjZJDPLBK4FlsWvYGaj4t5eDWwIph8HLjez4qBD/fJgnnQjLyudn37iHEry\nMrnp7pVsq6kPuyQRGcRSFiLu3gp8gdiX/wbgfndfb2a3mtnVwWpfMrP1ZrYW+BJwY7BtLfBNYkG0\nErg1mCcJGJGfzT03zafNnRt+uoKao03dbyQikgQbTJeEVlRUeGVlZdhl9Burdxzkoz9+mTNG5nPv\np89lWJZGuRGRdzKzVe5ekez2umN9EJs7vpjbl8xl3e7DfPTOV6iuU4tERHqXQmSQu2zGSH74sXls\n3HeEa77/Jzbvrwu7JBEZRBQiQ8AVM8u5/zMLaY628f4fLOfFzXoyooj0DoXIEDFrbBEPf/48Rhfl\ncONPV3Dfih1hlyQig4BCZAgZU5TDA59byKLTS7nlN69z2x820NY2eC6sEJG+pxAZYvKzM7jrhgqu\nWzCeHz23lc//cjWNLdGwyxKRAUohMgSlp0X4l2vO5B/fO53H1u/jI0tfpupIY9hlicgApBAZosyM\nT737NO742Dw27avjvbe/SOU23c8pIj2jEBnirphZzkOfX8SwzDSuXfoy9yzfpmeSiEjCFCLCtPIC\nfvuFd3HBGWV8fdl6/u7+teonEZGEKEQEgMKcDH788Qq+fOkZ/ObV3Xz8Jyv03HYR6ZZCRE6IRIy/\nvnQKty85mzU7D3Hpfz/Hc5uG7jNaRKR7ChE5yV/MHs2yL57HiPwsbvzpCr6xbD2HG1rCLktE+iGF\niHRoWnkBv/mrRVy3YDw/e2kbF/7nM/z85e20RtvCLk1E+hGFiHQqNzOdf7nmLB754ruZWp7PPz28\njvfd/iLLt2jsLRGJUYhIt2aMLuDeT5/LD6+by9GmVj565yt89uer2HGgIezSRCRkekqRJMTMeM9Z\no7ho2gjufGEr33/mLZ7eWMWn3jWJT7/7NIqHZYZdooiEQE82lKTsO9zItx97k9+8upvsjAjXnzuB\nL1w8hcKcjLBLE5EeONUnGypE5JRs3FfHj55/i4de3U1eZjofPmccH184gQklw8IuTUQSoBCJoxAJ\nz7rdh1n6/FYefX0vUXcumTaCT5w3iUWTSzCzsMsTkU4oROIoRMK3/0gjv3h5O798ZQcH6puZMiKP\njy+ayMXTRjCmKCfs8kSknX4dImZ2JfAdIA24092/1W753wKfAlqBauAmd98eLIsCrwer7nD3q7v7\nPIVI/9HYEuWR1/by0z+9zfo9RzCDq84cxUcXjOfc00pIi6h1ItIf9NsQMbM0YBNwGbALWAkscfc3\n4ta5CHjF3RvM7HPAhe7+kWDZUXfP68lnKkT6H3dnw946HnltDz9/eTt1ja2MLMji6tmjuXb+eCaX\n9eg/sYj0slMNkVRe4jsf2OLuWwHM7D5gMXAiRNz9mbj1XwY+lsJ6JARmxozRBcwYXcCXLpnCHzdU\n8dCru7l7+TbufPFtrphRzqfPP42zxxURUetEZMBJZYiMAXbGvd8FLOhi/U8Cf4h7n21mlcROdX3L\n3R/u/RKlL2VnpPHeWaN476xR1Bxt4p7l27hn+TYeW7+PwpwMLp0+kvfPHaPTXSIDSCpDpKNvgQ7P\nnZnZx4AK4IK42ePdfY+ZnQY8bWavu/tbHWx7M3AzwPjx40+9aukTpXlZ/N3lU/nMBZN5fN0+/vRW\nDU+s38eDq3dRXpDN4jmjWTxnDNNH5evqLpF+LJV9IguBb7j7FcH7rwG4+23t1rsUuB24wN2rOtnX\n3cAj7v5AV5+pPpGBrbElylMb9vPQ6t08t6ma1jZndGE2F00bwSXTR7DwtFJyMtPCLlNkUOnPHevp\nxDrWLwF2E+tY/6i7r49b52zgAeBKd98cN78YaHD3JjMrBV4CFsd3yndEITJ4HDjaxJNv7OfpN6t4\ncUsNDc1RstIjLJpcwsXTR+qSYZFe0m9DBMDMrgL+h9glvne5+7+a2a1ApbsvM7OngLOAvcEmO9z9\najNbBPwIaCM2SOT/uPtPuvs8hcjg1NQa5ZWttTz9ZhXPbKxiezDw49SR+Vw8fQQXTxvB2eOKSE/T\neKIiPdWvQ6SvKUQGP3dna009T2+o4uk3q1i5rZbWNqcwJ4MLp5Zx8bQRXHBGGUW5GhBShobWaBtp\nEUu671AhEkchMvQcaWzhhU01PP1mFc9urOJAfTMRg3kTirloWqyVMnWkOudl8Pre05t5blM1P7tp\nQVJ9hgqROAqRoa2tzVm76xDPvFnF0xurWLf7CABjinK4aFqslaLOeRksGluiPL5+H3//wGtcNHUE\nd1w/L6n9KETiKEQk3v4jjbFAieucz0yLMHdCEYsml7JocgmzxxWRob4UGWAamlu5/icrWLX9IJNK\nh/GLTy1I+kIThUgchYh05njn/Itbalj+Vg3r9xzBHXIz0zhn4nAWTS7hvNNLmT6qQDc6Sr/V2BLl\nv57YyG/X7KHmaBO3vf8s3j937Cn9IdSfhz0R6Tey0tM4/4wyzj+jDIBDDc28vPUAy9+KvW77w5sA\nFOZkcO5pw7lw6ggumzGS0rysMMsWAWIPgXt8/T6eeGMff9pygPPPKOO6BeO5YmZ52KWpJSICUHWk\nkZe2HmD5lgO8uKWG3YeOETGYPqqAueOLWTi5hEWTS3TVl/S5dbsP87GfvMKhhhZyMtL43++dzvXn\nTui1/et0VhyFiPSG4yMPP/HGPlZuq2XNjkPUN0cxg1ljCnnXlFLedXoZcycUkZWuTnpJjWPNUb73\nzGZ+tnw7BTkZ/OTGCk4rzSMzvXf78BQicRQikgot0TbW7jzEC5treHFLDWt2HiLa5mRnRKiYMJx5\nE4qZN6GYOeOLKMjWM+bl1LVE2/j0zyp5dmM1l80YyS3vmZayxyYoROIoRKQv1DW2nOikf3nrATbt\nr6PNwSx2F/3cCcXMHR8LloklubpHRRLm7vxq5U5+s3o3K7bVctv7z2LJ/NQOLKsQiaMQkTDUNbaw\ndudhVm0/yKodB3l1x0HqGlsBGD4s80SgzJtQzKyxhWRn6BSYnMzd+bdHN/DjF96mNC+TL192Btct\n6L2+j87o6iyRkOVnZ8T6SaaUArGbHrdUH42FyvaDrN5+kKc27AcgPWLMHF3A2eOLOX1EHpNKhzGx\ndBijCrL1UK4h7KW3DvDPv1vPm/vquGHhBL5x9cwB04JVS0SkD9TWN/PqjoMnguW1XYc51hI9sTwr\nPcKEklwmlgxjQkkuo4tyGF2Uw7jiXMYOz1FfyyDU1ua8sfcIz22q5kfPvUXxsEw+e8FkPlwxrk/v\nVVJLRGQAGD4sk0umj+SS6SOB2BfIviONbKup5+0D9bGfNQ1srannuU3VNLW2vWP7vKx0RhdlM6ow\nh9K8LErzMxlbnMvYohwKcjIYVZjNiPwsjWTcz7RG2zja1EpdY+y1v66RFW/XsnFfHSu31Z447Tm5\nbBh33nAOk0qHhVxxzylEREIQidiJ1sai00vfsczdqa1vZvehY+w6eIydtQ3sPdzInkPH2Hekkc37\n66g52kxz9J1BEzEYPiyL0rxMyvKzKMvLoqwgixH5sYAZWRD7OaIgi9xM/a/fmcaWKPVNrew6eIyW\naBstUae1re3EdLTNaYm2EW1zWtuc1qhT39RKXVMr1XWNbKtp4GBDMztrG6hvjp60/7SIMaEkl/fN\nGs2M0QVcOn0E5QXZA+b0VXv6TRLpZ8yMkrwsSvKymDW2qMN1jrdk9h1p5HBDC3sPN7L38DFqjjZR\nXddM9dEmtlbXU13XdFLYAAzLTKMkLxY4JXlZlAzLpDAng/zsdLIz0qhvipKTGSEnM53cjDRyM9PI\nyUwjNzM9bjqN3Ix0cjLTqGtsASAzPUJORlrKWkRtbU7N0SZ2HmxgZ+0xdh1sYPehYzS1tOFAtK2D\nL/m2NlqjTsSMrIwImUFt9c2t1DdFOdYcpb65lWPNUeqaWmluPfl4Jao4N4NJpcMYXZTDwsklFOdm\nkpeVTn527FWQncGc8UWDKsQHz79EZAiJb8l0xd051NBCVV0TVXWN7D/SxP4jjRw42syB+qbYF3Jt\nA2t2HqKusYXGluS/QONlpkXIzoiQmxkLmZyMtBM/szPSyEqPkBYx2jz2l3zsL/24n9E2Wtv8HdO1\n9c0caWyhfTduaV7miZGZ0yOx/aZHjPQ0Iy0SIT1ipEWM1rY26uv/HBLDgi/38oLsE8GYF3zR52Sk\nMbY4h6yMNDLSjIy0CBlpkRP7TY+8czo3K428zPQheXGEQkRkEDMziodlUjwsk6nl+d2u39Qa+8s8\nLyudxtY2GoK/0BuCV2y6lWMt8fNayc1MJy1iNLVGaWxpo6E5SmPL8XXbONYc5VhLKw3NrdQcbTrR\nUogc/8Js2annAAAJnUlEQVSPRMhIM9KDL+rczPQTX9DH5+dnpzM8N5MRBVmMK85l3PAcxhTlamj/\nkClEROSErPS0E0O55KVFyMvSV4R0TZdyiIhI0hQiIiKSNIWIiIgkLaUhYmZXmtlGM9tiZrd0sDzL\nzH4VLH/FzCbGLftaMH+jmV2RyjpFRCQ5KQsRM0sDvg+8B5gBLDGzGe1W+yRw0N1PB/4v8O/BtjOA\na4GZwJXAD4L9iYhIP5LKlsh8YIu7b3X3ZuA+YHG7dRYD9wTTDwCXWOy2zcXAfe7e5O5vA1uC/YmI\nSD+SyhAZA+yMe78rmNfhOu7eChwGShLcVkREQpbKEOno1s32QwZ3tk4i28Z2YHazmVWaWWV1dXUP\nSxQRkVORyjuJdgHj4t6PBfZ0ss4uM0sHCoHaBLcFwN2XAksBzKzazLZ3Uk8hsZZOV7pbp7PlPZnf\nfl4pUNNNXb0tkWPRm9snun5X6/V02WA99sns41R/91Nx7KHvj/9APPZdLe+t751Te/KVu6fkRSyg\ntgKTgExgLTCz3TqfB+4Ipq8F7g+mZwbrZwXbbwXSTrGepae6TmfLezK//TygMlX/DU7lWPTm9omu\n39V6PV02WI99qo5/Xx/7MI7/QDz2XS3vL987KWuJuHurmX0BeBxIA+5y9/VmdmvwD1gG/AT4uZlt\nIdYCuTbYdr2Z3Q+8AbQCn3f3k8dU7pnf9cI6nS3vyfxE6ki1U62hp9snun5X6/V02WA99sns41R/\n93Xsk9/HoP/eGVRPNhyIzKzST+GpYpI8Hftw6fiHpzePve5YD9/SsAsYwnTsw6XjH55eO/ZqiYiI\nSNLUEhERkaQpREREJGkKERERSZpCpJ8zs2FmtsrM3hd2LUOJmU03szvM7AEz+1zY9QwlZnaNmf3Y\nzH5rZpeHXc9QY2anmdlPzOyBRNZXiKSImd1lZlVmtq7d/C6Hx+/AV4H7U1Pl4NQbx97dN7j7Z4EP\nA7oMNUG9dOwfdvdPAzcCH0lhuYNOLx3/re7+yYQ/U1dnpYaZnQ8cBX7m7mcG89KATcBlxIZ2WQks\nIXYz5m3tdnETMIvY8ATZQI27P9I31Q9svXHs3b3KzK4GbgG+5+6/7Kv6B7LeOvbBdv8F/D93X91H\n5Q94vXz8H3D3D3b3makcO2tIc/fn4x+yFTgxPD6Amd0HLHb324CTTleZ2UXAMGLPYzlmZo+6e1tK\nCx8EeuPYB/tZBiwzs98DCpEE9NLvvQHfAv6gAOmZ3vrd7wmFSN/qaIj7BZ2t7O7/AGBmNxJriShA\nktejY29mFwLvJzZ+26MprWzw69GxB74IXAoUmtnp7n5HKosbAnr6u18C/Ctwtpl9LQibTilE+lbC\nQ9y/YwX3u3u/lCGnR8fe3Z8Fnk1VMUNMT4/9d4Hvpq6cIaenx/8A8NlEd66O9b6V8BD30ut07MOj\nYx+ulB5/hUjfWglMMbNJZpZJbNTiZSHXNFTo2IdHxz5cKT3+CpEUMbN7gZeAqWa2y8w+6bFHAB8f\nHn8DseenrA+zzsFIxz48OvbhCuP46xJfERFJmloiIiKSNIWIiIgkTSEiIiJJU4iIiEjSFCIiIpI0\nhYiIiCRNISKhMbOjffAZVyc45H5vfuaFZrYoie3ONrM7g+kbzex7vV9dz5nZxPZDi3ewTpmZPdZX\nNUn/oRCRAS8Y6rpD7r7M3b+Vgs/saty5C4Eehwjwv4HbkyooZO5eDew1s/PCrkX6lkJE+gUz+4qZ\nrTSz18zsn+PmPxw82XG9md0cN/+omd1qZq8AC81sm5n9s5mtNrPXzWxasN6Jv+jN7G4z+66ZLTez\nrWb2wWB+xMx+EHzGI2b26PFl7Wp81sz+zcyeA/7azP7CzF4xs1fN7CkzGxkMw/1Z4MtmtsbM3h38\nlf5g8O9b2dEXrZnlA7PcfW0HyyaY2R+DY/NHMxsfzJ9sZi8H+7y1o5adxZ6M+XszW2tm68zsI8H8\nc4LjsNbMVphZftDieCE4hqs7ak2ZWZqZ/Ufcf6vPxC1+GLiuw//AMni5u156hfICjgY/LweWEhtt\nNAI8ApwfLBse/MwB1gElwXsHPhy3r23AF4PpvwLuDKZvJPZQKYC7gV8HnzGD2DMWAD5IbLj3CFAO\nHAQ+2EG9zwI/iHtfzJ9HffgU8F/B9DeA/xW33i+BdwXT44ENHez7IuDBuPfxdf8OuCGYvgl4OJh+\nBFgSTH/2+PFst98PAD+Oe18IZAJbgXOCeQXERvTOBbKDeVOAymB6IrAumL4Z+MdgOguoBCYF78cA\nr4f9e6VX3740FLz0B5cHr1eD93nEvsSeB75kZn8ZzB8XzD8ARIEH2+3nN8HPVcSeBdKRhz32XJY3\nzGxkMO9dwK+D+fvM7Jkuav1V3PRY4FdmNorYF/PbnWxzKTDD7MSI3AVmlu/udXHrjAKqO9l+Ydy/\n5+fAt+PmXxNM/xL4zw62fR34TzP7d+ARd3/BzM4C9rr7SgB3PwKxVgvwPTObQ+z4ntHB/i4HZsW1\n1AqJ/Td5G6gCRnfyb5BBSiEi/YEBt7n7j94xM/ZgqEuBhe7eYGbPEntUMECju0fb7acp+Bml89/t\nprhpa/czEfVx07cD/+3uy4Jav9HJNhFi/4ZjXez3GH/+t3Un4QHv3H2Tmc0DrgJuM7MniJ126mgf\nXwb2A7ODmhs7WMeItfge72BZNrF/hwwh6hOR/uBx4CYzywMwszFmNoLYX7kHgwCZBpybos9/EfhA\n0DcykljHeCIKgd3B9A1x8+uA/Lj3TxAbRRWA4C/99jYAp3fyOcuJDd8NsT6HF4Ppl4mdriJu+TuY\n2Wigwd1/QaylMhd4ExhtZucE6+QHFwoUEmuhtAHXE3sGd3uPA58zs4xg2zOCFgzEWi5dXsUlg49C\nRELn7k8QOx3zkpm9DjxA7Ev4MSDdzF4DvknsSzMVHiT24J51wI+AV4DDCWz3DeDXZvYCUBM3/3fA\nXx7vWAe+BFQEHdFv0MFT49z9TWKPg81vvyzY/hPBcbge+Otg/t8Af2tmK4idDuuo5rOAFWa2BvgH\n4F/cvRn4CHC7ma0FniTWivgBcIOZvUwsEOo72N+dwBvA6uCy3x/x51bfRcDvO9hGBjENBS8CmFme\nux+12POlVwDnufu+Pq7hy0Cdu9+Z4Pq5wDF3dzO7llgn++KUFtl1Pc8Di939YFg1SN9Tn4hIzCNm\nVkSsg/ybfR0ggR8CH+rB+vOIdYQbcIjYlVuhMLMyYv1DCpAhRi0RERFJmvpEREQkaQoRERFJmkJE\nRESSphAREZGkKURERCRpChEREUna/wdBgdlE7WADOAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f0e1a5a24a8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "m.sched.plot(100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = md.get_learner(emb_szs, len(df.columns)-len(cat_vars),\n",
    "                   0.04, 1, [1000,500], [0.001,0.01], y_range=y_range)\n",
    "lr = 1e-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2127e0eaa44d4189b569de2c6544e6a5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.       0.02479  0.02205  0.19309]                          \n",
      "[ 1.       0.02044  0.01751  0.18301]                          \n",
      "[ 2.       0.01598  0.01571  0.17248]                          \n",
      "\n"
     ]
    }
   ],
   "source": [
    "m.fit(lr, 3, metrics=[exp_rmspe])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e03e0e29fbdb4098b8297a41f50a9054",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.       0.01258  0.01278  0.16   ]                          \n",
      "[ 1.       0.01147  0.01214  0.15758]                          \n",
      "[ 2.       0.01157  0.01157  0.15585]                          \n",
      "[ 3.       0.00984  0.01124  0.15251]                           \n",
      "[ 4.       0.00946  0.01094  0.15197]                           \n",
      "\n"
     ]
    }
   ],
   "source": [
    "m.fit(lr, 5, metrics=[exp_rmspe], cycle_len=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1fe2c401969c45a39dc02c2c7c86291d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.       0.01179  0.01242  0.15512]                           \n",
      "[ 1.       0.00921  0.01098  0.15003]                           \n",
      "[ 2.       0.00771  0.01031  0.14431]                           \n",
      "[ 3.       0.00632  0.01016  0.14358]                           \n",
      "[ 4.       0.01003  0.01305  0.16574]                           \n",
      "[ 5.       0.00827  0.01087  0.14937]                           \n",
      "[ 6.       0.00628  0.01025  0.14506]                           \n",
      "[ 7.       0.0053   0.01     0.14449]                           \n",
      "\n"
     ]
    }
   ],
   "source": [
    "m.fit(lr, 2, metrics=[exp_rmspe], cycle_len=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### All"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = md.get_learner(emb_szs, len(df.columns)-len(cat_vars),\n",
    "                   0.04, 1, [1000,500], [0.001,0.01], y_range=y_range)\n",
    "lr = 1e-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "15d0c7d5e9634030a624fb91c90b081c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.       0.01456  0.01544  0.1148 ]                            \n",
      "\n"
     ]
    }
   ],
   "source": [
    "m.fit(lr, 1, metrics=[exp_rmspe])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "58bd52a856754c359eae9fa25f951dd6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.       0.01418  0.02066  0.12765]                           \n",
      "[ 1.       0.01081  0.01276  0.11221]                           \n",
      "[ 2.       0.00976  0.01233  0.10987]                            \n",
      "\n"
     ]
    }
   ],
   "source": [
    "m.fit(lr, 3, metrics=[exp_rmspe])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "738899f6b0574b50be6d3f6efff9e578",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.       0.00801  0.01081  0.09899]                            \n",
      "[ 1.       0.00714  0.01083  0.09846]                            \n",
      "[ 2.       0.00707  0.01088  0.09878]                            \n",
      "\n"
     ]
    }
   ],
   "source": [
    "m.fit(lr, 3, metrics=[exp_rmspe], cycle_len=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = md.get_learner(emb_szs, len(df.columns)-len(cat_vars),\n",
    "                   0.04, 1, [1000,500], [0.001,0.01], y_range=y_range)\n",
    "lr = 1e-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "74827eff47794b5eb8d09982d30202fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.       0.01413  0.0063   0.07628]                           \n",
      "[ 1.       0.01022  0.00859  0.08851]                           \n",
      "[ 2.       0.00932  0.00001  0.00243]                            \n",
      "\n"
     ]
    }
   ],
   "source": [
    "m.fit(lr, 3, metrics=[exp_rmspe])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "07f0b0fea31c4afa8c6770085d233230",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.       0.00748  0.       0.00167]                            \n",
      "[ 1.       0.00717  0.00009  0.00947]                            \n",
      "[ 2.       0.00643  0.00013  0.01147]                            \n",
      "\n"
     ]
    }
   ],
   "source": [
    "m.fit(lr, 3, metrics=[exp_rmspe], cycle_len=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m.save('val0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m.load('val0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x,y=m.predict_with_targs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.01147316926177568"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exp_rmspe(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_test=m.predict(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_test = np.exp(pred_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_test['Sales']=pred_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_fn=f'{PATH}tmp/sub.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_test[['Id','Sales']].to_csv(csv_fn, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href='data/rossmann/tmp/sub.csv' target='_blank'>data/rossmann/tmp/sub.csv</a><br>"
      ],
      "text/plain": [
       "/home/ubuntu/fastai/courses/dl1/data/rossmann/tmp/sub.csv"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FileLink(csv_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "((val,trn), (y_val,y_trn)) = split_by_idx(val_idx, df.values, yl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = RandomForestRegressor(n_estimators=40, max_features=0.99, min_samples_leaf=2,\n",
    "                          n_jobs=-1, oob_score=True)\n",
    "m.fit(trn, y_trn);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.98086411192483902,\n",
       " 0.92614447508562714,\n",
       " 0.9193358549649463,\n",
       " 0.11557443993375387)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = m.predict(val)\n",
    "m.score(trn, y_trn), m.score(val, y_val), m.oob_score_, exp_rmspe(preds, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}