Gru4Rec_Keras.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Gru4Rec - Keras.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZbN_xrx6YQON",
        "colab_type": "text"
      },
      "source": [
        "# **GRU4Rec**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tilqU4efYjJs",
        "colab_type": "text"
      },
      "source": [
        "In order to train the GRU4Rec model, a dataset is needed with the following columns: **session**, **timestamp**, **item**. Each row corresponds to a single interaction. The dataset is sorted by session and timestamp to facilitate session retrieval.\n",
        "\n",
        "We use the SessionDataset class to store the training data plus some ready-to-use information, like session offsets (= indices where each session starts in the dataset) and item-to-integer mappings. We also define a function to extract a single specific session."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S493qprbsngR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "class SessionDataset:\n",
        "\n",
        "      def __init__(self, df):\n",
        "\n",
        "          self.df = df.sort_values(by = ['session', 'timestamp']).reset_index(drop = True) # session (int) | timestamp (int) | item (string)\n",
        "          self.offsets    = np.concatenate((np.zeros(1, dtype = np.int32), self.df.groupby('session').size().cumsum().values)) # indices in df where the sessions start\n",
        "          self.n_sessions = len(self.offsets) - 1\n",
        "\n",
        "          self.item_to_id = {item : i for i, item in enumerate(self.df.item.unique())}\n",
        "\n",
        "          self.n_items = len(self.item_to_id)\n",
        "\n",
        "      def item_to_one_hot(self, item):\n",
        "\n",
        "          return tf.one_hot(self.item_to_id[item], depth = self.n_items)\n",
        "\n",
        "      def extract_session(self, i, one_hot_encoded = True):\n",
        "\n",
        "          session = self.df[self.offsets[i]:self.offsets[i+1]].copy()\n",
        "          if one_hot_encoded:\n",
        "              session.loc[:, 'item'] = session.item.apply(lambda x : self.item_to_one_hot(x))\n",
        "          return session.item.values.tolist()\n"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JpvVHSYtbzme",
        "colab_type": "text"
      },
      "source": [
        "## Loss functions: TOP1 and BPR"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BL6qBm82U73G",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# y_true = (BATCH_SIZE, n_classes)   one-hot representations of the target items (ground truths)\n",
        "# y_pred = (BATCH_SIZE, n_classes)   model output = next item scores (logits) for each item in the batch\n",
        "\n",
        "sampling = False\n",
        "\n",
        "if sampling: # = the negative items considered in the loss computation are those within the same batch\n",
        "    \n",
        "    def BPR(y_true, y_pred):\n",
        "        to_lookup = tf.argmax(y_true, axis = 1)   # = indices of the target items\n",
        "        scores = tf.nn.embedding_lookup(tf.transpose(y_pred), to_lookup)  # embedding_lookup is the same as \"extract_rows\". In this way, the positive items end up on the diagonal\n",
        "        return tf.reduce_mean(-tf.math.log(tf.nn.sigmoid(tf.linalg.diag_part(scores) - scores)))\n",
        "\n",
        "    def TOP1(y_true, y_pred):\n",
        "        to_lookup = tf.argmax(y_true, axis = 1)\n",
        "        scores = tf.nn.embedding_lookup(tf.transpose(y_pred), to_lookup)\n",
        "        diag_scores = tf.linalg.diag_part(scores)\n",
        "        loss_by_sample  = tf.reduce_mean(tf.nn.sigmoid(scores - diag_scores) + tf.nn.sigmoid(tf.square(scores)), axis = 0)\n",
        "        loss_by_sample -= tf.nn.sigmoid(tf.square(diag_scores)) / tf.reduce_sum(tf.ones_like(diag_scores)) # only sigmoids of squares of negative items had to be added: remove those of positive items\n",
        "        return tf.reduce_mean(loss_by_sample)\n",
        "\n",
        "else: # = consider all negative items in the loss computation (only makes sense if the number of items is small, like the same order as the batch size)\n",
        "\n",
        "    def BPR(y_true, y_pred):  # both inputs have shape (BATCH_SIZE, n_classes)\n",
        "        _y_pred = tf.expand_dims(y_pred, axis = -1)  # (BATCH_SIZE, n_classes, 1) \n",
        "        mat = tf.matmul(tf.expand_dims(tf.ones_like(y_true), -1), tf.expand_dims(y_true, axis = 1)) # (BATCH_SIZE, n_classes, 1) x (BATCH_SIZE, 1, n_classes) = (BATCH_SIZE, n_classes, n_classes)\n",
        "        score_diffs = tf.matmul(mat, _y_pred) # (BATCH_SIZE, n_classes, n_classes) x (BATCH_SIZE, n_classes, 1) = (BATCH_SIZE, n_classes, 1)\n",
        "        score_diffs = tf.squeeze(score_diffs - _y_pred, -1) # (BATCH_SIZE, n_classes)\n",
        "        return -tf.reduce_sum(tf.math.log(tf.nn.sigmoid(score_diffs)))\n",
        "\n",
        "    def TOP1(y_true, y_pred):\n",
        "        _y_pred = tf.expand_dims(y_pred, axis = -1)  # (BATCH_SIZE, n_classes) ---> (BATCH_SIZE, n_classes, 1) \n",
        "        mat = tf.matmul(tf.expand_dims(tf.ones_like(y_true), -1), tf.expand_dims(y_true, axis = 1)) # (BATCH_SIZE, n_classes, 1) x (BATCH_SIZE, 1, n_classes) --> (BATCH_SIZE, n_classes, n_classes)\n",
        "        score_diffs = tf.matmul(mat, _y_pred) # (BATCH_SIZE, n_classes, n_classes) x (BATCH_SIZE, n_classes, 1) --> (BATCH_SIZE, n_classes, 1)\n",
        "        score_diffs = tf.squeeze(score_diffs - _y_pred, -1) # (BATCH_SIZE, n_classes)\n",
        "        loss_by_sample = tf.reduce_sum(tf.nn.sigmoid(tf.square(y_pred)), axis = -1) + \\\n",
        "                          tf.reduce_sum(tf.sigmoid(-score_diffs), axis = -1) + \\\n",
        "                        -tf.squeeze(tf.squeeze(tf.nn.sigmoid(tf.square(tf.matmul(tf.expand_dims(y_true, 1), _y_pred))), -1), -1)\n",
        "        return tf.reduce_sum(loss_by_sample)"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-L0RjkCmZo9c",
        "colab_type": "text"
      },
      "source": [
        "The Gru4Rec class is used to instantiate the model and train it on a SessionDataset object."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SYWjZhJHXcHY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Gru4Rec:\n",
        "\n",
        "    def __init__(self, n_classes, n_layers = 1, n_hidden = 64, loss = TOP1, batch_size = 8):\n",
        "\n",
        "        self.n_classes  = n_classes   # = number of items\n",
        "\n",
        "        self.n_layers = n_layers  # number of stacked GRU layers\n",
        "        self.n_hidden = n_hidden  # dimension of GRU cell's hidden state\n",
        "        self.loss     = loss\n",
        "        self.batch_size = batch_size\n",
        "\n",
        "        self.model = self.build_model()\n",
        "\n",
        "    def build_model(self):\n",
        "\n",
        "        model = tf.keras.models.Sequential()\n",
        "        for i in range(self.n_layers):\n",
        "            model.add(tf.keras.layers.GRU(name = 'GRU_{}'.format(i+1),\n",
        "                                          units      = self.n_hidden, \n",
        "                                          activation = 'relu', \n",
        "                                          stateful   = True,\n",
        "                                          return_sequences = (i < self.n_layers - 1)))\n",
        "        model.add(tf.keras.layers.Dense(units = self.n_classes, activation = 'linear'))   # class logits\n",
        "\n",
        "        # track top 3 accuracy (= how often the true item is among the top 3 recommended)\n",
        "        top3accuracy = lambda y_true, y_pred: tf.keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k = 3)\n",
        "        top3accuracy.__name__ = 'top3accuracy'\n",
        "        model.compile(loss = self.loss, optimizer = 'adam', metrics = ['accuracy', top3accuracy])\n",
        "\n",
        "        model.build(input_shape = (self.batch_size, 1, self.n_classes))\n",
        "        print(model.summary())\n",
        "\n",
        "        return model\n",
        "\n",
        "    def _reset_hidden(self, i):\n",
        "\n",
        "        for nl, layer in enumerate(self.model.layers):   # session has changed: reset corresponding hidden state\n",
        "            if self._is_GRU_layer(layer) and layer.states[0] is not None:\n",
        "                hidden_updated = layer.states[0].numpy()\n",
        "                hidden_updated[i, :] = 0.\n",
        "                self.model.layers[nl].reset_states(hidden_updated)\n",
        "\n",
        "    def _is_GRU_layer(self, layer):\n",
        "\n",
        "        return layer.name.startswith('GRU_')\n",
        "\n",
        "    def train_batch_generator(self, dataset):  # session | item | timestamp\n",
        "        # generates batches of training data X, y = session item, next session item\n",
        "\n",
        "        assert dataset.n_sessions > self.batch_size, \"Training set is too small. Reduce batch size or collect more training data\"\n",
        "        ixs = np.arange(dataset.n_sessions)\n",
        "\n",
        "        stacks = [[]] * self.batch_size   # stacks containing batch_size REVERSED (pieces of) sessions at once. Will be emptied progressively\n",
        "        next_session_id = 0\n",
        "\n",
        "        X, y = np.empty(shape = (self.batch_size, 1, self.n_classes)), np.empty(shape = (self.batch_size, self.n_classes))    \n",
        "        while True:\n",
        "            X[:], y[:] = None, None\n",
        "            for i in range(self.batch_size): # fill in X, y (current batch)\n",
        "                # 1. If stack i is empty (only happens at first round) or has only one element: fill it with a new session\n",
        "                if len(stacks[i]) <= 1:\n",
        "                    if next_session_id >= dataset.n_sessions: # no more sessions available: shuffle sessions and restart\n",
        "                        np.random.shuffle(ixs)\n",
        "                        next_session_id = 0\n",
        "                    while not len(stacks[i]) >= 2:   # ignore sessions with only one element (cannot contribute to the training)\n",
        "                        stacks[i] = dataset.extract_session(ixs[next_session_id])[::-1]  # the data does not have to be all in memory at the same time: we could e.g. load a session at once\n",
        "                        next_session_id += 1\n",
        "                    self._reset_hidden(i)   # if session changes, the corresponding hidden state must be reset\n",
        "                # 2. Stack i is now valid: set input + target variables\n",
        "                X[i, 0] = stacks[i].pop()\n",
        "                y[i]    = stacks[i][-1]\n",
        "\n",
        "            yield tf.constant(X, dtype = tf.float32), tf.constant(y, dtype = tf.float32)\n",
        "\n",
        "    def fit(self, dataset, steps_per_epoch = 10000, epochs = 5):\n",
        "\n",
        "        checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath = \"gru-chkpt-{epoch:02d}.hdf5\")\n",
        "        self.model.fit_generator(generator       = self.train_batch_generator(dataset), \n",
        "                                 steps_per_epoch = steps_per_epoch, \n",
        "                                 epochs          = epochs,\n",
        "                                 callbacks       = [checkpoint], \n",
        "                                 shuffle         = False)"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OiXqzSsc27eH",
        "colab_type": "text"
      },
      "source": [
        "## Test the model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rvmnM7fjaWLD",
        "colab_type": "text"
      },
      "source": [
        "We test the model on a simple workout recommendation task.\n",
        "\n",
        "We used the workout records dataset available at https://sites.google.com/eng.ucsd.edu/fitrec-project/home (see quoted paper in the Readme). We extracted the plain list of workouts in time and performed some cleaning (removal of one-item sessions and duplicate or overlapping sessions). The result is a dataset of workout sequences for slightly more than 1000 users. We treat each sequence (i.e., each user) as a distinct session. The goal is to predict the next user workout given all the previous ones. There is a high variance in session length (from 2 to about 200). The number of workouts (or classes) is only 48, so we use no sampling in the losses.\n",
        "\n",
        "The dataset is split into a training and test set. The test set contains the last two items in each session, the training set contains all the sessions with the last item removed."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gXH0nVjSGEed",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        },
        "outputId": "9dc20f5c-c23f-40b7-b0fb-ce7d30e7d807"
      },
      "source": [
        "import pandas as pd\n",
        "df = pd.read_csv(\"workouts_clean_2.csv\").sort_values(by = ['session', 'timestamp']).reset_index(drop = True)\n",
        "offsets = np.concatenate((np.zeros(1, dtype = np.int32), df.groupby('session').size().cumsum().values))\n",
        "\n",
        "dataset_train = SessionDataset(df.iloc[~df.index.isin(offsets[1:] - 1)])  # training set: remove last element from each session\n",
        "\n",
        "# Test set: x = penultimate item in each session, y = last item in each session\n",
        "X_test = df.iloc[offsets[1:] - 2][['session', 'item']].sort_values(by = ['session']).reset_index(drop = True)\n",
        "y_test = df.iloc[offsets[1:] - 1][['session', 'item']].sort_values(by = ['session']).reset_index(drop = True)\n",
        "\n",
        "print(\"X_test\")\n",
        "print(X_test.head())\n",
        "print('')\n",
        "print(\"y_test\")\n",
        "print(y_test.head())"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "X_test\n",
            "   session              item\n",
            "0       69               run\n",
            "1     2358               run\n",
            "2     3808               run\n",
            "3     4101     mountain bike\n",
            "4     4434  bike (transport)\n",
            "\n",
            "y_test\n",
            "   session              item\n",
            "0       69               run\n",
            "1     2358               run\n",
            "2     3808               run\n",
            "3     4101               run\n",
            "4     4434  bike (transport)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LMj1va3RaHc2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 490
        },
        "outputId": "7ccaee10-0ef9-4cb0-b888-2db1b309d7ba"
      },
      "source": [
        "g4r = Gru4Rec(n_classes = dataset_train.n_items)\n",
        "g4r.fit(dataset_train)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Layer GRU_1 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "GRU_1 (GRU)                  multiple                  21888     \n",
            "_________________________________________________________________\n",
            "dense (Dense)                multiple                  3120      \n",
            "=================================================================\n",
            "Total params: 25,008\n",
            "Trainable params: 25,008\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n",
            "WARNING:tensorflow:From <ipython-input-3-ff2011fbc225>:82: Model.fit_generator (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use Model.fit, which supports generators.\n",
            "Epoch 1/5\n",
            "10000/10000 [==============================] - 54s 5ms/step - loss: 224.9283 - accuracy: 0.6555 - top3accuracy: 0.9273\n",
            "Epoch 2/5\n",
            "10000/10000 [==============================] - 53s 5ms/step - loss: 206.0282 - accuracy: 0.7788 - top3accuracy: 0.9734\n",
            "Epoch 3/5\n",
            "10000/10000 [==============================] - 52s 5ms/step - loss: 204.8075 - accuracy: 0.7929 - top3accuracy: 0.9771\n",
            "Epoch 4/5\n",
            "10000/10000 [==============================] - 51s 5ms/step - loss: 203.5246 - accuracy: 0.7947 - top3accuracy: 0.9806\n",
            "Epoch 5/5\n",
            "10000/10000 [==============================] - 51s 5ms/step - loss: 203.8550 - accuracy: 0.7923 - top3accuracy: 0.9777\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DSliYlMqG1KA",
        "colab_type": "text"
      },
      "source": [
        "## Evaluation: calculate and visualize top-3 test accuracy"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CfBIp2kVa_7x",
        "colab_type": "text"
      },
      "source": [
        "To evaluate the model, we predict the last item in each session based on the previous items. Since the network is stateful, the batch size cannot be modified (at least in Keras), so we must always predict batch_size elements at once.\n",
        "\n",
        "First, we calculate and store the final hidden states for all sessions in the training set. This is done in sequence (although it could be parallelized) and takes a few minutes to complete."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9yaGD_NP1EYa",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 202
        },
        "outputId": "2833b23f-2761-4843-b83b-53e7efe2fbd5"
      },
      "source": [
        "final_states = np.empty(shape = (dataset_train.n_sessions, g4r.n_layers, g4r.n_hidden)) # final states will be stored here\n",
        "final_states[:] = None\n",
        "done = [False] * dataset_train.n_sessions   # keep track of the sessions for which the last state has already been calculated\n",
        "\n",
        "stacks = [dataset_train.extract_session(i)[::-1] for i in range(g4r.batch_size)]\n",
        "next_session_id = g4r.batch_size\n",
        "batch_idx_to_session = np.arange(g4r.batch_size)   # keep track of which session is in each batch element\n",
        "X = np.empty(shape = (g4r.batch_size, 1, g4r.n_classes))\n",
        "\n",
        "g4r.model.reset_states()    # all hidden states set to 0 (starting point)\n",
        "\n",
        "n_done = 0\n",
        "while n_done < dataset_train.n_sessions:\n",
        "    for i in range(g4r.batch_size):\n",
        "        while len(stacks[i]) == 1:  # stack i is at the end\n",
        "            if not done[batch_idx_to_session[i]]:\n",
        "                # save final hidden state\n",
        "                final_states[batch_idx_to_session[i], :] = np.array([layer.states[0][i, :] for layer in g4r.model.layers if g4r._is_GRU_layer(layer)])\n",
        "                done[batch_idx_to_session[i]] = True\n",
        "                n_done += 1\n",
        "                if n_done % 100 == 0:\n",
        "                    print(\"Progress: {} / {}\".format(n_done, dataset_train.n_sessions))\n",
        "            if next_session_id >= dataset_train.n_sessions: # restart from the beginning (just to reach required batch size)\n",
        "                next_session_id = 0\n",
        "            stacks[i] = dataset_train.extract_session(next_session_id)[::-1]\n",
        "            batch_idx_to_session[i] = next_session_id\n",
        "            next_session_id += 1\n",
        "            g4r._reset_hidden(i)   # session has changed --> reset corresponding hidden state\n",
        "        X[i, 0] = stacks[i].pop()\n",
        "\n",
        "    _ = g4r.model.predict(X)   # hidden states get updated when \"predict\" is called\n",
        "\n",
        "print(\"All final hidden states calculated\")\n",
        "np.save('final_states.npy', final_states, allow_pickle = False)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Progress: 100 / 1082\n",
            "Progress: 200 / 1082\n",
            "Progress: 300 / 1082\n",
            "Progress: 400 / 1082\n",
            "Progress: 500 / 1082\n",
            "Progress: 600 / 1082\n",
            "Progress: 700 / 1082\n",
            "Progress: 800 / 1082\n",
            "Progress: 900 / 1082\n",
            "Progress: 1000 / 1082\n",
            "All final hidden states calculated\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TsUJmOyodaRM",
        "colab_type": "text"
      },
      "source": [
        "We can now calculate predictions on the test set and accuracy metrics."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nOgKazJsDUJp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "final_states = np.load('final_states.npy')\n",
        "\n",
        "g4r.model.reset_states()\n",
        "\n",
        "rem = dataset_train.n_sessions % g4r.batch_size\n",
        "if rem > 0:\n",
        "    X_test = pd.concat((X_test, X_test[:(g4r.batch_size - rem)]), axis = 0)\n",
        "\n",
        "# Calculate next item predictions for all sessions\n",
        "y_pred = np.empty(shape = (dataset_train.n_sessions, g4r.n_classes))\n",
        "y_pred[:] = None\n",
        "X = np.empty(shape = (g4r.batch_size, 1, g4r.n_classes))\n",
        "for batch_id in range(dataset_train.n_sessions // g4r.batch_size):\n",
        "    # X contains the penultimate item in the session (= last item in the training set)\n",
        "    X[:] = None\n",
        "    for i in range(g4r.batch_size):\n",
        "        X[i, :] = dataset_train.item_to_one_hot(X_test.iloc[batch_id * g4r.batch_size + i]['item'])\n",
        "    # set hidden states equal to final hidden states for sessions in the batch\n",
        "    nlg = 0\n",
        "    for nl, layer in enumerate(g4r.model.layers):\n",
        "        if g4r._is_GRU_layer(layer):\n",
        "            g4r.model.layers[nl].reset_states(final_states[batch_id * g4r.batch_size : (batch_id + 1) * g4r.batch_size, nlg, :])\n",
        "            nlg += 1\n",
        "    # objective: predict last element in the session\n",
        "    y_pred[batch_id * g4r.batch_size : (batch_id + 1) * g4r.batch_size, :] = g4r.model.predict(X)[:g4r.batch_size]\n",
        "\n",
        "y_pred = tf.constant(y_pred[:dataset_train.n_sessions], dtype = tf.float32)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zwEr9AQ8DezI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Retrieve ground truths\n",
        "y_true = np.empty(shape = (dataset_train.n_sessions, dataset_train.n_items))\n",
        "for i in range(y_true.shape[0]):\n",
        "    y_true[i, :] = dataset_train.item_to_one_hot(y_test.item.values[i])\n",
        "y_true = tf.constant(y_true, dtype = tf.float32)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lWzJCzaqGe7W",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "outputId": "a5c8fc3d-85ea-4665-cb5a-00d90f05665f"
      },
      "source": [
        "acc       = (tf.reduce_sum(tf.keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k = 1)) / y_true.shape[0]).numpy()\n",
        "top_3_acc = (tf.reduce_sum(tf.keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k = 3)) / y_true.shape[0]).numpy()\n",
        "\n",
        "print(\"Accuracy = {}\".format(acc))\n",
        "print(\"Top-3 accuracy = {}\".format(top_3_acc))"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy = 0.8133087158203125\n",
            "Top-3 accuracy = 0.9611830115318298\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LokLpm9bdh9X",
        "colab_type": "text"
      },
      "source": [
        "For this particular problem, which is quite simple, the performances are not significantly better than those of simpler baselines (e.g. a linear autoregressive model)."
      ]
    }
  ]
}