Pseudo_Labelling_MNIST_2nd.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Pseudo-Labelling-MNIST-2nd",
      "provenance": [],
      "private_outputs": true,
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPIrhyzG8ti9PXpAWgRx1/d",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/consequencesunintended/Pseudo-Labelling/blob/master/Pseudo_Labelling_MNIST_2nd.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nt86cVf9qy7-",
        "colab_type": "text"
      },
      "source": [
        "# **Pseudo Labelling on MNIST dataset**\n",
        "\n",
        "References:\n",
        "\n",
        "*1 - Pseudo-Label : The Simple and Efficient Semi-Supervised Learning\n",
        "Method for Deep Neural Networks, Dong-Hyun Lee\n",
        "http://deeplearning.net/wp-content/uploads/2013/03/pseudo_label_final.pdf*\n",
        "\n",
        "*2 - Naive semi-supervised deep learning using pseudo-label, Zhun Li, ByungSoo Ko & Ho-Jin Choi\n",
        "https://link.springer.com/article/10.1007/s12083-018-0702-9*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2ZyrRGAgyGod",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras.layers import Input, Dense, Reshape, Flatten, Softmax, Dropout, Conv2D, MaxPooling2D\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras import losses\n",
        "from tensorflow.keras import Sequential\n",
        "import tensorflow as tf\n",
        "import tensorflow.keras as keras\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t5axBk82yU61",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.datasets import mnist\n",
        "import numpy as np\n",
        "(x_train, y_train), (x_test, y_test) = mnist.load_data()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "htVHk9a5z-4N",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "classifier = Sequential()\n",
        "classifier.add(Conv2D(32, kernel_size=(3, 3),\n",
        "                 activation='relu',\n",
        "                 input_shape=(28, 28, 1)))\n",
        "classifier.add(Conv2D(64, (3, 3), activation='relu'))\n",
        "classifier.add(MaxPooling2D(pool_size=(2, 2)))\n",
        "classifier.add(Dropout(0.25))\n",
        "classifier.add(Flatten())\n",
        "classifier.add(Dense(128, activation='relu'))\n",
        "classifier.add(Dropout(0.5))\n",
        "classifier.add(Dense(10, activation='softmax'))\n",
        "\n",
        "classifier.build()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MlACEjfzyWTl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x_train = x_train.astype('float32') / 255.\n",
        "x_test = x_test.astype('float32') / 255.\n",
        "x_train = x_train.reshape( (len(x_train),28,28,1))\n",
        "x_test = x_test.reshape( (len(x_test),28,28,1))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CxW_OQCcpHiw",
        "colab_type": "text"
      },
      "source": [
        "# **Split the data**\n",
        "The idea here is to have a small labelled dataset and a large unlablled set to help improve increasing the accuracy of the model in scenario were we dont have access to large labelled set. We also set aside some data for evaluation test. We are making sure they are equal number of samples for each number exist in each dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8EBnOFUfyjm_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data_0 = [ x for x,y in zip(x_train,y_train) if y == 0 ] \n",
        "data_1 = [ x for x,y in zip(x_train,y_train) if y == 1 ] \n",
        "data_2 = [ x for x,y in zip(x_train,y_train) if y == 2 ] \n",
        "data_3 = [ x for x,y in zip(x_train,y_train) if y == 3 ] \n",
        "data_4 = [ x for x,y in zip(x_train,y_train) if y == 4 ] \n",
        "data_5 = [ x for x,y in zip(x_train,y_train) if y == 5 ] \n",
        "data_6 = [ x for x,y in zip(x_train,y_train) if y == 6 ] \n",
        "data_7 = [ x for x,y in zip(x_train,y_train) if y == 7 ] \n",
        "data_8 = [ x for x,y in zip(x_train,y_train) if y == 8 ] \n",
        "data_9 = [ x for x,y in zip(x_train,y_train) if y == 9 ] "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0PqmlE1M2hQc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "source = 0\n",
        "target = 10\n",
        "x_train_shorten = data_0[source:target] + data_1[source:target] + data_2[source:target] + data_3[source:target] + data_4[source:target] + data_5[source:target] + data_6[source:target] + data_7[source:target] + data_8[source:target] + data_9[source:target]\n",
        "y_train_shorten = list(np.zeros(target)) + list(np.ones(target)) + list( np.ones(target) * 2 ) + list( np.ones(target) * 3 ) + list( np.ones(target) * 4 ) + list( np.ones(target) * 5 ) + list( np.ones(target) * 6 ) + list( np.ones(target) * 7 ) + list( np.ones(target) * 8 ) + list( np.ones(target) * 9 )\n",
        "\n",
        "x_train_shorten = np.array( x_train_shorten )\n",
        "y_train_shorten = np.array( y_train_shorten )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lBGrsleIK70U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "source = 10\n",
        "target = 250\n",
        "x_train_unlabelled = data_0[source:target] + data_1[source:target] + data_2[source:target] + data_3[source:target] + data_4[source:target] + data_5[source:target] + data_6[source:target] + data_7[source:target] + data_8[source:target] + data_9[source:target]\n",
        "y_train_unlabelled = list(np.zeros(target - source)) + list(np.ones(target - source)) + list( np.ones(target - source) * 2 ) + list( np.ones(target - source) * 3 ) + list( np.ones(target - source) * 4 ) + list( np.ones(target - source) * 5 ) + list( np.ones(target - source) * 6 ) + list( np.ones(target - source) * 7 ) + list( np.ones(target - source) * 8 ) + list( np.ones(target - source) * 9 )\n",
        "\n",
        "x_train_unlabelled = np.array( x_train_unlabelled )\n",
        "y_train_unlabelled = np.array( y_train_unlabelled )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2WRPcnIQLOST",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "source = 250\n",
        "target = 450\n",
        "x_eval_shorten = data_0[source:target] + data_1[source:target] + data_2[source:target] + data_3[source:target] + data_4[source:target] + data_5[source:target] + data_6[source:target] + data_7[source:target] + data_8[source:target] + data_9[source:target]\n",
        "y_eval_shorten = list(np.zeros(target - source)) + list(np.ones(target - source)) + list( np.ones(target - source) * 2 ) + list( np.ones(target - source) * 3 ) + list( np.ones(target - source) * 4 ) + list( np.ones(target - source) * 5 ) + list( np.ones(target - source) * 6 ) + list( np.ones(target - source) * 7 ) + list( np.ones(target - source) * 8 ) + list( np.ones(target - source) * 9 )\n",
        "\n",
        "x_eval_shorten = np.array( x_eval_shorten )\n",
        "y_eval_shorten = np.array( y_eval_shorten )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5PvmPHds52Ir",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "classifier.compile(optimizer='adam',\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R7txwYuyog8o",
        "colab_type": "text"
      },
      "source": [
        "# **Define the datasets**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EU40O0A84sSo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "batch_size = 100\n",
        "\n",
        "train_dataset = tf.data.Dataset.from_tensor_slices((x_train_shorten,y_train_shorten))\n",
        "train_dataset = train_dataset.shuffle(1000).batch(batch_size)\n",
        "\n",
        "pseudo_dataset = tf.data.Dataset.from_tensor_slices((x_train_unlabelled))\n",
        "pseudo_dataset = pseudo_dataset.shuffle(1000).batch(batch_size)\n",
        "\n",
        "eval_dataset = tf.data.Dataset.from_tensor_slices((x_eval_shorten,y_eval_shorten))\n",
        "eval_dataset = eval_dataset.shuffle(1000).batch(batch_size)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_ujmYyVPhvNk",
        "colab_type": "text"
      },
      "source": [
        "# **Pre-train the Model with labelled data**\n",
        "Firstly we pre-train the model here with 30 epochs on only labelled data, and achieving a maximum accuracy of around 70 percent."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_TX6I91756zE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "classifier.fit(train_dataset, validation_data=eval_dataset, batch_size=100, epochs=100)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l6bLYyqHow_a",
        "colab_type": "text"
      },
      "source": [
        "## **Train the model with unlabelled dataset**\n",
        "Train the model with unlabelled dataset and fine tune it by training the model for one epoch on labelled dataset. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TBv30SM7c7r9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "avg_main_loss = tf.keras.metrics.Mean(name='avg_main_loss', dtype=tf.float32)\n",
        "pseudo_steps = int(x_train_unlabelled.shape[0] / batch_size )\n",
        "eval_steps = int(x_eval_shorten.shape[0] / batch_size )\n",
        "\n",
        "epoch = 30\n",
        "\n",
        "classifier_optimizer = tf.keras.optimizers.Adam()\n",
        "loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
        "\n",
        "x_values = np.array([])\n",
        "threshold = 0.90\n",
        "\n",
        "for epoch_idx in range( epoch ):\n",
        "\n",
        "  print( \"Epoch {}/{}\".format( epoch_idx + 1, epoch ) )\n",
        "  classifier.evaluate(eval_dataset)\n",
        "\n",
        "  for _ in range(20):\n",
        "\n",
        "    pseudo_labels = classifier.predict( x_train_unlabelled ) \n",
        "\n",
        "    # only accept psuedo labels with certain threshold accuracy in their probability   \n",
        "    pseudo_labels = np.array( [ np.argmax(x) if np.max(x) > threshold else -1 for x in pseudo_labels])\n",
        "    dataset = np.array( [ [x,y] for x,y in zip( x_train_unlabelled, pseudo_labels ) if y != -1 ])\n",
        "    x_values = np.array( [x for x,y in dataset ])\n",
        "    y_values = np.array( [y for x,y in dataset ])\n",
        "\n",
        "    pseudo_dataset = tf.data.Dataset.from_tensor_slices((x_values, y_values ))\n",
        "    pseudo_dataset = pseudo_dataset.shuffle(1000).batch(batch_size)\n",
        "\n",
        "\n",
        "    classifier.fit(pseudo_dataset, batch_size=100, epochs=1, verbose=0)\n",
        "    classifier.fit(train_dataset, batch_size=100, epochs=1, verbose=0)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vdI-CSVFMGi-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}