M2_5_Bonus_Tutorial.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/uvads6050/DS6050---24F/blob/main/M2_5_Bonus_Tutorial.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CjXAiZfEDWTR"
      },
      "source": [
        "# You might want to use the following packages\n",
        "import numpy as np\n",
        "from sklearn.model_selection import train_test_split\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "def make_xor(n_points):\n",
        "    centers = np.array([[0,0],[1,1]])\n",
        "    labels = np.array([0,1])\n",
        "    data = np.array([]).reshape(-1,3)\n",
        "    for center, label in zip(centers,labels):\n",
        "        points = np.random.normal(loc=center,scale=0.3,size=(n_points//4,2))\n",
        "        points_labels = np.hstack((points,label*np.ones(n_points//4).reshape((-1, 1))))\n",
        "        data = np.vstack((data,points_labels))\n",
        "    return (data[:,[0,1]],data[:,2])\n",
        "\n",
        "\n",
        "X, y = make_xor(1000)\n",
        "y=y.astype(np.int64)\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=49)\n",
        "print(X_train.shape)\n",
        "print(X_test.shape)\n",
        "plt.scatter(X[:,0], X[:,1], s=40, c=y, cmap=plt.cm.Spectral)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uC51CGJi1jep"
      },
      "source": [
        "$f = \\sigma(WX + b)$\n",
        "\n",
        "- $X$: data matrix of a dimension $d \\times N$\n",
        "- $W$, $b$: weight ($k \\times d$) and bias ($k \\times 1$) of the model (aka model parameters)\n",
        "- $\\sigma$: activation function (=step function in perceptron)\n",
        "\n",
        "\n",
        "**Goal:** find the optimal $W$ and $b$ that minimize\n",
        "\n",
        "$\\mathcal{L}_\\text{MSE} := \\mathbb{E}\\left( \\|f - y\\|^2 \\right)$\n",
        "\n",
        "**Strategy:**\n",
        "1.   Begin with some random initial values of $W$, $b$\n",
        "2.   Compute $\\frac{\\partial \\mathcal{L}}{\\partial W}$ and $\\frac{\\partial \\mathcal{L}}{\\partial b}$.\n",
        "3.   Update $W$, $b$ using gradient decent\n",
        "4.   Repeat 2~3 for a user-specified number of epochs\n",
        "\n",
        "**Gradients:**\n",
        "\n",
        "$\\frac{\\partial \\mathcal{L}}{\\partial W} = (f-y) X^\\top$\n",
        "\n",
        "$\\frac{\\partial \\mathcal{L}}{\\partial b} = (f-y)$  "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nAVsFWaWzVvg"
      },
      "source": [
        "from sklearn.base import BaseEstimator\n",
        "import numpy as np\n",
        "\n",
        "class MyPerceptron(BaseEstimator):\n",
        "  def __init__(self, d_in, d_out):\n",
        "    self.W = np.random.normal(loc=0, scale=0.01, size=(d_out, d_in))\n",
        "    self.b = np.random.normal(loc=0, scale=0.01, size=(d_out, 1))\n",
        "\n",
        "  def fit(self, X, y, epochs, learning_rate):\n",
        "    for i in range(epochs):\n",
        "      f = self.forward_pass(X)\n",
        "      dW, db = self.backward_pass(X, y, f)\n",
        "      self.W -= learning_rate*dW\n",
        "      self.b -= learning_rate*db\n",
        "\n",
        "      if i % 10 == 0:\n",
        "        print('Epoch %d/%d: loss %f - accuracy %f'\n",
        "              %(i, epochs, self.loss(y, f), self.evaluate(X, y)))\n",
        "\n",
        "  def forward_pass(self, X):\n",
        "    Z = np.matmul(self.W, X) + self.b\n",
        "    f = np.where(Z>=0, 1, 0)\n",
        "    return f\n",
        "\n",
        "  def backward_pass(self, X, y, f):\n",
        "    m = y.shape[0]\n",
        "    df = f - y\n",
        "    dW = np.dot(df, X.T)/m\n",
        "    db = np.sum(df, axis=1, keepdims=True)/m\n",
        "    return dW, db\n",
        "\n",
        "  def loss(self, y, f):\n",
        "    m = y.shape[0]\n",
        "    L = np.sum( (f-y)**2 )/m\n",
        "    return L\n",
        "\n",
        "  def predict(self, X):\n",
        "    f = self.forward_pass(X)\n",
        "    y_hat = np.squeeze(f)\n",
        "    return y_hat\n",
        "\n",
        "  def evaluate(self, X, y):\n",
        "    m = y.shape[0]\n",
        "    y_hat = self.predict(X)\n",
        "    correct_y = (y_hat == y).astype(int)\n",
        "\n",
        "    return sum(correct_y)/m"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cdGYfrlzDNng"
      },
      "source": [
        "model = MyPerceptron(2, 1)\n",
        "model.fit(X_train.T, y_train, epochs=50, learning_rate=0.01)\n",
        "print(\"Test Accuracy:\", model.evaluate(X_test.T, y_test))"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}