pain_tf.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Poisson Autoencoder Inverting Network (PAIN)\n",
    "Uses the Tensoflow machine learning library to build and train the Poisson autoencoder inverting network (PAIN) to reconstruct compressed ***MNIST*** images with Poisson noise.\n",
    "\n",
    "**Author:** Fabian Santiago  \n",
    "**Update:** August 21, 2024\n",
    "\n",
    "**Implementation:** PAIN architecture presented in:\n",
    "**[IEEE publication](https://ieeexplore.ieee.org/abstract/document/8682767)**\n",
    "\n",
    "***NOTE:*** *Code is setup to reconstruct 7x7 compressed noisy images. Batch size and learning rates may need to be adjusted for 4x4, 14x14, and 28x28 cases.*\n",
    "\n",
    "Python Version: 3.11.5  \n",
    "Jupyer Notebook Version: 6.5.4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Modules and Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import necessary modules and libraries \n",
    "from __future__ import division, print_function, absolute_import\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Model\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import h5py\n",
    "import os\n",
    "from skimage.util.shape import view_as_windows\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.decomposition import PCA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set Values: Seed, Compression, and Output Dimension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the seed for TensorFlow and Keras\n",
    "tf.random.set_seed(101)\n",
    "\n",
    "# Dimension of the compressed/noisy images (width=height)  \n",
    "# cmp_dim = 4:(for 4x4), 7:(7x7), 14:(14x14), or 28:(28x284)\n",
    "cmp_dim = 7\n",
    "\n",
    "# Dimension of output, original are 28 x 28\n",
    "out_dim  = 28 \n",
    "\n",
    "# Number of training epochs\n",
    "num_epochs = 25\n",
    "\n",
    "# Size of training batch sizes\n",
    "batch_size = 250"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper Function Definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define sliding window compression\n",
    "def median_downsampling(in_image, cmp_dim):\n",
    "    if cmp_dim < 15:\n",
    "        # Compression dimension to window size\n",
    "        window_size = 28//cmp_dim\n",
    "        # Create sliding windows\n",
    "        windows = view_as_windows(in_image, (window_size, window_size), step=window_size)\n",
    "        # Calculate the median over each window\n",
    "        return np.median(windows, axis=(2, 3))\n",
    "    else:\n",
    "        return in_image\n",
    "\n",
    "# Define compression of entries in an array\n",
    "def down_sample_list(in_array,cmp_dim):\n",
    "    out_array = np.empty((len(in_array), cmp_dim, cmp_dim))\n",
    "    \n",
    "    # Iterate over array\n",
    "    for idx, image in enumerate(in_array):\n",
    "        out_array[idx] = median_downsampling(image,cmp_dim)\n",
    "        \n",
    "    # Return array containing compressed entries \n",
    "    return out_array \n",
    "\n",
    "# Split dataset into two\n",
    "def split_dataset_rnd(in_images):\n",
    "\n",
    "    # Total number of images\n",
    "    num_images = in_images.shape[0]\n",
    "    \n",
    "    # Random permutation of indices\n",
    "    shuffled_indices = np.random.permutation(num_images)\n",
    "    \n",
    "    # Calculate the split index\n",
    "    split_index = num_images // 2\n",
    "    \n",
    "    # Split the shuffled indices\n",
    "    idx1 = shuffled_indices[:split_index]\n",
    "    idx2 = shuffled_indices[split_index:]\n",
    "    \n",
    "    # Use the indices to create two random datasets\n",
    "    dat1 = in_images[idx1]\n",
    "    dat2 = in_images[idx2]\n",
    "    \n",
    "    return dat1, dat2\n",
    "\n",
    "# Create compressed noisy data\n",
    "def create_training_test_data(clean_train, clean_test, cmp_dim, out_dim):\n",
    "    \n",
    "    # Compress images using median in sliding window\n",
    "    cmp_train = down_sample_list(clean_train, cmp_dim)\n",
    "    cmp_test  = down_sample_list(clean_test, cmp_dim)\n",
    "\n",
    "    # Add Poisson noise to compressed images\n",
    "    noisy_train_ = np.random.poisson(lam=cmp_train)\n",
    "    noisy_test_  = np.random.poisson(lam=cmp_test)\n",
    "    \n",
    "    # Scale to [0,255]\n",
    "    noisy_train  = np.clip(noisy_train_,0,255)\n",
    "    noisy_test   = np.clip(noisy_test_,0,255)\n",
    "    \n",
    "    # Reshape Compressed/Noisy Arrays and scale to [0,1]\n",
    "    noisy_train = np.array([matrix.reshape(cmp_dim**2,) for matrix in noisy_train/255])\n",
    "    noisy_test  = np.array([matrix.reshape(cmp_dim**2,) for matrix in noisy_test/255])\n",
    "    \n",
    "    # Return training data\n",
    "    return noisy_train, noisy_test\n",
    "\n",
    "# PAIN architecture builder function\n",
    "def build_PAIN(in_dim, out_dim, enc_dim = 256):\n",
    "    # Input layer\n",
    "    inputs = tf.keras.Input(shape=(in_dim**2,))\n",
    "\n",
    "    # First Decoder\n",
    "    Dec1 = tf.keras.layers.Dense(out_dim**2, activation='sigmoid')(inputs)\n",
    "\n",
    "    # Encoder\n",
    "    Enc2_hidden = tf.keras.layers.Dense(enc_dim, activation='sigmoid')(Dec1)\n",
    "    Enc2 = tf.keras.layers.Dense(in_dim**2, activation='sigmoid')(Enc2_hidden)\n",
    "\n",
    "    # Second Decoder\n",
    "    Dec2_hidden = tf.keras.layers.Dense(enc_dim, activation='sigmoid')(Enc2)\n",
    "    outputs = tf.keras.layers.Dense(out_dim**2, activation='sigmoid')(Dec2_hidden)\n",
    "\n",
    "    # Define the model\n",
    "    model = tf.keras.Model(inputs=inputs, outputs=outputs)\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load/Create Training Data\n",
    "Load traning data or load MNIST data and create training data if it has not already been created"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check compression dimention variable:\n",
    "if not (cmp_dim in [4, 7, 14, 28]):\n",
    "    raise ValueError(f'cmp_dim = {cmp_dim}, not handled. The cmp_dim value must be 4, 7, 14, or 28.')\n",
    "\n",
    "directory = 'training_data'\n",
    "\n",
    "# If the directory does not exist, create it\n",
    "if not os.path.exists(directory):\n",
    "    os.makedirs(directory)\n",
    "\n",
    "# Load the MNIST Dataset\n",
    "dat_file = f'{directory}/mnist_{cmp_dim}x{cmp_dim}_train.h5'\n",
    "    \n",
    "# Load MNIST\n",
    "(clean_train, train_labels), (clean_test, test_labels) = tf.keras.datasets.mnist.load_data()\n",
    "\n",
    "# If the compressed/noisy data does not exist, create it\n",
    "if not os.path.exists(dat_file):\n",
    "    # Create compressed/noisy training/test data\n",
    "    noisy_train, noisy_test  = create_training_test_data(clean_train, clean_test, cmp_dim, out_dim)\n",
    "        \n",
    "    # Save compressed/noisy training/test data\n",
    "    with h5py.File(dat_file, 'w') as f:\n",
    "        # Save compressed noisy images\n",
    "        f.create_dataset('noisy_train', data=noisy_train, compression='gzip')\n",
    "        f.create_dataset('noisy_test', data=noisy_test, compression='gzip')\n",
    "        \n",
    "else: \n",
    "    # Load compressed/noisy training data\n",
    "    with h5py.File(dat_file, 'r') as dat_file:\n",
    "        # Load compressed noisy training/test images\n",
    "        noisy_train = dat_file['noisy_train'][:]\n",
    "        noisy_test  = dat_file['noisy_test'][:]\n",
    "        \n",
    "# Prepare original mnist data for model training\n",
    "clean_train = np.array([matrix.reshape(out_dim**2,) for matrix in clean_train/255])\n",
    "clean_test  = np.array([matrix.reshape(out_dim**2,) for matrix in clean_test/255])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Display Training Dataset Examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a 2 by 4 subplot handle\n",
    "fig, axes = plt.subplots(2, 4, figsize=(9, 4.5))\n",
    "axes      = axes.flatten()\n",
    "\n",
    "# Add title\n",
    "fig.suptitle('Training Set Examples', fontsize=20, fontweight='bold', fontfamily='serif')\n",
    "\n",
    "# Shift window through training dataset \n",
    "sft_idx = 0\n",
    "\n",
    "# Plot noisy and decompressed realizations in each subplot\n",
    "for idx in range(4):\n",
    "    # Plot noisy\n",
    "    axes[idx].imshow(noisy_train[idx+sft_idx].reshape(cmp_dim,cmp_dim),cmap='gray')\n",
    "    axes[idx].set_xticks([]) # Remove xticklabels\n",
    "    axes[idx].set_yticks([]) # Remove yticklabels\n",
    "    \n",
    "    # Plot original\n",
    "    axes[idx+4].imshow(clean_train[idx+sft_idx].reshape(out_dim,out_dim),cmap='gray')\n",
    "    axes[idx+4].set_xticks([]) # Remove xticklabels\n",
    "    axes[idx+4].set_yticks([]) # Remove yticklabels \n",
    "    \n",
    "# Set ylables \n",
    "axes[0].set_ylabel(f'Input\\n{cmp_dim} x {cmp_dim}\\nCompressed & Noisy',fontdict={'fontsize': 12, 'fontfamily': 'serif', 'color':'blue'})\n",
    "axes[4].set_ylabel(f'Original\\n{out_dim} x {out_dim}\\nMNIST',fontdict={'fontsize': 12, 'fontfamily': 'serif'})\n",
    "\n",
    "# Adjust layout to decrease padding between subplots\n",
    "plt.subplots_adjust(wspace=0.1, hspace=0.1)\n",
    "\n",
    "# Display the figure\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compile & Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Create & Compile the PAIN model\n",
    "PAIN = build_PAIN(in_dim=cmp_dim, out_dim=out_dim)\n",
    "\n",
    "# Create an optimizer with a specific learning rate\n",
    "tf_opt = tf.keras.optimizers.Adam(learning_rate=0.01)\n",
    "\n",
    "# Compile the model\n",
    "PAIN.compile(optimizer=tf_opt, loss='mean_squared_error')\n",
    "\n",
    "# Train model and saving fitting history\n",
    "fit_history = PAIN.fit(noisy_train, clean_train, epochs=num_epochs, batch_size=batch_size, validation_split=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PAIN.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Apply PAIN to Training Set and Display Example Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save output\n",
    "imdir = 'method_results'\n",
    "\n",
    "# If the save directory does not exist, create it\n",
    "if not os.path.exists(imdir):\n",
    "    os.makedirs(imdir)\n",
    "\n",
    "# Save file name\n",
    "imfile = f'mnist_reconstruct'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_train_out = PAIN.predict(noisy_train[0:101])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a 3 by 4 subplot handle\n",
    "fig, axes = plt.subplots(3, 4, figsize=(9, 8))\n",
    "axes      = axes.flatten()\n",
    "\n",
    "# Add title\n",
    "fig.suptitle('Application of PAIN Architecture\\n(MNIST Training Set)',fontsize=20,fontweight='bold', fontfamily='serif')\n",
    "\n",
    "# Shift window through training dataset \n",
    "sft_idx = 0\n",
    "\n",
    "# Plot noisy and decompressed realizations in each subplot\n",
    "for idx in range(4):\n",
    "    # Plot noisy\n",
    "    axes[idx].imshow(noisy_train[idx+sft_idx].reshape(cmp_dim,cmp_dim),cmap='gray')\n",
    "    axes[idx].set_xticks([]) # Remove xticklabels\n",
    "    axes[idx].set_yticks([]) # Remove yticklabels\n",
    "    axes[idx].set_xlabel('⇩',fontdict={'fontsize': 25, 'fontweight': 'bold', 'fontfamily': 'serif', 'color':'blue'})\n",
    "    \n",
    "    # Plot decompressed with PAIN\n",
    "    axes[idx+4].imshow(pred_train_out[idx+sft_idx].reshape(out_dim,out_dim),cmap='gray')\n",
    "    axes[idx+4].set_xticks([]) # Remove xticklabels\n",
    "    axes[idx+4].set_yticks([]) # Remove yticklabels\n",
    "    \n",
    "    # Plot original\n",
    "    axes[idx+8].imshow(clean_train[idx+sft_idx].reshape(out_dim,out_dim),cmap='gray')\n",
    "    axes[idx+8].set_xticks([]) # Remove xticklabels\n",
    "    axes[idx+8].set_yticks([]) # Remove yticklabels\n",
    "    \n",
    "# Set ylables \n",
    "axes[0].set_ylabel(f'Input\\n{cmp_dim} x {cmp_dim}\\nCompressed & Noisy',fontdict={'fontsize': 12, 'fontfamily': 'serif', 'color':'blue'})\n",
    "axes[4].set_ylabel(f'Output\\n{out_dim} x {out_dim}\\nPAIN Reconstruction',fontdict={'fontsize': 12, 'fontfamily': 'serif', 'color':'blue'})\n",
    "axes[8].set_ylabel(f'Original\\n{out_dim} x {out_dim}\\nMNIST',fontdict={'fontsize': 12, 'fontfamily': 'serif'})\n",
    "\n",
    "# Adjust layout to decrease padding between subplots\n",
    "plt.subplots_adjust(wspace=0.1, hspace=0)\n",
    "\n",
    "# Save results \n",
    "plt.savefig(f'{imdir}/{imfile}_{cmp_dim}x{cmp_dim}_train_PAIN.png')\n",
    "\n",
    "# Display the figure\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Apply PAIN to Test Set and Display Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply PAIN to all test data inputs\n",
    "pred_test_out = PAIN.predict(noisy_test[0:101])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a 3 by 4 subplot handle\n",
    "fig, axes = plt.subplots(3, 4, figsize=(10, 8))\n",
    "axes      = axes.flatten()\n",
    "\n",
    "# Add title\n",
    "fig.suptitle('Application of PAIN Architecture\\n(MNIST Test Set)',fontsize=20,fontweight='bold', fontfamily='serif')\n",
    "\n",
    "# Shift window through test dataset \n",
    "sft_idx = 0\n",
    "\n",
    "# Plot noisy and decompressed realizations in each subplot\n",
    "for idx in range(4):\n",
    "    # Plot noisy\n",
    "    axes[idx].imshow(noisy_test[idx+sft_idx].reshape(cmp_dim,cmp_dim),cmap='gray')\n",
    "    axes[idx].set_xticks([]) # Remove xticklabels\n",
    "    axes[idx].set_yticks([]) # Remove yticklabels\n",
    "    axes[idx].set_xlabel('⇩',fontdict={'fontsize': 25, 'fontweight': 'bold', 'fontfamily': 'serif', 'color':'blue'})\n",
    "    \n",
    "    # Plot decompressed with PAIN\n",
    "    axes[idx+4].imshow(pred_test_out[idx+sft_idx].reshape(28,28),cmap='gray')\n",
    "    axes[idx+4].set_xticks([]) # Remove xticklabels\n",
    "    axes[idx+4].set_yticks([]) # Remove yticklabels\n",
    "    \n",
    "    # Plot original\n",
    "    axes[idx+8].imshow(clean_test[idx+sft_idx].reshape(28,28),cmap='gray')\n",
    "    axes[idx+8].set_xticks([]) # Remove xticklabels\n",
    "    axes[idx+8].set_yticks([]) # Remove yticklabels\n",
    "    \n",
    "# Set ylables \n",
    "axes[0].set_ylabel(f'(Input)\\n{cmp_dim} x {cmp_dim}\\nCompressed & Noisy',fontdict={'fontsize': 12, 'fontfamily': 'serif', 'color':'blue'})\n",
    "axes[4].set_ylabel(f'(Output)\\n{out_dim} x {out_dim}\\nPAIN Reconstruction',fontdict={'fontsize': 12, 'fontfamily': 'serif', 'color':'blue'})\n",
    "axes[8].set_ylabel(f'(Original)\\n{out_dim} x {out_dim}\\nMNIST',fontdict={'fontsize': 12, 'fontfamily': 'serif'})\n",
    "\n",
    "# Adjust layout to decrease padding between subplots\n",
    "plt.subplots_adjust(wspace=0.1, hspace=0)\n",
    "\n",
    "# Save results \n",
    "plt.savefig(f'{imdir}/{imfile}_{cmp_dim}x{cmp_dim}_test_PAIN.png')\n",
    "\n",
    "# Display the figure\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "@webio": {
   "lastCommId": null,
   "lastKernelId": null
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}