fix_classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import pickle\n",
    "from IMDBModel import IMDBModel\n",
    "from embedding import Embedding\n",
    "from keras.preprocessing import sequence\n",
    "import time\n",
    "from glove_utils import load_embedding\n",
    "from data_utils import IMDBDataset\n",
    "from pprint import pprint\n",
    "from attacker import Attacker\n",
    "from explainers import SBE, LIMEExplainer\n",
    "from pos_taggers import TextBlobTagger, SpacyTagger\n",
    "from display_utils import html_render, display_html\n",
    "import utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Vectors for Attack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading vectors for attack...\n",
      "Loaded 173378 word vectors in 28.003089 seconds\n",
      "Loading counter-fitted vectors...\n",
      "Loaded 59975 word vectors in 7.931080 seconds\n"
     ]
    }
   ],
   "source": [
    "# Load main vectors\n",
    "print('Loading vectors for attack...')\n",
    "start_time = time.time()\n",
    "# EMBEDDING_FILENAME = 'data/glove.6B.300d.txt'\n",
    "EMBEDDING_FILENAME = 'data/fasttext300d.txt'\n",
    "word2index, index2word, index2embedding = load_embedding(EMBEDDING_FILENAME)\n",
    "print('Loaded %s word vectors in %f seconds' % (len(word2index), time.time() - start_time))\n",
    "embedding = Embedding(word2index, index2word, index2embedding)\n",
    "\n",
    "# Load counterfitted embeddings\n",
    "print('Loading counter-fitted vectors...')\n",
    "start_time = time.time()\n",
    "# COUNTERFITTED_EMBEDDING_NAME = 'data/counter-fitted-vectors-300.txt'\n",
    "COUNTERFITTED_EMBEDDING_FILENAME = 'data/fasttext-counter-fitted-vectors.txt'\n",
    "c_word2index, c_index2word, c_index2embedding = load_embedding(COUNTERFITTED_EMBEDDING_FILENAME)\n",
    "print('Loaded %s word vectors in %f seconds' % (len(c_word2index), time.time() - start_time))\n",
    "counter_embedding = Embedding(c_word2index, c_index2word, c_index2embedding)\n",
    "\n",
    "# create joined representation of original embedding with counterfitted vectors\n",
    "synonyms_embedding = Embedding.replace_embeddings(embedding, counter_embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load GLoVe vectors used by the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading GLoVe...\n",
      "Loaded 400002 word vectors in 16.188296 seconds\n"
     ]
    }
   ],
   "source": [
    "# Load main vectors\n",
    "print('Loading GLoVe...')\n",
    "start_time = time.time()\n",
    "MODEL_EMBEDDING_FILENAME = 'data/glove.6B.100d.txt'\n",
    "m_word2index, m_index2word, m_index2embedding = load_embedding(MODEL_EMBEDDING_FILENAME)\n",
    "print('Loaded %s word vectors in %f seconds' % (len(m_word2index), time.time()-start_time))\n",
    "model_embedding = Embedding(m_word2index, m_index2word, m_index2embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\User\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From C:\\Users\\User\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From C:\\Users\\User\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    }
   ],
   "source": [
    "# Load model\n",
    "imdb_model = IMDBModel('models/lstm_model.h5', model_embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load IMDB data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data...\n",
      "Data loaded.\n"
     ]
    }
   ],
   "source": [
    "# Load data\n",
    "maxlen = 200\n",
    "batch_size = 32\n",
    "print('Loading data...')\n",
    "(train_text, x_train, y_train), (test_text, x_test, y_test) = IMDBDataset.load_data()\n",
    "x_train = sequence.pad_sequences(x_train, maxlen=maxlen, padding = 'pre', truncating = 'pre')\n",
    "x_test = sequence.pad_sequences(x_test, maxlen=maxlen, padding = 'pre', truncating = 'pre')\n",
    "print('Data loaded.')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# predicted probabilities\n",
    "p_hat = imdb_model.model.predict(x_test, batch_size = 32).flatten()\n",
    "# predicted_ classes\n",
    "y_hat = np.where(p_hat >= 0.5, 1, 0)\n",
    "# indexes where the predictions were wrong\n",
    "wrong_indexes = np.where(y_hat != y_test)[0]\n",
    "# indexes where the prediction were correct\n",
    "correct_indexes = np.where(y_hat == y_test)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22082"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(correct_indexes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "confident_correct_indexes = [i for i in correct_indexes if abs(y_test[i]-p_hat[i])<=0.1 and y_test[i] == y_hat[i]]\n",
    "confident_correct_indexes = np.array(confident_correct_indexes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16790"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(confident_correct_indexes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9603212"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_hat[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use this when running the notebook remotely to redirect output to a file\n",
    "import sys\n",
    "jupyter_stdout = sys.stdout # save jupyter's stdout\n",
    "sys.stdout = open('/dev/stdout', 'w')\n",
    "print('this is printed in the console', flush = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create an Explainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "lime_explainer = LIMEExplainer(imdb_model, nsamples = 1000)\n",
    "sbe_explainer = SBE(imdb_model, m = 1000, SIGMA = 2/5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create a POS tagger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "spacy_tagger = SpacyTagger()\n",
    "textblob_tagger = TextBlobTagger()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create an Attacker object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Choice of hyperparameters\n",
    "explainer = None\n",
    "tagger = spacy_tagger\n",
    "percentage = 0.3  # explanation size and also maximum allowed percentage of words changed\n",
    "neighborhood_size = 30 # how many nearest neighbors to consider\n",
    "max_distance = 0.5 # OPTIONAL: the maximum allowed distance from a word to its neighbor\n",
    "beam_size = 4 # the beam size\n",
    "syn_dict_path = 'data/syn_dict/syn_dict_fasttext.pickle' # OPTIONAL: indicate file path of cached nearest neighbors dictionary\n",
    "dist_dict_path = 'data/syn_dict/dist_dict_fasttext.pickle'# OPTIONAL:indicate file path of cached distances to nearest neighbors\n",
    "attacker = Attacker(imdb_model, synonyms_embedding, explainer  , tagger, \n",
    "    percentage = percentage, neighborhood_size = neighborhood_size, max_distance = max_distance,\n",
    "    syn_dict_path = syn_dict_path,\n",
    "    dist_dict_path = dist_dict_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fix classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_handle = 'fix_fasttext_threshold_0.5_none'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "np.random.seed(23)\n",
    "sample_size = 500\n",
    "nwords_changed = []\n",
    "replacements = []\n",
    "original_texts = []\n",
    "adversarial_texts = []\n",
    "original_predictions = []\n",
    "adversarial_predictions = []\n",
    "sampled_indexes = np.random.choice(wrong_indexes, sample_size, replace = False)\n",
    "successes = sample_size*[0] # binary vector where 1 indicates success for sampled_indexes[i] else 0\n",
    "doc_lengths = [len(imdb_model.unpad_sequence(imdb_model.text2seq(test_text[wrong_index]))) for wrong_index in sampled_indexes]\n",
    "for (i, wrong_index) in enumerate(sampled_indexes):\n",
    "    print(\"#\",i,\" index = \", wrong_index, flush = True)\n",
    "    text = imdb_model.preprocess_text(test_text[wrong_index])\n",
    "    print(\"Predicted as: \",y_hat[wrong_index], \", True class: \", y_test[wrong_index])\n",
    "    original_prediction = imdb_model.predict(text)\n",
    "    original_predictions.append(original_prediction)\n",
    "    print(\"Original prediction: \", original_prediction)\n",
    "    original_texts.append(text)\n",
    "    target_class = 1 - y_hat[wrong_index]\n",
    "    suggestions = attacker.fix(text, target_class = 1 - y_hat[wrong_index], beam_size = beam_size, random_fix = False)\n",
    "    print(\"Suggestions: \")\n",
    "    pprint(suggestions)\n",
    "    replacements.append(suggestions)\n",
    "    if suggestions == [] :\n",
    "            print(\"Unable to change classification.\")\n",
    "            successes[i] = 0\n",
    "            nwords_changed.append(0)\n",
    "    else :\n",
    "            print(\"%f%% of words were changed.\" % (100* len(suggestions[0][0]) / doc_lengths[i]))\n",
    "            successes[i] = 1\n",
    "            nwords_changed.append(len(suggestions[0][0]))\n",
    "    if i % 20 == 0 :\n",
    "        print(\"Current success rate : \", sum(successes)/(i+1))\n",
    "    print(80*'-')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "success_rate = sum(successes) / sample_size\n",
    "print(\"Success rate = \", success_rate)\n",
    "percents_changed = [nchanges / doc_length for (nchanges,doc_length) in list(zip(nwords_changed,doc_lengths))]\n",
    "modification_rate = sum(percents_changed) / len(percents_changed)\n",
    "print(\"Average percent of words changed = \", modification_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_data = {\n",
    "    \"sampled_indexes\" : sampled_indexes,\n",
    "    \"original_texts\" : original_texts,\n",
    "    \"original_predictions\" : original_predictions,\n",
    "    \"replacements\" : replacements,\n",
    "    \"successes\" : successes,\n",
    "    \"nwords_changed\": nwords_changed,\n",
    "    \"doc_lengths\" : doc_lengths,\n",
    "    \"success_rate\" : success_rate,\n",
    "    \"modification_rate\": modification_rate,\n",
    "    \"percents_changed\" : percents_changed\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_name = 'results/'\n",
    "# Create directory if it does not exits\n",
    "if not os.path.exists(dir_name):\n",
    "    print(\"Created directory\")\n",
    "    os.mkdir(dir_name)\n",
    "# Pickle attack results\n",
    "f = open(dir_name + experiment_handle+'.pickle', 'wb')\n",
    "pickle.dump(results_data,f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Load results\n",
    "f = open(dir_name + experiment_handle+'.pickle', 'rb')\n",
    "results = pickle.load(f)\n",
    "f.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}