cache_nearest_neighbors.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "from IMDBModel import IMDBModel\n",
    "from embedding import Embedding\n",
    "from keras.preprocessing import sequence\n",
    "import time\n",
    "from glove_utils import load_embedding\n",
    "from data_utils import IMDBDataset\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use this when running the notebook remotely to redirect output\n",
    "import sys\n",
    "jupyter_stdout = sys.stdout # save jupyter's stdout\n",
    "sys.stdout = open('/dev/stdout', 'w')\n",
    "print('Printing in console', flush = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load GLoVe vectors\n",
    "print('Loading GLoVe vectors...')\n",
    "start_time = time.time()\n",
    "GLOVE_FILENAME = 'data/glove.6B.300d.txt'\n",
    "word2index, index2word, index2embedding = load_embedding(GLOVE_FILENAME)\n",
    "print('Loaded %s word vectors in %f seconds' % (len(word2index), time.time() - start_time))\n",
    "embedding = Embedding(word2index, index2word, index2embedding)\n",
    "\n",
    "# Load counterfitted embeddings\n",
    "print('Loading counter-fitted vectors...')\n",
    "start_time = time.time()\n",
    "COUNTERFITTED_GLOVE_FILENAME = 'data/counter-fitted-vectors-300.txt'\n",
    "c_word2index, c_index2word, c_index2embedding = load_embedding(COUNTERFITTED_GLOVE_FILENAME)\n",
    "print('Loaded %s word vectors in %f seconds' % (len(c_word2index), time.time() - start_time))\n",
    "counter_embedding = Embedding(c_word2index, c_index2word, c_index2embedding)\n",
    "\n",
    "# create joined representation of original GLoVe embedding with counterfitted vectors\n",
    "synonyms_embedding = Embedding.replace_embeddings(embedding, counter_embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load data\n",
    "maxlen = 400\n",
    "batch_size = 32\n",
    "print('Loading data...')\n",
    "(train_text, x_train, y_train), (test_text, x_test, y_test) = IMDBDataset.load_data()\n",
    "x_train = sequence.pad_sequences(x_train, maxlen=maxlen, padding = 'pre', truncating = 'pre')\n",
    "x_test = sequence.pad_sequences(x_test, maxlen=maxlen, padding = 'pre', truncating = 'pre')\n",
    "print('Data loaded.')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build Frequency table\n",
    "from collections import Counter\n",
    "word_indexes_freqs = Counter()\n",
    "for i in range(len(x_train)):\n",
    "    word_indexes_freqs+= Counter(x_train[i])\n",
    "for i in range(len(x_test)):\n",
    "    word_indexes_freqs+= Counter(x_test[i])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(word_indexes_freqs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_common_indexes = word_indexes_freqs.most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_common_words = [(embedding.index2word[index], freq) for (index, freq) in most_common_indexes]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_common_words[0:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# location to save synonyms dict \n",
    "import pickle\n",
    "prefix = 'data/syn_dict/'\n",
    "if not os.path.exists(prefix):\n",
    "    print('Creating directory ',prefix)\n",
    "    os.mkdir(prefix)\n",
    "\n",
    "handle = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cache nearest neighbors for 25 words each time\n",
    "batch_size = 25\n",
    "synonyms_dict = dict()\n",
    "distances_dict = dict()\n",
    "for i in range(0, len(most_common_words), batch_size):\n",
    "    common_words = most_common_words[i:i+batch_size]\n",
    "    words = [word for (word,_) in common_words]\n",
    "    print(\"Getting the nearest neighbors for the following words: \", words, flush = True)\n",
    "    start_time = time.time()\n",
    "    synonyms_map, distances_map = synonyms_embedding.build_neighbors_map(words, N = 30, return_distances = True)\n",
    "    synonyms_dict = {**synonyms_dict, **synonyms_map}\n",
    "    distances_dict = {**distances_dict, **distances_map}\n",
    "    print(\"Built synonyms_dict in \", time.time() - start_time, \" seconds\" , flush = True)\n",
    "    print(\"Saving synonyms_dict_%d\" % (i//batch_size), flush = True)\n",
    "    print(40*'-')\n",
    "    syn_file = open(prefix+'syn_dict_'+handle+'.pickle', 'wb')\n",
    "    pickle.dump(synonyms_dict, syn_file)\n",
    "    syn_file.close()\n",
    "    dist_file = open(prefix + 'dist_dict_'+handle+'.pickle', 'wb')\n",
    "    pickle.dump(distances_dict, dist_file)\n",
    "    dist_file.close"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Finished\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "infile = open(prefix + 'syn_dict_'+handle+ '.pickle','rb')\n",
    "new_dict = pickle.load(infile)\n",
    "infile.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pprint(new_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "infile = open(prefix + 'dist_dict_'+handle+'.pickle','rb')\n",
    "dist_dict = pickle.load(infile)\n",
    "infile.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pprint(dist_dict)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}