train_phone_ru.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#!/usr/bin/python\n",
    "\n",
    "from tensorflow.contrib.learn.python.learn.datasets import base\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os,sys\n",
    "sys.path.insert(0, './scripts')\n",
    "import py_compile\n",
    "py_compile.compile('scripts/ivector_tools.py')\n",
    "py_compile.compile('scripts/siamese_model_phone_ru.py')\n",
    "import ivector_dataset\n",
    "import siamese_model_phone_ru as siamese_model\n",
    "import ivector_tools as it\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_dataset_size(dict_file,feat_file):\n",
    "# Counting feature dimension and total number of utterances\n",
    "    f = open(dict_file)\n",
    "    dict_dim = 0\n",
    "    for line in f:\n",
    "        dict_dim+=1\n",
    "    f.close()\n",
    "    feat_len = 0\n",
    "    f = open(feat_file)\n",
    "    for line in f:\n",
    "        feat_len+=1\n",
    "    f.close()\n",
    "    return dict_dim, feat_len\n",
    "\n",
    "def get_feat_label(dict_file, feat_file):\n",
    "# Get feature vectors from files\n",
    "    dict_dim, feat_len = get_dataset_size(dict_file,feat_file)\n",
    "    features = np.zeros((feat_len,dict_dim),dtype='float32')\n",
    "    labels = np.zeros((feat_len),dtype='int8')\n",
    "    names = []\n",
    "    f = open(feat_file)\n",
    "    count = 0\n",
    "    for line in f:\n",
    "        names.append(line.split()[0])\n",
    "        labels[count] = line.split()[1]\n",
    "        line= line.split()[2:]\n",
    "        for iter in range(0,len(line)):\n",
    "            elements = line[iter].split(':')\n",
    "            features[count][ int( elements[0] ) -1 ] = elements[1]\n",
    "        count = count + 1 \n",
    "    f.close()\n",
    "    \n",
    "    return features, labels, names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13825, 51102) (1524, 51102) (1492, 51102)\n"
     ]
    }
   ],
   "source": [
    "context = 3\n",
    "dict_file = 'data/train.vardial2017/dict.phone_ru.c'+str(context)\n",
    "feat_file = 'data/train.vardial2017/phone_ru.c'+str(context)\n",
    "trn_features, trn_labels, trn_names = get_feat_label(dict_file,feat_file)\n",
    "\n",
    "feat_file = 'data/dev.vardial2017/phone_ru.c'+str(context)\n",
    "dev_features, dev_labels, dev_names = get_feat_label(dict_file,feat_file)\n",
    "\n",
    "feat_file = 'data/test.MGB3/phone_ru.c'+str(context)\n",
    "tst_features, tst_labels, tst_names = get_feat_label(dict_file,feat_file)\n",
    "\n",
    "print trn_features.shape, dev_features.shape, tst_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "languages = ['EGY','GLF','LAV','MSA','NOR']\n",
    "\n",
    "\n",
    "\n",
    "# load tst.MGB3 labels\n",
    "filename = 'data/test.MGB3/reference'\n",
    "tst_ref_names = np.loadtxt(filename,usecols=[0],dtype='string')\n",
    "tst_ref_labels = np.loadtxt(filename,usecols=[1],dtype='int')\n",
    "\n",
    "tst_labels_index = []\n",
    "for i,name in enumerate(tst_names):\n",
    "    for j, name_ref in enumerate(tst_ref_names):\n",
    "        if name == name_ref:\n",
    "            tst_labels_index = np.append(tst_labels_index,j)\n",
    "\n",
    "tst_labels = np.empty((np.size(tst_labels_index)))\n",
    "for i,j in enumerate(tst_labels_index):\n",
    "    tst_labels[i]=tst_ref_labels[int(j)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((13825, 51102), (1524, 51102), (5, 51102), (1492, 51102))\n"
     ]
    }
   ],
   "source": [
    "#language modeling\n",
    "lang_mean=[]\n",
    "for i, lang in enumerate(languages):\n",
    "     lang_mean.append(np.mean(np.append(trn_features[np.nonzero(trn_labels == i+1)] ,dev_features[np.nonzero(dev_labels == i+1)],axis=0),axis=0))\n",
    "\n",
    "lang_mean = np.array(lang_mean)\n",
    "lang_mean = it.length_norm(lang_mean)\n",
    "\n",
    "print( np.shape(trn_features), np.shape(dev_features), np.shape(lang_mean),np.shape(tst_features) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline accurary on test dataset : 0.368\n",
      "Baseline accurary on dev dataset : 0.386\n"
     ]
    }
   ],
   "source": [
    "# Baseline performance on TST using CDS\n",
    "tst_scores = lang_mean.dot(tst_features.transpose() )\n",
    "# print(tst_scores.shape)\n",
    "hypo_lang = np.argmax(tst_scores,axis = 0)\n",
    "temp = ((tst_labels-1) - hypo_lang)\n",
    "acc =1- np.size(np.nonzero(temp)) / float(np.size(tst_labels))\n",
    "print 'Baseline accurary on test dataset : %0.3f' %(acc)\n",
    "\n",
    "# Baseline performance on DEV using CDS\n",
    "dev_scores = lang_mean.dot(dev_features.transpose() )\n",
    "hypo_lang = np.argmax(dev_scores,axis = 0)\n",
    "temp = ((dev_labels-1) - hypo_lang)\n",
    "acc =1- np.size(np.nonzero(temp)) / float(np.size(dev_labels))\n",
    "print 'Baseline accurary on dev dataset : %0.3f' %(acc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((69125,), (69125,), (69125,))\n",
      "((13825, 51102), (13825, 51102), (13825,), (55300, 51102), (55300, 51102), (55300,))\n"
     ]
    }
   ],
   "source": [
    "# making pair of train i-vector with mean of each language i-vector\n",
    "#  example : for total 3 ivectors\n",
    "#  ivector   ivector_p  label\n",
    "#     1         1         1\n",
    "#     1         2         0\n",
    "#     1         3         0\n",
    "#     2         1         0\n",
    "#     2         2         1\n",
    "#     ...      ...       ...\n",
    "#     3         3         1\n",
    "\n",
    "# preparing pair labels\n",
    "sim = []\n",
    "pair_a_idx = []\n",
    "pair_b_idx = []\n",
    "for i, lang in enumerate(languages):\n",
    "    for j, label in enumerate(trn_labels):\n",
    "#         print i, j, label\n",
    "        pair_a_idx.append(i+1)\n",
    "        pair_b_idx.append(j)\n",
    "        if i+1 == label:\n",
    "            sim.append(1)\n",
    "        else:\n",
    "            sim.append(0)\n",
    "print(np.shape(pair_a_idx),np.shape(pair_b_idx), np.shape(sim))\n",
    "pair_a_idx=np.array(pair_a_idx)\n",
    "pair_b_idx=np.array(pair_b_idx)\n",
    "sim = np.array(sim)\n",
    "\n",
    "#shuffling\n",
    "shuffleidx = np.arange(0,np.size(pair_a_idx))\n",
    "np.random.shuffle(shuffleidx)\n",
    "pair_a_idx = pair_a_idx[shuffleidx]\n",
    "pair_b_idx = pair_b_idx[shuffleidx]\n",
    "sim = sim[shuffleidx]\n",
    "\n",
    "\n",
    "data = []\n",
    "data_p = []\n",
    "    \n",
    "for iter in np.arange(0,np.size(sim)) :\n",
    "    data.append( lang_mean[pair_a_idx[iter]-1] )\n",
    "    data_p.append( trn_features[pair_b_idx[iter]] )\n",
    "data = np.array(data)\n",
    "data_p = np.array(data_p)\n",
    "\n",
    "# TRN dataset loading for feeding \n",
    "tar_data = data[sim==1]\n",
    "tar_data_p = data_p[sim==1]\n",
    "tar_sim = sim[sim==1]\n",
    "non_data = data[sim==0]\n",
    "non_data_p = data_p[sim==0]\n",
    "non_sim = sim[sim==0]\n",
    "print(tar_data.shape, tar_data_p.shape,tar_sim.shape,non_data.shape,non_data_p.shape,non_sim.shape)\n",
    "\n",
    "trn_tar = ivector_dataset.DataSet(tar_data,tar_sim)\n",
    "trn_tar_p = ivector_dataset.DataSet(tar_data_p,tar_sim)\n",
    "\n",
    "trn_non = ivector_dataset.DataSet(non_data,non_sim)\n",
    "trn_non_p = ivector_dataset.DataSet(non_data_p,non_sim)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((7620,), (7620,), (7620,))\n",
      "((1524, 51102), (1524, 51102), (1524,), (6096, 51102), (6096, 51102), (6096,))\n"
     ]
    }
   ],
   "source": [
    "# preparing pair labels of DEV dataset\n",
    "sim = []\n",
    "pair_a_idx = []\n",
    "pair_b_idx = []\n",
    "for i, lang in enumerate(languages):\n",
    "    for j, label in enumerate(dev_labels):\n",
    "#         print i, j, label\n",
    "        pair_a_idx.append(i+1)\n",
    "        pair_b_idx.append(j)\n",
    "        if i+1 == label:\n",
    "            sim.append(1)\n",
    "        else:\n",
    "            sim.append(0)\n",
    "print(np.shape(pair_a_idx),np.shape(pair_b_idx), np.shape(sim))\n",
    "pair_a_idx=np.array(pair_a_idx)\n",
    "pair_b_idx=np.array(pair_b_idx)\n",
    "sim = np.array(sim)\n",
    "\n",
    "#shuffling\n",
    "shuffleidx = np.arange(0,np.size(pair_a_idx))\n",
    "np.random.shuffle(shuffleidx)\n",
    "pair_a_idx = pair_a_idx[shuffleidx]\n",
    "pair_b_idx = pair_b_idx[shuffleidx]\n",
    "sim = sim[shuffleidx]\n",
    "\n",
    "\n",
    "data = []\n",
    "data_p = []\n",
    "    \n",
    "for iter in np.arange(0,np.size(sim)) :\n",
    "    data.append( lang_mean[pair_a_idx[iter]-1] )\n",
    "    data_p.append( dev_features[pair_b_idx[iter]] )\n",
    "data = np.array(data)\n",
    "data_p = np.array(data_p)\n",
    "\n",
    "# DEV dataset loading for feeding \n",
    "tar_data = data[sim==1]\n",
    "tar_data_p = data_p[sim==1]\n",
    "tar_sim = sim[sim==1]\n",
    "non_data = data[sim==0]\n",
    "non_data_p = data_p[sim==0]\n",
    "non_sim = sim[sim==0]\n",
    "print(tar_data.shape, tar_data_p.shape,tar_sim.shape,non_data.shape,non_data_p.shape,non_sim.shape)\n",
    "\n",
    "dev_tar = ivector_dataset.DataSet(tar_data,tar_sim)\n",
    "dev_tar_p = ivector_dataset.DataSet(tar_data_p,tar_sim)\n",
    "\n",
    "dev_non = ivector_dataset.DataSet(non_data,non_sim)\n",
    "dev_non_p = ivector_dataset.DataSet(non_data_p,non_sim)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(?, 92920)\n",
      "(?, 1500)\n",
      "(?, 600)\n",
      "(?, 92920)\n",
      "(?, 1500)\n",
      "(?, 600)\n"
     ]
    }
   ],
   "source": [
    "# init variables\n",
    "sess = tf.InteractiveSession()\n",
    "siamese = siamese_model.siamese(np.shape(trn_features)[1]);\n",
    "global_step = tf.Variable(0, trainable=False)\n",
    "learning_rate = tf.train.exponential_decay(0.01, global_step,\n",
    "                                           5000, 0.99, staircase=True)\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(siamese.loss, global_step=global_step)\n",
    "saver = tf.train.Saver()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 0: loss 1.000433, Acc.: (DEV)0.287 (TST)0.299, lr : 0.01000\n",
      "Step 3300: loss 1.000023, Acc.: (DEV)0.312 (TST)0.303, lr : 0.01000\n",
      "Step 3400: loss 0.999996, Acc.: (DEV)0.320 (TST)0.305, lr : 0.01000\n",
      "Step 3500: loss 1.000026, Acc.: (DEV)0.320 (TST)0.306, lr : 0.01000\n",
      "Step 3700: loss 0.999985, Acc.: (DEV)0.316 (TST)0.307, lr : 0.01000\n",
      "Step 3800: loss 1.000103, Acc.: (DEV)0.318 (TST)0.308, lr : 0.01000\n",
      "Step 5100: loss 0.999928, Acc.: (DEV)0.316 (TST)0.312, lr : 0.00990\n",
      "Step 5300: loss 0.999905, Acc.: (DEV)0.320 (TST)0.314, lr : 0.00990\n",
      "Step 5400: loss 1.000028, Acc.: (DEV)0.332 (TST)0.316, lr : 0.00990\n",
      "Step 8800: loss 0.999512, Acc.: (DEV)0.329 (TST)0.317, lr : 0.00990\n",
      "Step 9600: loss 0.999934, Acc.: (DEV)0.329 (TST)0.318, lr : 0.00990\n",
      "Step 11100: loss 0.999912, Acc.: (DEV)0.329 (TST)0.320, lr : 0.00980\n",
      "Step 22700: loss 1.000817, Acc.: (DEV)0.333 (TST)0.325, lr : 0.00961\n",
      "Step 23000: loss 0.985440, Acc.: (DEV)0.333 (TST)0.326, lr : 0.00961\n",
      "Step 23500: loss 0.984918, Acc.: (DEV)0.364 (TST)0.348, lr : 0.00961\n",
      "Step 23600: loss 1.002515, Acc.: (DEV)0.383 (TST)0.367, lr : 0.00961\n",
      "Step 35600: loss 0.952540, Acc.: (DEV)0.431 (TST)0.372, lr : 0.00932\n",
      "Step 36100: loss 0.968409, Acc.: (DEV)0.439 (TST)0.378, lr : 0.00932\n",
      "Step 36400: loss 0.959635, Acc.: (DEV)0.430 (TST)0.380, lr : 0.00932\n",
      "Step 36500: loss 0.975117, Acc.: (DEV)0.438 (TST)0.384, lr : 0.00932\n",
      "Step 36800: loss 0.982347, Acc.: (DEV)0.438 (TST)0.387, lr : 0.00932\n",
      "Step 38600: loss 0.956199, Acc.: (DEV)0.451 (TST)0.389, lr : 0.00932\n",
      "Step 39300: loss 0.956118, Acc.: (DEV)0.458 (TST)0.391, lr : 0.00932\n",
      "Step 39800: loss 0.973996, Acc.: (DEV)0.463 (TST)0.393, lr : 0.00932\n",
      "Step 39900: loss 1.000715, Acc.: (DEV)0.476 (TST)0.398, lr : 0.00932\n",
      "Step 40100: loss 0.968138, Acc.: (DEV)0.465 (TST)0.399, lr : 0.00923\n",
      "Step 40200: loss 0.920945, Acc.: (DEV)0.475 (TST)0.403, lr : 0.00923\n",
      "Step 41200: loss 0.945677, Acc.: (DEV)0.490 (TST)0.412, lr : 0.00923\n",
      "Step 41800: loss 0.994009, Acc.: (DEV)0.496 (TST)0.413, lr : 0.00923\n",
      "Step 41900: loss 0.939014, Acc.: (DEV)0.495 (TST)0.416, lr : 0.00923\n",
      "Step 42400: loss 0.958933, Acc.: (DEV)0.512 (TST)0.418, lr : 0.00923\n",
      "Step 43200: loss 0.887068, Acc.: (DEV)0.518 (TST)0.420, lr : 0.00923\n",
      "Step 43600: loss 0.930033, Acc.: (DEV)0.522 (TST)0.422, lr : 0.00923\n",
      "Step 43700: loss 0.994143, Acc.: (DEV)0.527 (TST)0.422, lr : 0.00923\n",
      "Step 43900: loss 0.965630, Acc.: (DEV)0.540 (TST)0.432, lr : 0.00923\n",
      "Step 44300: loss 0.904206, Acc.: (DEV)0.544 (TST)0.432, lr : 0.00923\n",
      "Step 45400: loss 0.886188, Acc.: (DEV)0.546 (TST)0.434, lr : 0.00914\n",
      "Step 45900: loss 0.942656, Acc.: (DEV)0.566 (TST)0.439, lr : 0.00914\n",
      "Step 46400: loss 0.933826, Acc.: (DEV)0.575 (TST)0.440, lr : 0.00914\n",
      "Step 47300: loss 0.969411, Acc.: (DEV)0.577 (TST)0.442, lr : 0.00914\n"
     ]
    }
   ],
   "source": [
    "#start training\n",
    "batch_size = 40\n",
    "max_acc = 0.0\n",
    "max_step=0\n",
    "saver_folder='snnmodel_phone_ru'\n",
    "if not os.path.exists(saver_folder):\n",
    "    os.mkdir(saver_folder)\n",
    "for step in range(500000):\n",
    "  \n",
    "    if step %5 ==0:\n",
    "        batch_x1_a, batch_y1_a = trn_tar.next_batch(batch_size,shuffle=False)\n",
    "        batch_x2_a, batch_y2_a = trn_tar_p.next_batch(batch_size,shuffle=False)\n",
    "        batch_x1_b, batch_y1_b = trn_non.next_batch(batch_size,shuffle=False)\n",
    "        batch_x2_b, batch_y2_b = trn_non_p.next_batch(batch_size,shuffle=False)\n",
    "        batch_x1 = np.append(batch_x1_a,batch_x1_b,axis=0)\n",
    "        batch_y1 = np.append(batch_y1_a,batch_y1_b,axis=0)\n",
    "        batch_x2 = np.append(batch_x2_a,batch_x2_b,axis=0)\n",
    "        batch_y2 = np.append(batch_y2_a,batch_y2_b,axis=0)\n",
    "    else:\n",
    "        batch_x1_a, batch_y1_a = dev_tar.next_batch(batch_size,shuffle=False)\n",
    "        batch_x2_a, batch_y2_a = dev_tar_p.next_batch(batch_size,shuffle=False)\n",
    "        batch_x1_b, batch_y1_b = dev_non.next_batch(batch_size,shuffle=False)\n",
    "        batch_x2_b, batch_y2_b = dev_non_p.next_batch(batch_size,shuffle=False)\n",
    "        batch_x1 = np.append(batch_x1_a,batch_x1_b,axis=0)\n",
    "        batch_y1 = np.append(batch_y1_a,batch_y1_b,axis=0)\n",
    "        batch_x2 = np.append(batch_x2_a,batch_x2_b,axis=0)\n",
    "        batch_y2 = np.append(batch_y2_a,batch_y2_b,axis=0)\n",
    "        \n",
    "    batch_y = batch_y1*2-1\n",
    "    \n",
    "    _, loss_v = sess.run([train_step, siamese.loss], feed_dict={\n",
    "        siamese.x1: batch_x1,\n",
    "        siamese.x2: batch_x2,\n",
    "        siamese.y_: batch_y\n",
    "    })\n",
    "    \n",
    "    if np.isnan(loss_v):\n",
    "        print ('Model diverged with loss = NAN')\n",
    "        quit()\n",
    "        \n",
    "    if step % 100 ==0:\n",
    "        dev_features_siam = siamese.o1.eval({siamese.x1:dev_features})\n",
    "        lang_mean_siam = siamese.o1.eval({siamese.x1:lang_mean})\n",
    "        tst_features_siam = siamese.o1.eval({siamese.x1:tst_features})\n",
    "        \n",
    "        dev_scores = lang_mean_siam.dot(dev_features_siam.transpose() )\n",
    "        hypo_lang = np.argmax(dev_scores,axis = 0)\n",
    "        temp = ((dev_labels-1) - hypo_lang)\n",
    "        acc =1- np.size(np.nonzero(temp)) / float( np.size(dev_labels) )\n",
    "        \n",
    "        tst_scores = lang_mean_siam.dot(tst_features_siam.transpose() )\n",
    "        hypo_lang = np.argmax(tst_scores,axis = 0)\n",
    "        temp = ((tst_labels-1) - hypo_lang)\n",
    "        acc_tst =1- np.size(np.nonzero(temp)) / float(np.size(tst_labels))\n",
    "\n",
    "        if max_acc < acc_tst:\n",
    "            max_acc = acc_tst\n",
    "            max_step=step\n",
    "            print ('Step %d: loss %f, Acc.: (DEV)%.3f (TST)%.3f, lr : %.5f' % (step,loss_v,acc,acc_tst,sess.run(learning_rate)))\n",
    "            saver.save(sess, saver_folder+'/model'+str(step)+'.ckpt')\n",
    "        if loss_v<0.6:\n",
    "            break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "47300\n",
      "INFO:tensorflow:Restoring parameters from snnmodel_phone_ru/model47300.ckpt\n",
      "Final accurary on test dataset : 0.442\n"
     ]
    }
   ],
   "source": [
    "print max_step\n",
    "RESTORE_STEP=max_step\n",
    "saver.restore(sess, saver_folder+'/model'+str(RESTORE_STEP)+'.ckpt')\n",
    "\n",
    "\n",
    "# trn_features_siam = siamese.o1.eval({siamese.x1:trn_features})\n",
    "dev_features_siam = siamese.o1.eval({siamese.x1:dev_features})\n",
    "tst_features_siam = siamese.o1.eval({siamese.x1:tst_features})\n",
    "lang_mean_siam = siamese.o1.eval({siamese.x1:lang_mean})\n",
    "\n",
    "tst_scores = lang_mean_siam.dot(tst_features_siam.transpose() )\n",
    "# print(tst_scores.shape)\n",
    "hypo_lang = np.argmax(tst_scores,axis = 0)\n",
    "temp = ((tst_labels-1) - hypo_lang)\n",
    "acc =1- np.size(np.nonzero(temp)) / float(np.size(tst_labels))\n",
    "print 'Final accurary on test dataset : %0.3f' %(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix\n",
      "[[   1.    9.    2.    6.    1.]\n",
      " [ 180.  161.  225.   37.    0.]\n",
      " [   0.    0.    0.    0.   81.]\n",
      " [  30.   26.   26.  179.   15.]\n",
      " [  91.   54.   81.   40.  247.]]\n",
      "Precision\n",
      "[ 0.05263158  0.26699834  0.          0.64855072  0.48148148]\n",
      "Recall\n",
      "[ 0.00331126  0.644       0.          0.68320611  0.71802326]\n",
      "\n",
      "\n",
      "<Performance evaluation on Test dataset>\n",
      "Accurary  : 0.442\n",
      "Precision : 0.290\n",
      "Recall    : 0.410\n"
     ]
    }
   ],
   "source": [
    "confusionmat = np.zeros((5,5))\n",
    "for i,lang in enumerate(languages):\n",
    "    hypo_bylang = hypo_lang[ tst_labels == i+1]\n",
    "    hist_bylang = np.histogram(hypo_bylang,5)\n",
    "    confusionmat[:,i] = hist_bylang[0]\n",
    "\n",
    "precision = np.diag(confusionmat) / np.sum(confusionmat,axis=1) #precision\n",
    "recall = np.diag(confusionmat) / np.sum(confusionmat,axis=0) # recall\n",
    "    \n",
    "print 'Confusion matrix'\n",
    "print confusionmat\n",
    "print 'Precision'\n",
    "print precision\n",
    "print 'Recall'\n",
    "print recall\n",
    "\n",
    "print '\\n\\n<Performance evaluation on Test dataset>'\n",
    "print 'Accurary  : %0.3f' %(acc)\n",
    "print 'Precision : %0.3f' %(np.mean(precision))\n",
    "print 'Recall    : %0.3f' %(np.mean(recall))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}