cluster_new.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import fnmatch\n",
    "import zipfile\n",
    "import xmltodict\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import json\n",
    "import gzip\n",
    "import pickle\n",
    "import csv\n",
    "import scipy.sparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setup the update to work despite the broken scipy documentation\n",
    "try:\n",
    "    a = scipy.sparse.dok_matrix((10,10))\n",
    "    a.update({(0,0):1.0})\n",
    "    scipy.sparse.dok_matrix.my_update = scipy.sparse.dok_matrix.update\n",
    "except:\n",
    "    a = scipy.sparse.dok_matrix((10,10))\n",
    "    a._update({(0,0):1.0})\n",
    "    scipy.sparse.dok_matrix.my_update = scipy.sparse.dok_matrix._update\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "faculty_affil = pd.read_csv('faculty-affiliations.csv')\n",
    "def csv2dict_str_str(fname):\n",
    "    with open(fname, mode='r') as infile:\n",
    "        rdr = csv.reader(infile)\n",
    "        d = {rows[0].strip(): rows[1].strip() for rows in rdr}\n",
    "    return d\n",
    "aliasdict = csv2dict_str_str('dblp-aliases-expanded.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "csr_unis = faculty_affil.affiliation.unique()\n",
    "faculty_affil['numb'] = 1\n",
    "sum_of_profs = faculty_affil.groupby('affiliation').sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "csranks = pd.read_csv('other_ranks/ranks.csv')\n",
    "\n",
    "conf_fix = csv2dict_str_str('name_alias.csv')\n",
    "conf_fix_inv = {v:k for k,v in conf_fix.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r1 = pd.read_csv('other_ranks/r1.csv',header=None)\n",
    "\n",
    "\n",
    "i = 0\n",
    "for row in r1.itertuples():\n",
    "    if not row[1] in csr_unis:\n",
    "        print(row[1])\n",
    "        i +=1\n",
    "print(i)\n",
    "r1 = list(r1[0])\n",
    "r1 = [_ for _ in r1 if _ in csr_unis and sum_of_profs.loc[_].numb >=0]\n",
    "# add to r1\n",
    "# have a blacklist for giant uni\n",
    "top_unis = []#[_ for _ in list(csranks[csranks['rank'] <= 45].uni) if _ not in ['HKUST','Peking University','Tsinghua University','National University of Singapore']] \n",
    "r1 = list(set(r1 + top_unis))\n",
    "len(r1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r1_faculty_affil = {k:v for k,v in faculty_affil.iteritems()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get r1\n",
    "filter_set = [1 if row[2] in r1 else 0 for row in faculty_affil.itertuples()]\n",
    "fss = np.array(filter_set).astype(bool)\n",
    "r1_faculty_affil = faculty_affil[fss]\n",
    "\n",
    "# get alias\n",
    "r1_unique = sorted(set([aliasdict.get(row[1],row[1]) for row in r1_faculty_affil.itertuples()]))\n",
    "len(r1_unique),r1_faculty_affil.shape\n",
    "r1_unique_dict = {k:1 for k in r1_unique}\n",
    "affils_dict = {row[1]:row[2] for row in r1_faculty_affil.itertuples()} \n",
    "affils_dict_full = {row[1]:row[2] for row in faculty_affil.itertuples()} \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if False:\n",
    "    r1_unique_dict2 = {k:1 for k in r1_unique_2}\n",
    "    new_names = []\n",
    "    for name in r1_unique:\n",
    "        if name not in r1_unique_dict2:\n",
    "            new_names.append(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with gzip.open('useful_venue_list.pkl.gz','rb') as fp:\n",
    "    all_venues = pickle.load(fp)\n",
    "with gzip.open('useful_authors_list.pkl.gz','rb') as fp:\n",
    "    all_authors = pickle.load(fp)\n",
    "with gzip.open('useful_papers.pkl.gz','rb') as fp:\n",
    "    all_papers = pickle.load(fp)\n",
    "conf_idx = {v:i for i,v in enumerate(all_venues)}\n",
    "name_idx = {v:i for i,v in enumerate(all_authors)}\n",
    "n_confs = len(all_venues)\n",
    "n_auths = len(all_authors)\n",
    "#r1_confs = pickle.load(open('old_version/r1_confs.pkl','rb'))\n",
    "#r1_confs_dict = {_:1 for _ in r1_confs}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MIN_YEAR = 2010\n",
    "Xr1 = scipy.sparse.dok_matrix((len(r1),n_confs))\n",
    "xdict = {}\n",
    "for paper in all_papers:\n",
    "    tag,title, authors, conf, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    j = conf_idx[conf]\n",
    "    if year < MIN_YEAR:\n",
    "        continue\n",
    "    for a in authors:\n",
    "        name = aliasdict.get(a,a)\n",
    "        if name in affils_dict:\n",
    "            uni = affils_dict[name]\n",
    "            i = r1.index(uni)\n",
    "            xdict[(i,j)] = 1\n",
    "Xr1.my_update(xdict)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_confs = (Xr1.sum(0) >= 20)\n",
    "np.save('mask',np.squeeze(np.array(filtered_confs)))\n",
    "\n",
    "\n",
    "r1_confs = [conf for conf in all_venues if filtered_confs[0,conf_idx[conf]]]\n",
    "print(len(r1_confs))\n",
    "#print(r1_confs)\n",
    "r1_conf_idx = {v:i for i,v in enumerate(r1_confs)}\n",
    "r1_conf_idx_orig = [conf_idx[conf] for conf in all_venues if filtered_confs[0,conf_idx[conf]]]\n",
    "\n",
    "r1_scores = Xr1[:,np.array(filtered_confs).reshape((-1))]\n",
    "r1_scores = np.array(r1_scores.sum(0)).reshape((-1))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r1_confs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xpaper = scipy.sparse.dok_matrix((len(r1_unique),len(r1_confs)))\n",
    "xdict = {}\n",
    "for paper in all_papers:\n",
    "    tag,title, authors, conf, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    if conf not in r1_conf_idx:\n",
    "        continue\n",
    "    if year < MIN_YEAR:\n",
    "        continue\n",
    "    j = r1_conf_idx[conf]\n",
    "    n = len(authors)\n",
    "    av = np.ones(n)/(np.arange(n)+1)\n",
    "    av /= av.sum()\n",
    "    for ai,a in enumerate(authors):\n",
    "        name = aliasdict.get(a,a)\n",
    "        if name in affils_dict:\n",
    "            i = r1_unique.index(name)\n",
    "            xdict[(i,j)] = av[ai] + xdict.get((i,j),0) #/len(authors)\n",
    "Xpaper.my_update(xdict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r1counts = np.squeeze(np.array(Xpaper.sum(0)))\n",
    "'SIGIR' in r1_conf_idx\n",
    "if False:\n",
    "    indexs = []\n",
    "    for name in new_names:\n",
    "        indexs.append(r1_unique.index(name))\n",
    "    DP = np.array(Xpaper.todense())\n",
    "    for idx  in np.argsort(DP[np.array(indexs),:].sum(1))[::-1]:\n",
    "        idxt = indexs[idx]\n",
    "        print(r1_unique[idxt],DP[idxt].sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "v = np.argsort(np.array(Xpaper[:,r1_conf_idx['IROS']].todense()).flatten())[::-1]\n",
    "for i in v[:20]:\n",
    "    print(r1_unique[i],Xpaper[i,r1_conf_idx['IROS']],Xpaper[i,r1_conf_idx['INFOCOM']])\n",
    "    v2 = np.argsort(np.array(Xpaper[i].todense()).flatten())[::-1]\n",
    "    #print([r1_confs[i2] for i2 in v2[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "authors_per = np.squeeze(np.array(Xpaper.sum(1)))\n",
    "sorted_num = np.copy(authors_per)\n",
    "sorted_num.sort()\n",
    "print(authors_per.max(),authors_per.mean(),authors_per.std(),sorted_num[5*sorted_num.shape[0]//8])\n",
    "filtered_authors = (authors_per > max(authors_per.mean(),sorted_num[5*sorted_num.shape[0]//8])) #& (authors_per < 180)\n",
    "Xpaper_da = Xpaper[filtered_authors,:]\n",
    "filtered_authors_lookup = np.arange(Xpaper.shape[0])[filtered_authors]\n",
    "\n",
    "print(Xpaper_da.shape,print(filtered_authors.astype(np.float32).sum()),filtered_authors.astype(np.int).sum())\n",
    "#Xpaper_da = Xpaper\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xpaper_da,Xpaper_da.sum(0).max(),Xpaper_da.sum(1).max()\n",
    "v =np.array(Xpaper.sum(1)).ravel()\n",
    "for idx in np.argsort(v )[::-1]:\n",
    "    if r1_unique[idx] + ' 0001' in name_idx:\n",
    "        print(r1_unique[idx],v[idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "i1,i2 = r1_conf_idx['RECOMB'],r1_conf_idx['ASPLOS']\n",
    "Xpaper2 = np.array(Xpaper_da.todense())\n",
    "filters_tmp = (Xpaper2[:,i1] > 2) & (Xpaper2[:,i2] > 2)\n",
    "fidx = np.where(filters_tmp > 0)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import LatentDirichletAllocation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if True:\n",
    "    clf = LatentDirichletAllocation(50,learning_method='online',max_iter=150) #,max_iter=150\n",
    "    what = clf.fit_transform(Xpaper_da.T)\n",
    "    #what = clf.components_.T\n",
    "else:\n",
    "    from scipy.sparse.linalg import svds\n",
    "    u,s,vt = svds(Xpaper_da,50)\n",
    "    what = vt.T\n",
    "what.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "whatorig = np.copy(what)\n",
    "what = what/np.linalg.norm(what,axis=1,keepdims=True)\n",
    "#what = np.copy(whatorig)\n",
    "np.linalg.norm(what,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.manifold import TSNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed = TSNE(perplexity=16,n_iter=12000,metric='cosine',init='random') # cosine euclidean\n",
    "ys = embed.fit_transform(what)\n",
    "print(ys.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cluster import MiniBatchKMeans\n",
    "from sklearn import metrics\n",
    "CLASS_NUM = 28\n",
    "cv = []\n",
    "#for CLASS_NUM in range(15,50):\n",
    "#    rowv = []\n",
    "#    for i in range(10):\n",
    "km = MiniBatchKMeans(CLASS_NUM,n_init=10,max_iter=2500)#n_init=3200,max_iter=450,#, max_iter=5000,max_no_improvement=None,reassignment_ratio=0.02)\n",
    "km.fit(what)\n",
    "s = metrics.silhouette_score(what,km.labels_,metric='cosine')\n",
    "s2 = metrics.silhouette_score(what,km.labels_,metric='euclidean')\n",
    "#rowv.append(np.sqrt(s*s2))\n",
    "#    cv.append(rowv)\n",
    "#    print('{}\\t\\t{:.2f}\\t\\t{:.2f}\\t\\t{:.2f}'.format(CLASS_NUM,s,s2,np.sqrt(s*s2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if len(cv) > 0:\n",
    "    A = np.array(cv)\n",
    "    xp = np.arange(15,50)\n",
    "    fig = plt.figure(figsize=(5,3))\n",
    "    plt.plot(xp,A.mean(1),lw=4)\n",
    "    plt.fill_between(xp, A.mean(1)-A.std(1), A.mean(1)+A.std(1),alpha=0.3)\n",
    "    plt.xlabel('Number of Clusters')\n",
    "    plt.ylabel('Silhouette Score')\n",
    "    plt.title('Optimal Number of Clusters')\n",
    "    plt.grid()\n",
    "    plt.tight_layout()\n",
    "    plt.savefig('cluster.pdf', facecolor=fig.get_facecolor(), edgecolor='none')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ya = (ys - ys.mean(0))/ys.std(0)\n",
    "print(ya.max())\n",
    "consta = 2.9\n",
    "ya[np.where(abs(ya) > consta)] = consta*np.sign(ya[np.where(abs(ya) > consta)])\n",
    "ya = (ya - ya.mean(0))/ya.std(0)\n",
    "print(ya.max())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.figure(figsize=(10,10))\n",
    "plt.scatter(ya[:,0],ya[:,1],c=km.labels_/CLASS_NUM,cmap='tab20c')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filt_vec = np.squeeze(np.array(filtered_confs))\n",
    "\n",
    "try:\n",
    "    clf_gold = np.load('clf_gold.pkl.npy')\n",
    "    years_per_conf = clf_gold.shape[0]//n_confs\n",
    "    max_clf = clf_gold.reshape((-1,years_per_conf))[:,-5:].max(1)\n",
    "    max_clf_filt = max_clf[filt_vec]\n",
    "    vector_to_look_at = max_clf_filt\n",
    "except:\n",
    "    vector_to_look_at = r1_scores  #max_clf_filt or r1_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cname = 'ECCV'\n",
    "conf_near_neighbor = conf_fix_inv.get(cname,cname)\n",
    "NN = 30\n",
    "authors_per_f =  np.squeeze(np.array(Xpaper_da.sum(0)))\n",
    "if conf_near_neighbor in r1_confs:\n",
    "    dists = np.linalg.norm(what - what[r1_conf_idx[conf_near_neighbor]],axis=1)\n",
    "    dists /= dists.std()\n",
    "    idx_d = np.argsort(dists)\n",
    "\n",
    "    for i in range(NN):\n",
    "        idx = idx_d[i]\n",
    "        name = r1_confs[idx]\n",
    "        print('{:30s}\\t{:.0f}\\t{:.2f}\\t\\t{:.2f}'.format(name[:30],authors_per_f[idx],dists[idx],vector_to_look_at[idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idxc = r1_conf_idx[conf_near_neighbor]\n",
    "print(Xpaper_da.shape)\n",
    "confs_per_f = np.squeeze(np.array(Xpaper_da[:,idxc].todense()))\n",
    "author_conf_fault = np.argsort(confs_per_f)[::-1]\n",
    "for i in author_conf_fault:\n",
    "    v = confs_per_f[i]\n",
    "    if v < 1e-2:\n",
    "        break\n",
    "    print(r1_unique[filtered_authors_lookup[i]])#,v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(CLASS_NUM):\n",
    "    tmp = [(vector_to_look_at[i],i) for i,v in enumerate(km.labels_ == i) if v ] \n",
    "    score_idx = sorted(tmp,reverse=True)\n",
    "    print(i,len(score_idx))\n",
    "    for s,idx in score_idx[:5]:\n",
    "        print('\\t',conf_fix.get(r1_confs[idx],r1_confs[idx]))\n",
    "    #print('NEXT')\n",
    "    #tmp = [(r1_scores[i],i) for i,v in enumerate(km.labels_ == i) if v ] \n",
    "    #score_idx = sorted(tmp,reverse=True)\n",
    "    #for s,idx in score_idx[:5]:\n",
    "    #    print('\\t',r1_confs[idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patheffects as path_effects\n",
    "plt.figure(figsize=(25,25))\n",
    "cmap = plt.get_cmap('tab20c_r')\n",
    "#plt.subplot(1,2,1)\n",
    "plt.scatter(ya[:,0],ya[:,1],c=[cmap(_) for _ in km.labels_/CLASS_NUM],s=0.35*r1_scores**2,lw=0.1,edgecolors='k')\n",
    "for i in range(CLASS_NUM):\n",
    "    tmp = [(vector_to_look_at[i],i) for i,v in enumerate(km.labels_ == i) if v ] \n",
    "    score_idx = sorted(tmp,reverse=True)\n",
    "    #print(i)\n",
    "    k = 3\n",
    "    j = 0\n",
    "    for s,idx in reversed(score_idx[:k]):\n",
    "        #if s < 1.5:\n",
    "        #    continue\n",
    "        rv = np.random.randn(2)\n",
    "        xr,yr = 0.001*(rv)#/np.linalg.norm(rv)\n",
    "        text = plt.text(xr+ya[idx,0],0.1/3*k*(j-(k-1)/2)/(k-1)+ya[idx,1],conf_fix.get(r1_confs[idx],r1_confs[idx]),size='20',color=np.array(cmap(i/CLASS_NUM)),\n",
    "                 ha='center',va='center',alpha=0.9,weight='bold')\n",
    "        text.set_path_effects([path_effects.Stroke(linewidth=1, foreground='black'),\n",
    "                       path_effects.Normal()])\n",
    "        j+=1\n",
    "        #print('\\t',r1_confs[idx])\n",
    "    #print()\n",
    "plt.xticks([],[])\n",
    "plt.yticks([],[])\n",
    "plt.tight_layout()\n",
    "plt.axis('off')\n",
    "plt.savefig('tsne4.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('labels',km.labels_)\n",
    "np.save('vectors',what)\n",
    "np.save('mask',np.squeeze(np.array(filtered_confs)))\n",
    "with open('clusters.csv','tw') as fp:\n",
    "    for i in range(CLASS_NUM):\n",
    "        tmp = [(vector_to_look_at[i],i) for i,v in enumerate(km.labels_ == i) if v ] \n",
    "        score_idx = sorted(tmp,reverse=True)\n",
    "        for s,idx in score_idx:\n",
    "            fp.write('{},'.format(r1_confs[idx]))\n",
    "            j+=1\n",
    "        fp.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with gzip.open('useful_authors_list.pkl.gz','rb') as fp:\n",
    "    all_authors = pickle.load(fp)\n",
    "name_idx = {v:i for i,v in enumerate(all_authors)}\n",
    "n_auths = len(all_authors)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_filt_confs = what.shape[0]\n",
    "old_idx_to_small = {}\n",
    "j = 0\n",
    "for i in range(filt_vec.shape[0]):\n",
    "    if filt_vec[i]:\n",
    "        old_idx_to_small[i] = j\n",
    "        j+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_auth_papers = np.zeros(n_auths)\n",
    "for paper in all_papers:\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    n = len(authors)\n",
    "    if filt_vec[conf_idx[venue]]:\n",
    "        for ai in range(n):\n",
    "            i = name_idx[authors[ai]]\n",
    "            num_auth_papers[i] += 1/n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.sparse\n",
    "Xauth = scipy.sparse.dok_matrix((n_auths,num_filt_confs))\n",
    "xdict = {}\n",
    "for paper in all_papers:\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    n = len(authors)\n",
    "    if filt_vec[conf_idx[venue]]:\n",
    "        j = old_idx_to_small[conf_idx[venue]]\n",
    "        for ai in range(n):#zip(count_vecs[n],authors):\n",
    "            i = name_idx[authors[ai]]\n",
    "            xdict[(i,j)] = (1/n)/num_auth_papers[i] + xdict.get((i,j),0)\n",
    "    #j = years_per_conf*conf_idx[venue] + (year-min_year)\n",
    "    #for ai in range(n):#zip(count_vecs[n],authors):\n",
    "    #    i = name_idx[authors[ai]]\n",
    "    #    #xdict[(i,j)] = 1/n + xdict.get((i,j),0)\n",
    "    #    xdict[(i,j)] = count_vecs[n][ai] + xdict.get((i,j),0)\n",
    "\n",
    "\n",
    "Xauth.my_update(xdict)\n",
    "\n",
    "Xauth = scipy.sparse.csr_matrix(Xauth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xmapped = Xauth @ what\n",
    "Xmapped_2d = Xauth @ ya"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranks = pd.read_csv('other_ranks/ranks.csv')\n",
    "top_k = 9\n",
    "subplot = int(round(np.sqrt(top_k)))\n",
    "min_v = ya.min(0)\n",
    "max_v = ya.max(0)\n",
    "plt.figure(figsize=(subplot*4,subplot*4))\n",
    "plt.rcParams = plt.rcParamsDefault\n",
    "plt.set_cmap('viridis')\n",
    "for i in range(top_k):\n",
    "    Uname = ranks.iloc[i,:].uni\n",
    "    uni_faculty = faculty_affil[faculty_affil.affiliation == Uname] \n",
    "    uni_names = np.array(uni_faculty.name)\n",
    "    uni_names = list(uni_names)\n",
    "    cmu_scores = []\n",
    "    for name in set([aliasdict.get(n, n) for n in uni_names]):\n",
    "        if name in name_idx:\n",
    "            loc = Xmapped_2d[name_idx[name],:]\n",
    "            cmu_scores.append((loc))\n",
    "    cmu_scores = np.squeeze(np.array(cmu_scores))\n",
    "    plt.subplot(subplot,subplot,i+1)\n",
    "    plt.hexbin(cmu_scores[:,0],cmu_scores[:,1],gridsize=12,extent=(min_v[0],max_v[0],min_v[1],max_v[1]),vmin=0,vmax=4)\n",
    "    plt.title(Uname,color='k',weight='demibold')\n",
    "    plt.xticks([],[])\n",
    "    plt.yticks([],[])\n",
    "    plt.tight_layout()\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('fingers4.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pickle.dump(what,open('top_conf_embed.pkl','wb'))\n",
    "#pickle.dump(filtered_confs,open('top_conf_ind.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pickle.dump(r1_scores,open('r1_scores.pkl','wb'))\n",
    "#pickle.dump(r1_confs,open('r1_confs.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cmu_uni = pd.read_csv('other_ranks/cmu_faculty.csv')\n",
    "cmu_uni = cmu_uni.fillna('CSD')\n",
    "#print(list(cmu_uni.name))\n",
    "if False:\n",
    "    uni_names = list(set(faculty_affil[faculty_affil.affiliation == 'Cornell University'].name))#list(cmu_uni.name)\n",
    "    uni_labels = ['McGill' for _ in range((len(uni_names)))]#list(cmu_uni.dept)\n",
    "    uni_labels_unique = list(set(uni_labels))\n",
    "\n",
    "else:\n",
    "    uni_names = list(cmu_uni.name)\n",
    "    uni_labels =list(cmu_uni.dept)\n",
    "    uni_labels_unique = list(set(uni_labels))\n",
    "    uni_labels_unique = ['CSD','RI','MLD',\"LTI\",'HCI','ISR','BIO']\n",
    "\n",
    "cmu_scores = []\n",
    "uni_colors = []\n",
    "for name,d in set([(aliasdict.get(n, n),dept) for n,dept in zip(uni_names,uni_labels)]):\n",
    "    if name in name_idx:\n",
    "        loc = Xmapped[name_idx[name],:]\n",
    "        cmu_scores.append((loc,name))\n",
    "        uni_colors.append( uni_labels_unique.index(d))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cmap = plt.get_cmap('tab20c')\n",
    "Cmapped = Xauth @ np.array([cmap(i) for i in km.labels_])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed2 = np.vstack([_[0] for _ in cmu_scores])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedder2 = TSNE(2,perplexity=12,init='pca',n_iter=1000)\n",
    "ys2 = embedder2.fit_transform(embed2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = [_[1] for _ in cmu_scores]\n",
    "dists = []#scipy.spatial.distance.squareform(scipy.spatial.distance.pdist(embed2))[names.index('Noah Snavely')]\n",
    "for i in np.argsort(dists):\n",
    "    print('{:20s}\\t{:.2f}'.format(names[i],dists[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.colors\n",
    "import matplotlib.patheffects as path_effects\n",
    "\n",
    "nc = (len(uni_labels_unique))\n",
    "cmap = plt.get_cmap('tab10')\n",
    "plt.figure(figsize=(20,20))\n",
    "\n",
    "color_vec = []\n",
    "\n",
    "for i in range(ys2.shape[0]):\n",
    "    if nc == 1:\n",
    "        color = np.clip(Cmapped[name_idx[cmu_scores[i][1]],:],0,1)\n",
    "    else:\n",
    "        color=cmap(uni_colors[i]/nc)\n",
    "    color_vec.append(color)\n",
    "\n",
    "plt.scatter(ys2[:,0],ys2[:,1],s=120,lw=1,edgecolors='k',c=color_vec)\n",
    "for i in range(ys2.shape[0]):\n",
    "    rv = np.random.randn(2)\n",
    "    xr,yr = 0.2*(rv)#/np.linalg.norm(rv)\n",
    "    color = color_vec[i]\n",
    "    print(cmu_scores[i][1],ys2[i,0],ys2[i,1])\n",
    "\n",
    "    text= plt.text(ys2[i,0]+xr,yr+ys2[i,1],cmu_scores[i][1],size='14',color=color,\n",
    "             horizontalalignment='center',verticalalignment='center',alpha=0.9,weight='bold')\n",
    "    text.set_path_effects([path_effects.Stroke(linewidth=0.2, foreground='black'),\n",
    "               path_effects.Normal()])\n",
    "plt.savefig('uni_faculty.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.colors\n",
    "import matplotlib.patheffects as path_effects\n",
    "from cycler import cycler\n",
    "#plt.style.use('Solarize_Light2') #bmh\n",
    "plt.style.use('default')\n",
    "plt.style.use('seaborn-white')\n",
    "#plt.rcParams[\"axes.prop_cycle\"] =  cycler(color=['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf'])\n",
    "plt.rcParams[\"axes.prop_cycle\"] =  cycler(color=['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b',  '#bcbd22', '#17becf'])\n",
    "nc = (len(uni_labels_unique))\n",
    "cmap = plt.get_cmap('tab10')\n",
    "plt.figure(figsize=(12,12))\n",
    "colors_for_now = []\n",
    "uni_colors = np.array(uni_colors)\n",
    "legends = []\n",
    "\n",
    "for i in [0,1,2,3,4,5,6]:\n",
    "    #c=[cmap(uni_colors[i]/10+0.2) for j in uni_colors if j==i],\n",
    "    lgd = plt.scatter(ys2[i == uni_colors,0],ys2[i == uni_colors,1],s=25,lw=0.2,edgecolors='k',label=uni_labels_unique[i])\n",
    "    legends.append(lgd)\n",
    "colors_for_now = [l.get_facecolor() for l in legends]\n",
    "plt.legend(markerscale = 5,prop={'size': 24}) #\n",
    "\n",
    "if True:\n",
    "    for i in range(ys2.shape[0]):\n",
    "        rv = np.random.randn(2)\n",
    "        xr,yr = 0.15*(rv)#/np.linalg.norm(rv)\n",
    "        def name(txt):\n",
    "            txts = txt.split(' ')\n",
    "            if txts[-1].isdigit():\n",
    "                txts = txts[:-1]\n",
    "            return txts[0][0] + ' ' + txts[-1]\n",
    "        the_name = name(cmu_scores[i][1])\n",
    "        text= plt.text(ys2[i,0]+xr,yr+ys2[i,1],the_name,size='12',color=colors_for_now[uni_colors[i]][0],\n",
    "                 horizontalalignment='center',verticalalignment='center',alpha=0.8,weight='demibold')\n",
    "        #text.set_path_effects([path_effects.Stroke(linewidth=0.05, foreground='black'),\n",
    "        #           path_effects.Normal()])\n",
    "plt.gca().axis('off')\n",
    "plt.xticks([],[])\n",
    "plt.yticks([],[])\n",
    "plt.tight_layout()\n",
    "plt.savefig('cmu_faculty4.pdf')"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}