component_cosine_similarity.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from gensim.models import Word2Vec\n",
    "import re\n",
    "from nltk.tokenize import word_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data() :\n",
    "    # load dataset for stemming\n",
    "    # return dataset\n",
    "    data = pd.read_csv('kamus.txt', sep='\\t')\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def look_for_imbuh(context) :\n",
    "    # parameter is list input\n",
    "    # return the result of a word which processes from stemming\n",
    "    data = load_data()\n",
    "    imbuhan = data.iloc[:, 1].values\n",
    "    i = 0\n",
    "    for item in imbuhan :\n",
    "        if item == context :\n",
    "            return data.iloc[i,0]\n",
    "        i += 1\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_model():\n",
    "    # to load model word2vec \n",
    "    # return model word2vec\n",
    "    model = Word2Vec.load(\"id.bin\")\n",
    "    model.init_sims(replace=True)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_in_word2vec(w1, w2v):\n",
    "    # to tokenize input sentence\n",
    "    # returning list of all words from sentence\n",
    "    \n",
    "    token = word_tokenize(w1)\n",
    "    \n",
    "    for item in token :\n",
    "        temp = item\n",
    "        stem = look_for_imbuh(item)\n",
    "        \n",
    "        if stem :\n",
    "            temp = stem\n",
    "    \n",
    "        if temp not in w2v:\n",
    "            token.remove(item)\n",
    "        \n",
    "    return token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_max(word1,word2,model, w2v):\n",
    "    # to count between two words in difference sentence which have a big weight similarity\n",
    "    # return weigt similarity between two sentence and how many word being processed\n",
    "    w1 = check_in_word2vec(word1, w2v)\n",
    "    w2 = check_in_word2vec(word2, w2v)\n",
    "    \n",
    "    ww1 = word_tokenize(word1)\n",
    "    ww2 = word_tokenize(word2)\n",
    "           \n",
    "    temp,y, count = 0, 0, 0\n",
    "    temp1,y1 = 0,0\n",
    "    unik, listtt = [], []\n",
    "    \n",
    "    for item in ww1 :\n",
    "        if (item not in unik) and (item not in w1) and (item not in w2) and (item in ww2) :\n",
    "                unik.append(item)\n",
    "                count = 1\n",
    "                listtt.append(1)\n",
    "                y1 += 1\n",
    "\n",
    "    if count != 0 :\n",
    "        temp1 += 0 if len(listtt) == 0 else max(listtt)\n",
    "    \n",
    "    temp,y, count = 0, 0, 0\n",
    "    for i in w1 :\n",
    "        listt = []\n",
    "        count = 0\n",
    "            \n",
    "        if i in w2v :\n",
    "            count = 1\n",
    "            y += 1\n",
    "            \n",
    "        for j in w2 :\n",
    "            if i in w2v and j in w2v :\n",
    "                listt.append(model.similarity(i, j))\n",
    "                \n",
    "        if count != 0 :\n",
    "            temp += 0 if len(listt) == 0 else max(listt)\n",
    "                            \n",
    "    return temp, temp1, y, y1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_treshold(w1,w2):\n",
    "    # to assign threshold for a sentence can be similar\n",
    "    # return True if two sentence are similar, and False if the opposite\n",
    "    model = load_model()\n",
    "    w2v = list(model.wv.vocab)\n",
    "    \n",
    "    temp, temp1, y, y1 = count_max(w1.lower(),w2.lower(),model,w2v)\n",
    "    temp += temp1\n",
    "    y += y1\n",
    "    \n",
    "    threshold = 0.45\n",
    "    \n",
    "#     print (temp, y)\n",
    "    \n",
    "    if temp > 0 : return temp/y > threshold\n",
    "    else : return False\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kata 1: makan di pinggir jalan adalah kesukaanku\n",
      "kata 2: daging steak sangat enak \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/cindy/anaconda3/lib/python3.7/site-packages/smart_open/smart_open_lib.py:398: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
      "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n",
      "/home/cindy/anaconda3/lib/python3.7/site-packages/smart_open/smart_open_lib.py:398: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
      "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/cindy/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:35: DeprecationWarning: Call to deprecated `similarity` (Method will be removed in 4.0.0, use self.wv.similarity() instead).\n"
     ]
    }
   ],
   "source": [
    "word1 = input('kata 1: ')\n",
    "word2 = input('kata 2: ')\n",
    "\n",
    "print(check_treshold(word1,word2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}