preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import jieba\n",
    "import os\n",
    "import jieba.posseg as pseg\n",
    "if not os.path.exists('data/dict.big.txt'):\n",
    "    os.system(\"wget %s -O %s\" % (\n",
    "        'https://raw.githubusercontent.com/fxsjy/jieba/master/extra_dict/dict.txt.big',\n",
    "        'data/dict.big.txt'))\n",
    "jieba.set_dictionary('data/dict.big.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocessing (seg)\n",
    "# Comebine test data & review \n",
    "test_data = pd.read_csv('data/test.csv')\n",
    "review = {}\n",
    "with open('data/test_review.txt') as f:\n",
    "    lines = [line.strip() for line in f.readlines()]\n",
    "    for i in range(0, len(lines), 2):\n",
    "        review[int(lines[i])] = ' '.join([tok for tok in jieba.cut(lines[i+1])])\n",
    "\n",
    "test_data['Review'] = [review[review_id] for review_id in test_data.Review_id]\n",
    "test_data.to_csv('data/test_seg.csv', index=None)\n",
    "\n",
    "# Seg polarity review\n",
    "review, polarity = [], []\n",
    "with open('data/polarity_review.txt') as f:\n",
    "    for line in [line.strip() for line in f.readlines()]:\n",
    "        l = line.split('\\t')\n",
    "        polarity.append(int(l[0]))\n",
    "        review.append(' '.join([tok for tok in jieba.cut(l[1])]))\n",
    "polarity_review_seg = pd.DataFrame({'Polarity': polarity, 'Review': review})\n",
    "polarity_review_seg.to_csv('data/polarity_review_seg.csv', index=None)\n",
    "\n",
    "# Seg aspect review\n",
    "aspect_review = []\n",
    "with open('data/aspect_review.txt') as f:\n",
    "    lines = [line.strip() for line in f.readlines()]\n",
    "    for i in range(0, len(lines), 4):\n",
    "        aspect_review.append({\n",
    "                'Id': int(lines[i]),\n",
    "                'Review': ' '.join([tok for tok in jieba.cut(lines[i+1])]),\n",
    "                'Pos': lines[i+2],\n",
    "                'Neg': lines[i+3]\n",
    "            })\n",
    "aspect_review = pd.DataFrame(aspect_review)\n",
    "aspect_review.to_csv('data/aspect_review_seg.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Pos & seg\n",
    "def posseg(s):\n",
    "    s = s.replace(' ', '')\n",
    "    tokens = pseg.cut(s)\n",
    "    return '\\n'.join(['%s %s' % (token.word, token.flag) for token in tokens])\n",
    "\n",
    "test_data.Review = test_data.Review.apply(posseg)\n",
    "test_data.to_csv('data/test_posseg.csv', index=None)\n",
    "\n",
    "polarity_review_seg.Review = polarity_review_seg.Review.apply(posseg)\n",
    "polarity_review_seg.to_csv('data/polarity_review_posseg.csv', index=None)\n",
    "\n",
    "aspect_review.Review = aspect_review.Review.apply(posseg)\n",
    "aspect_review.to_csv('data/aspect_review_posseg.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "aspect_review = pd.read_csv('data/aspect_review_seg.csv')\n",
    "polarity_review = pd.read_csv('data/polarity_review_seg.csv')\n",
    "test_review = pd.read_csv('data/test_seg.csv')\n",
    "test_review = test_review.groupby('Review_id').first()\n",
    "reviews = aspect_review['Review'].append(polarity_review['Review']).append(test_review['Review'])\n",
    "print('#reviews:', len(reviews))\n",
    "\n",
    "if os.path.exists('data/external_corpus.txt'):\n",
    "    print('using extrenal corpus, it may take pretty loooong')\n",
    "    with open('data/external_corpus.txt', \"r\") as inputs:\n",
    "        with open('data/extrenal_corpus.txt', 'w') as cutted:\n",
    "            for line in inputs.readlines():\n",
    "                cutted.write(jieba.cut(line))\n",
    "else:\n",
    "    print('external corpus dosen\\'t exitst (put it as data/external_corpus.txt plz QAQ)')\n",
    "    \n",
    "\n",
    "with open(\"data/word2vec_corpus.tmp\", \"w\") as f:\n",
    "    if os.path.exists('data/external_corpus_seg.txt'):\n",
    "        with open(\"data/external_corpus_seg.txt\") as ext:\n",
    "            f.write(ext.read())\n",
    "        f.write((\"\\n\".join(reviews)+\"\\n\")*5)\n",
    "    else:\n",
    "        f.write((\"\\n\".join(reviews)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if os.path.exists('word2vec/word2vec'):\n",
    "    print('running word2vec ...')\n",
    "    os.system('time word2vec/word2vec -train data/word2vec_corpus.tmp -output vectors.bin -cbow 1 -size 200 -window 8 -negative 25 -hs 0 -sample 1e-4 -threads 20 -binary 1 -iter 15')\n",
    "else:\n",
    "    print('word2vec doesn\\'t exitst (put it as word2vec/word2vec plz QAQ)')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}