text_cnn.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "text_cnn.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "7hQ3Ps515Ji5"
      },
      "source": [
        "import os\n",
        "path = '/content/drive/MyDrive/VLSP_ReINTEL'\n",
        "#you should change this path to your project folder path\n",
        "os.chdir(path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4acXFjsq6s89",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8923ac6a-4f56-476f-d2f1-c6fbbe7d4a49"
      },
      "source": [
        "!pip install pyvi\n",
        "\n",
        "import numpy as np\n",
        "from tqdm import tqdm\n",
        "\n",
        "np.random.seed(42)\n",
        "import pandas as pd\n",
        "from pyvi import ViTokenizer\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import roc_auc_score\n",
        "\n",
        "from keras.models import Model\n",
        "from keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D\n",
        "from keras.layers import BatchNormalization, MaxPooling1D, Conv1D#, Merge\n",
        "from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D, LSTM, Bidirectional\n",
        "from keras.preprocessing import text, sequence\n",
        "from keras.callbacks import Callback\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "from keras.layers import Dense, Input, LSTM, Bidirectional, Conv1D\n",
        "from keras.layers import Dropout, Embedding\n",
        "from keras.preprocessing import text, sequence\n",
        "from keras.layers import GlobalMaxPooling1D, GlobalAveragePooling1D, concatenate, SpatialDropout1D\n",
        "from keras.models import Model\n",
        "from keras import backend as K\n",
        "from keras.models import model_from_json\n",
        "from keras.models import load_model"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pyvi\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/10/e1/0e5bc6b5e3327b9385d6e0f1b0a7c0404f28b74eb6db59a778515b30fd9c/pyvi-0.1-py2.py3-none-any.whl (8.5MB)\n",
            "\u001b[K     |████████████████████████████████| 8.5MB 7.2MB/s \n",
            "\u001b[?25hCollecting sklearn-crfsuite\n",
            "  Downloading https://files.pythonhosted.org/packages/25/74/5b7befa513482e6dee1f3dd68171a6c9dfc14c0eaa00f885ffeba54fe9b0/sklearn_crfsuite-0.3.6-py2.py3-none-any.whl\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from pyvi) (0.22.2.post1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sklearn-crfsuite->pyvi) (1.15.0)\n",
            "Collecting python-crfsuite>=0.8.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/95/99/869dde6dbf3e0d07a013c8eebfb0a3d30776334e0097f8432b631a9a3a19/python_crfsuite-0.9.7-cp36-cp36m-manylinux1_x86_64.whl (743kB)\n",
            "\u001b[K     |████████████████████████████████| 747kB 41.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=2.0 in /usr/local/lib/python3.6/dist-packages (from sklearn-crfsuite->pyvi) (4.41.1)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.6/dist-packages (from sklearn-crfsuite->pyvi) (0.8.7)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->pyvi) (1.18.5)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->pyvi) (1.4.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->pyvi) (0.17.0)\n",
            "Installing collected packages: python-crfsuite, sklearn-crfsuite, pyvi\n",
            "Successfully installed python-crfsuite-0.9.7 pyvi-0.1 sklearn-crfsuite-0.3.6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "18vnPwZv7nKu",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7a5a5794-2c77-4ba7-8f93-b66f6607ad64"
      },
      "source": [
        "#download fasttext \n",
        "#! wget https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.vi.300.vec.gz"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-12-07 06:31:17--  https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.vi.300.vec.gz\n",
            "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 104.22.75.142, 172.67.9.4, 104.22.74.142, ...\n",
            "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|104.22.75.142|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1235219084 (1.1G) [binary/octet-stream]\n",
            "Saving to: ‘cc.vi.300.vec.gz’\n",
            "\n",
            "cc.vi.300.vec.gz    100%[===================>]   1.15G  40.8MB/s    in 25s     \n",
            "\n",
            "2020-12-07 06:31:43 (46.4 MB/s) - ‘cc.vi.300.vec.gz’ saved [1235219084/1235219084]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KLir27wv7quo"
      },
      "source": [
        "#extract fasttext\n",
        "'''\n",
        "import gzip\n",
        "import shutil\n",
        "with gzip.open('cc.vi.300.vec.gz', 'rb') as f_in:\n",
        "    with open('cc.vi.300.vec', 'wb') as f_out:\n",
        "        shutil.copyfileobj(f_in, f_out)\n",
        "        '''"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DU3PFGyr5QYR",
        "outputId": "47820ffc-f973-439d-85ad-a868d3d222c6"
      },
      "source": [
        "#download phow2v\n",
        "#! wget https://public.vinai.io/word2vec_vi_words_300dims.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-12-07 06:33:47--  https://public.vinai.io/word2vec_vi_words_300dims.zip\n",
            "Resolving public.vinai.io (public.vinai.io)... 99.86.35.23, 99.86.35.91, 99.86.35.72, ...\n",
            "Connecting to public.vinai.io (public.vinai.io)|99.86.35.23|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 2315583928 (2.2G) [application/zip]\n",
            "Saving to: ‘word2vec_vi_words_300dims.zip’\n",
            "\n",
            "word2vec_vi_words_3 100%[===================>]   2.16G  14.7MB/s    in 2m 33s  \n",
            "\n",
            "2020-12-07 06:36:23 (14.4 MB/s) - ‘word2vec_vi_words_300dims.zip’ saved [2315583928/2315583928]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U4_v0lzx505z"
      },
      "source": [
        "'''\n",
        "from zipfile import ZipFile\n",
        "zip = ZipFile('/content/word2vec_vi_words_300dims.zip')\n",
        "zip.extractall()\n",
        "'''"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HisAcsMR51Kf",
        "outputId": "4c74e3b2-c2e4-437c-91a1-329f3fe4cc5c"
      },
      "source": [
        "#download phow2v syllables\n",
        "#! wget https://public.vinai.io/word2vec_vi_syllables_300dims.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-12-07 06:39:53--  https://public.vinai.io/word2vec_vi_syllables_300dims.zip\n",
            "Resolving public.vinai.io (public.vinai.io)... 99.86.35.91, 99.86.35.98, 99.86.35.23, ...\n",
            "Connecting to public.vinai.io (public.vinai.io)|99.86.35.91|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1422871284 (1.3G) [application/zip]\n",
            "Saving to: ‘word2vec_vi_syllables_300dims.zip’\n",
            "\n",
            "word2vec_vi_syllabl 100%[===================>]   1.32G  14.6MB/s    in 96s     \n",
            "\n",
            "2020-12-07 06:41:30 (14.1 MB/s) - ‘word2vec_vi_syllables_300dims.zip’ saved [1422871284/1422871284]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ebnUmThp6K4h"
      },
      "source": [
        "'''\n",
        "zip = ZipFile('word2vec_vi_syllables_300dims.zip')\n",
        "zip.extractall()\n",
        "'''"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o0k3sBHU64-1"
      },
      "source": [
        "train=pd.read_csv('/content/drive/MyDrive/BERT/SA/VSLP_data/public_train.csv')\n",
        "train['post_message']=train['post_message'].fillna('none')\n",
        "public_test = pd.read_csv('/content/drive/MyDrive/BERT/SA/VSLP_data/public_test.csv')\n",
        "private_test = pd.read_csv('/content/drive/MyDrive/BERT/SA/VSLP_data/final_private_test_dropped_no_label - final_private_test_dropped_no_label.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UGgNvYlW9mZq"
      },
      "source": [
        "X_train = train[\"post_message\"].fillna(\"none\").values\n",
        "y_train = train[['label']].values\n",
        "X_test = private_test[\"post_message\"].fillna(\"none\").values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jYejwPor9OE8"
      },
      "source": [
        "def preprocess(text):\n",
        "    text = text.split(\" \")\n",
        "    return text\n",
        "\n",
        "max_features =  30000\n",
        "maxlen = 500\n",
        "embed_size = 300\n",
        "batch_size = 32\n",
        "epochs = 5\n",
        "l2_reg = .00001\n",
        "filter_sizes = [3,4,5]\n",
        "num_filters = 32\n",
        "EMBEDDING_FILE = 'cc.vi.300.vec'\n",
        "#EMBEDDING_FILE = 'word2vec_vi_words_300dims.txt' \n",
        "#you can change path here to load another word embedding pretrained model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ec2sGgUl_MZN"
      },
      "source": [
        "def recall_m(y_true, y_pred):\n",
        "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
        "    recall = true_positives / (possible_positives + K.epsilon())\n",
        "    return recall\n",
        "\n",
        "def precision_m(y_true, y_pred):\n",
        "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
        "    precision = true_positives / (predicted_positives + K.epsilon())\n",
        "    return precision\n",
        "\n",
        "def f1_m(y_true, y_pred):\n",
        "    precision = precision_m(y_true, y_pred)\n",
        "    recall = recall_m(y_true, y_pred)\n",
        "    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h5qoS6Km_Uix"
      },
      "source": [
        "tokenizer = text.Tokenizer(num_words=max_features)\n",
        "tokenizer.fit_on_texts(list(X_train) + list(X_test)) \n",
        "X_train = tokenizer.texts_to_sequences(X_train)\n",
        "X_test = tokenizer.texts_to_sequences(X_test)\n",
        "x_train = sequence.pad_sequences(X_train, maxlen=maxlen)\n",
        "x_test = sequence.pad_sequences(X_test, maxlen=maxlen)\n",
        "\n",
        "\n",
        "embeddings_index = {}\n",
        "with open(EMBEDDING_FILE, encoding='utf8') as f:\n",
        "    for line in f:\n",
        "        values = line.rstrip().rsplit(' ')\n",
        "        word = values[0]\n",
        "        coefs = np.asarray(values[1:], dtype='float32')\n",
        "        embeddings_index[word] = coefs\n",
        "\n",
        "word_index = tokenizer.word_index\n",
        "num_words = len(word_index) + 1\n",
        "embedding_matrix = np.zeros((num_words, embed_size))\n",
        "\n",
        "max_features = num_words\n",
        "\n",
        "for word, i in word_index.items():\n",
        "    if i >= max_features:\n",
        "        continue\n",
        "\n",
        "    embedding_vector = embeddings_index.get(word)\n",
        "    if embedding_vector is not None:\n",
        "        embedding_matrix[i] = embedding_vector"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p7FHc-FUQEzO"
      },
      "source": [
        "from keras.callbacks import ModelCheckpoint, TensorBoard, EarlyStopping\n",
        "\n",
        "import tensorflow as tf\n",
        "\n",
        "X_tra, X_val, y_tra, y_val = train_test_split(x_train, y_train, train_size=0.8, random_state=233)\n",
        "\n",
        "checkpoint = ModelCheckpoint('w2v_balanced_v1.hdf5',\n",
        "                             monitor='val_acc',\n",
        "                             save_best_only=True,\n",
        "                             mode='max',\n",
        "                             verbose=1)\n",
        "\n",
        "def get_model():\n",
        "    inp = Input(shape=(maxlen,))\n",
        "    x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\n",
        "    x = SpatialDropout1D(0.3)(x)\n",
        "    x = Reshape((maxlen, embed_size, 1))(x)\n",
        "\n",
        "    conv_0 = Conv2D(num_filters, kernel_size=(filter_sizes[0], embed_size), kernel_initializer='normal',\n",
        "                    activation='elu')(x)\n",
        "    conv_1 = Conv2D(num_filters, kernel_size=(filter_sizes[1], embed_size), kernel_initializer='normal',\n",
        "                    activation='elu')(x)\n",
        "    conv_2 = Conv2D(num_filters, kernel_size=(filter_sizes[2], embed_size), kernel_initializer='normal',\n",
        "                    activation='elu')(x)\n",
        "\n",
        "    maxpool_0 = MaxPool2D(pool_size=(maxlen - filter_sizes[0] + 1, 1))(conv_0)\n",
        "    maxpool_1 = MaxPool2D(pool_size=(maxlen - filter_sizes[1] + 1, 1))(conv_1)\n",
        "    maxpool_2 = MaxPool2D(pool_size=(maxlen - filter_sizes[2] + 1, 1))(conv_2)\n",
        "\n",
        "    z = Concatenate(axis=1)([maxpool_0, maxpool_1, maxpool_2])\n",
        "    z = Flatten()(z)\n",
        "    z = Dropout(0.35)(z)\n",
        "    outp = Dense(1, activation=\"sigmoid\")(z)\n",
        "\n",
        "  \n",
        "\n",
        "    model = Model(inputs=inp, outputs=outp)\n",
        "    model.compile(loss='binary_crossentropy',\n",
        "              optimizer='adam',\n",
        "              metrics=['acc',f1_m,precision_m, recall_m])\n",
        "\n",
        "    return model\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dcNQJU6B5UwS",
        "outputId": "470984b6-cb4e-4003-b3a1-74f03fe787ac"
      },
      "source": [
        "model = get_model()\n",
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"functional_5\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_6 (InputLayer)            [(None, 500)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding_5 (Embedding)         (None, 500, 300)     5555100     input_6[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout1d_3 (SpatialDro (None, 500, 300)     0           embedding_5[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "reshape_3 (Reshape)             (None, 500, 300, 1)  0           spatial_dropout1d_3[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_9 (Conv2D)               (None, 498, 1, 32)   28832       reshape_3[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_10 (Conv2D)              (None, 497, 1, 32)   38432       reshape_3[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_11 (Conv2D)              (None, 496, 1, 32)   48032       reshape_3[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_9 (MaxPooling2D)  (None, 1, 1, 32)     0           conv2d_9[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_10 (MaxPooling2D) (None, 1, 1, 32)     0           conv2d_10[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_11 (MaxPooling2D) (None, 1, 1, 32)     0           conv2d_11[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_3 (Concatenate)     (None, 3, 1, 32)     0           max_pooling2d_9[0][0]            \n",
            "                                                                 max_pooling2d_10[0][0]           \n",
            "                                                                 max_pooling2d_11[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "flatten_3 (Flatten)             (None, 96)           0           concatenate_3[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "dropout_3 (Dropout)             (None, 96)           0           flatten_3[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "dense_3 (Dense)                 (None, 1)            97          dropout_3[0][0]                  \n",
            "==================================================================================================\n",
            "Total params: 5,670,493\n",
            "Trainable params: 5,670,493\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MXr85dmnBEgE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "251cf9fd-097c-4b52-8d05-0c1d22174cfd"
      },
      "source": [
        "history = model.fit(X_tra, y_tra, validation_data=(X_val, y_val), batch_size=batch_size, epochs=epochs, verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/3\n",
            "122/122 [==============================] - 92s 755ms/step - loss: 0.4320 - acc: 0.8347 - f1_m: 0.0617 - precision_m: 0.1510 - recall_m: 0.0423 - val_loss: 0.3698 - val_acc: 0.8388 - val_f1_m: 0.0988 - val_precision_m: 0.2903 - val_recall_m: 0.0608\n",
            "Epoch 2/3\n",
            "122/122 [==============================] - 92s 752ms/step - loss: 0.3228 - acc: 0.8647 - f1_m: 0.3934 - precision_m: 0.6717 - recall_m: 0.3095 - val_loss: 0.3289 - val_acc: 0.8552 - val_f1_m: 0.2983 - val_precision_m: 0.5070 - val_recall_m: 0.2345\n",
            "Epoch 3/3\n",
            "122/122 [==============================] - 92s 752ms/step - loss: 0.2494 - acc: 0.8970 - f1_m: 0.5844 - precision_m: 0.8108 - recall_m: 0.4917 - val_loss: 0.3047 - val_acc: 0.8706 - val_f1_m: 0.4903 - val_precision_m: 0.6862 - val_recall_m: 0.4136\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bf-gi2JW96Zh"
      },
      "source": [
        "from keras.models import load_model\n",
        "y_pred = model.predict(x_test, batch_size=32)\n",
        "\n",
        "model_json = model.to_json()\n",
        "with open(\"cnn_.json\", \"w\") as json_file:\n",
        "    json_file.write(model_json)\n",
        "\n",
        "model.save('cnn_.h5')\n",
        "#new_model = load_model('/content/drive/MyDrive/VSLP_ReINTEL/text cnn/CNN.h5')\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5sh40sBT17L-"
      },
      "source": [
        "y_pred = model.predict(x_test, batch_size=32)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hvyX3khiBXA3"
      },
      "source": [
        "results = pd.DataFrame({'id':private_test['id']})\n",
        "results['probability']=y_pred\n",
        "results.to_csv('results.csv',index=False,header=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a9DxCHZvBYAc"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}