Label2Glove_Embedding.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Label2Glove_Embedding.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOgmINTMoGSEMrpt3vQ2g1Z",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/stxupengyu/Milti-Label-Classification-Data-Preprocessing/blob/master/Label2Glove_Embedding.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KjuCh5cTZNRu",
        "outputId": "df03e5a4-ad1e-4abb-ccc6-5c71d7d2e349",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "#挂载云盘 \n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0x1qkkh8ZNPS",
        "outputId": "0277439b-b1a9-4506-c125-4c1502c9e570",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 183
        }
      },
      "source": [
        "#进入项目目录\n",
        "print('original document')\n",
        "!ls\n",
        "import os\n",
        "os.chdir('/content/drive/My Drive/colab/XML')\n",
        "print('current document')\n",
        "!ls"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "original document\n",
            "EURLex.py\t   glove.6B.50d.txt\t      rcv_embedding.npz\n",
            "glove.6B.100d.txt  glove.6B.zip\t\t      XML-Glove-Embedding.ipynb\n",
            "glove.6B.200d.txt  rcv1.topics.hier.orig.txt\n",
            "glove.6B.300d.txt  rcv_embedding_mean.npz\n",
            "current document\n",
            "EURLex.py\t   glove.6B.50d.txt\t      rcv_embedding.npz\n",
            "glove.6B.100d.txt  glove.6B.zip\t\t      XML-Glove-Embedding.ipynb\n",
            "glove.6B.200d.txt  rcv1.topics.hier.orig.txt\n",
            "glove.6B.300d.txt  rcv_embedding_mean.npz\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8GMndyP3apgn",
        "outputId": "270279c0-cb2b-4042-ee5e-37e1dda96747",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 83
        }
      },
      "source": [
        "#下载glove\n",
        "#!wget http://nlp.stanford.edu/data/glove.6B.zip\n",
        "#!ls\n",
        "#!unzip glove.6B.zip\n",
        "import numpy as np\n",
        "import re\n",
        "import string\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "nltk.download('wordnet')\n",
        "from nltk.stem import WordNetLemmatizer \n",
        "from nltk.corpus import stopwords \n",
        "\n",
        "glove_dim=300\n",
        "data_path='rcv1.topics.hier.orig.txt'\n",
        "\n",
        "def body_remove_sw(sentence):\n",
        "    #去除stopwords 并把句子转换成单词的列表\n",
        "    words= [w for w in sentence.split(' ') if w not in sw]\n",
        "    return words\n",
        "def body_wnl(words):\n",
        "    #做词形还原\n",
        "    words=[wnl.lemmatize(word) for word in words]\n",
        "    return words\n",
        "def preprocessing(sentence,del_estr):\n",
        "  #处理每个原始标签\n",
        "    sentence=sentence.lower()#字母小写化\n",
        "    replace = \" \"*len(del_estr)\n",
        "    tran_tab = str.maketrans(del_estr, replace)\n",
        "    sentence = sentence.translate(tran_tab)#完成上述去除标点符号的功能\n",
        "    words=sentence.split(' ')#根据空格把句子分隔成单词\n",
        "    words=body_remove_sw(sentence)#去除停用词\n",
        "    while '' in words:\n",
        "      words.remove('')#去除为空格的空值\n",
        "    words=body_wnl(words)#词性还原\n",
        "    return words\n",
        "def read_data(data_path):\n",
        "  #读取原始数据，得到处理好的标签列表\n",
        "    iFile = open(data_path, \"r\")\n",
        "    label=[]\n",
        "    wnl = WordNetLemmatizer() \n",
        "    sw=stopwords.words('english')#这两步都是在循环前调用为了加快速度\n",
        "    del_estr = string.punctuation + string.digits+string.whitespace  # 去除ASCII 标点符号，数字\n",
        "    for line in iFile.readlines():\n",
        "        lineVec = line.strip().split(\":\")#根据:将一行分割\n",
        "        label_i=lineVec[-1]\n",
        "        label_i=preprocessing(label_i,del_estr)\n",
        "        label.append(label_i)\n",
        "    return label \n",
        "label=read_data(data_path)[1:]\n",
        "#label#展示一下处理好的label"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sSXSQ1SxwEJ4",
        "outputId": "06c03991-701d-4461-b168-f0ada26df519",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "def sum_tag_words(label):\n",
        "  #统计所有标签词\n",
        "  sum_tag=[]\n",
        "  for label_i in label:\n",
        "    for i in label_i:\n",
        "      if i not in sum_tag:\n",
        "        sum_tag.append(i)\n",
        "  return sum_tag\n",
        "def glove_dict(glove_dim):\n",
        "  #构建一个当前标签词在glove的embedding字典\n",
        "  all_tag_words=sum_tag_words(label)\n",
        "  print('使用的是 glove embedding dim=%d'%glove_dim)\n",
        "  #所有的标签词的list，用于从glove中抽取词表示\n",
        "  iFile = open(\"glove/glove.6B.%dd.txt\"%glove_dim, \"r\")#有119万个词\n",
        "  gloveDict = {}#glove word 对应id的字典\n",
        "  count=0\n",
        "  for line in iFile.readlines():\n",
        "      lineVec = line.strip().split(\" \")#将一行转化为一个列表\n",
        "      if lineVec[0] in all_tag_words:\n",
        "        gloveDict[lineVec[0]]=lineVec[1:]\n",
        "  print('%d个tag词有%d有glove embedding'%(len(all_tag_words),len(gloveDict)))\n",
        "  return gloveDict\n",
        "def str2float(str_list):\n",
        "  #将glove的词向量权重从str转化为float np.array\n",
        "  record=[]\n",
        "  for i in str_list:\n",
        "    record.append(float(i))\n",
        "  return np.array(record)\n",
        "def many2one(label_embedding_i):\n",
        "  #将多个词的embediding合并为二维array，并取均值\n",
        "  label_embedding=label_embedding_i[0]\n",
        "  for inst in label_embedding_i[1:]:\n",
        "    label_embedding=np.vstack((label_embedding,inst))\n",
        "  if len(label_embedding.shape)>1:\n",
        "    label_embedding=np.mean(label_embedding,axis=0)#均值操作\n",
        "  return label_embedding\n",
        "def glove_embedding(label):\n",
        "  #将原始标签表示为embedding\n",
        "  label_embedding_list=[]\n",
        "  for label_i in label:\n",
        "    label_embedding_i=[]\n",
        "    for word in label_i:\n",
        "      label_embedding_i.append(str2float(glovedict[word]))\n",
        "    label_embedding_i=many2one(label_embedding_i)\n",
        "    label_embedding_list.append(label_embedding_i)\n",
        "  return label_embedding_list\n",
        "glovedict=glove_dict(glove_dim)\n",
        "label_embedding=glove_embedding(label)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "使用的是 glove embedding dim=300\n",
            "139个tag词有139有glove embedding\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MdzQYy5o--3p",
        "outputId": "123587c8-0a6b-4a37-adcc-c56dbcab6f74",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "file_name='rcv_embedding_mean.npz'\n",
        "#file_name='rcv_embedding.npz'\n",
        "def save_data(label_embedding):\n",
        "  label_embedding=np.array(label_embedding)\n",
        "  np.savez(file_name,label_embedding=label_embedding,allow_pickle=True)\n",
        "  print('数据集已经保存成功在文件%s中！'%(file_name))\n",
        "def load_data():\n",
        "  data=np.load(file_name,allow_pickle=True)\n",
        "  label_embedding=data['label_embedding']\n",
        "  return label_embedding\n",
        "save_data(label_embedding)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "数据集已经保存成功在文件rcv_embedding_mean.npz中！\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6PBtuS_K--8z"
      },
      "source": [
        "label_embedding=load_data()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SjG53FbH--_h",
        "outputId": "e6d6c160-2e5f-4537-e95d-2689048cf4dc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "label_embedding.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(103, 300)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 142
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ic7GBREH6J0r"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}