RNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 很简单的RNN模型，没有GPU跑得我天荒地老，随便找个模型结构直接跑了。。。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import gc\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import log_loss\n",
    "%matplotlib inline\n",
    "\n",
    "from keras.models import Sequential, Model,load_model\n",
    "from keras.layers import Bidirectional,Convolution1D,MaxPool1D,Flatten, TimeDistributed,RepeatVector\n",
    "from keras.layers import SimpleRNN, GRU, LSTM, Conv1D, MaxPooling1D, GlobalAveragePooling1D, GlobalMaxPooling1D,SpatialDropout1D\n",
    "from keras.layers import Dense, Activation, Reshape, Dropout, BatchNormalization, Input,concatenate\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.optimizers import RMSprop, Adam\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from keras import initializers\n",
    "from keras import backend as K\n",
    "from tensorflow import set_random_seed\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "set_random_seed(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data = pd.read_csv('input/train.csv')\n",
    "test_data = pd.read_csv('input/test.csv')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 编码 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "apis = list(set(list(train_data.api.unique())+list(test_data.api.unique())))\n",
    "enc = LabelEncoder().fit(apis)\n",
    "train_data['enc'] = enc.transform(train_data.api)\n",
    "test_data['enc'] = enc.transform(test_data.api)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tr = train_data.groupby('file_id').enc.apply(list).reset_index()\n",
    "te = test_data.groupby('file_id').enc.apply(list).reset_index()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "label = train_data.groupby(['file_id'])['label'].agg('first').reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.utils.np_utils import to_categorical\n",
    "\n",
    "categorical_labels = to_categorical(label.label, num_classes=None)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('feature/df_train_v4.csv')\n",
    "df_test = pd.read_csv('feature/df_test_v4.csv')\n",
    "\n",
    "df_train = pd.concat([df_train, pd.get_dummies(df_train.most_common_classes, prefix='most_common_classes')], axis=1)\n",
    "df_test = pd.concat([df_test, pd.get_dummies(df_test.most_common_classes, prefix='most_common_classes')], axis=1)\n",
    "\n",
    "df_train.drop('most_common_classes', axis=1, inplace=True)\n",
    "df_test.drop('most_common_classes', axis=1, inplace=True)\n",
    "\n",
    "cols = [c for c in df_train.columns if c not in ['file_id', 'label']]\n",
    "\n",
    "df_train.fillna(0, inplace=True)\n",
    "df_test.fillna(0, inplace=True)\n",
    "\n",
    "scaler = MinMaxScaler().fit(df_train[cols])\n",
    "mm_train = scaler.transform(df_train[cols])\n",
    "\n",
    "mm_test = scaler.transform(df_test[cols])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 由于时间及内存关系只取整个进程的前5000个api\n",
    "data = tr.enc.values\n",
    "\n",
    "data = pad_sequences(data, 5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## 测试集数据\n",
    "te_data = te.enc.values\n",
    "\n",
    "te_data = pad_sequences(te_data, 7000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 划分验证集 8-2分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_val, y_train, y_val = train_test_split(data, categorical_labels, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_train, num_val, y_train, y_val = train_test_split(mm_train, categorical_labels, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RNN模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_model():\n",
    "    models = []\n",
    "    inp = Input(shape=(5000, ))\n",
    "    x = Embedding(max_enc+1, 50)(inp)\n",
    "    x = Bidirectional(GRU(80。))(x)\n",
    "    avg_pool = GlobalAveragePooling1D()(x)\n",
    "    max_pool = GlobalMaxPooling1D()(x)\n",
    "    conc = concatenate([avg_pool, max_pool])\n",
    "    models.append(conc)\n",
    "    \n",
    "    num_inp = Input(shape=(len(cols),) )\n",
    "    x_num = Dense(256, activation=\"relu\")(num_inp)\n",
    "    x_num = Dense(64, activation=\"relu\")(x_num)\n",
    "    models.append(x_num)\n",
    "    \n",
    "    all_inp = concatenate(models)\n",
    "    outp = Dense(6, activation=\"softmax\")(all_inp)\n",
    "    \n",
    "    model = Model(inputs=[inp, num_inp], outputs=outp)\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer='adam')\n",
    "\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 93299 samples, validate on 23325 samples\n",
      "Epoch 1/20\n",
      "93299/93299 [==============================] - 9783s 105ms/step - loss: 0.1486 - val_loss: 0.0683\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.06827, saving model to /tmp/lstm_model.h5\n",
      "Epoch 2/20\n",
      "93299/93299 [==============================] - 9774s 105ms/step - loss: 0.0489 - val_loss: 0.0439\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.06827 to 0.04392, saving model to /tmp/lstm_model.h5\n",
      "Epoch 3/20\n",
      "93299/93299 [==============================] - 9758s 105ms/step - loss: 0.0328 - val_loss: 0.0345\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.04392 to 0.03450, saving model to /tmp/lstm_model.h5\n",
      "Epoch 4/20\n",
      "93299/93299 [==============================] - 9764s 105ms/step - loss: 0.0257 - val_loss: 0.0303\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.03450 to 0.03026, saving model to /tmp/lstm_model.h5\n",
      "Epoch 5/20\n",
      "93299/93299 [==============================] - 9763s 105ms/step - loss: 0.0216 - val_loss: 0.0279\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.03026 to 0.02787, saving model to /tmp/lstm_model.h5\n",
      "Epoch 6/20\n",
      "93299/93299 [==============================] - 9767s 105ms/step - loss: 0.0185 - val_loss: 0.0258\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.02787 to 0.02583, saving model to /tmp/lstm_model.h5\n",
      "Epoch 7/20\n",
      "93299/93299 [==============================] - 9750s 105ms/step - loss: 0.0165 - val_loss: 0.0264\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.02583\n",
      "Epoch 00007: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fabb761ce10>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = get_model()\n",
    "\n",
    "model.fit([X_train, num_train], y_train,\n",
    "          batch_size=256,\n",
    "          epochs=20,\n",
    "          validation_data=([X_val, num_val], y_val),\n",
    "          verbose=1,\n",
    "          callbacks=[\n",
    "              EarlyStopping(monitor='val_loss', patience=1, verbose=1, mode='min'),\n",
    "              ModelCheckpoint('/tmp/lstm_model.h5', monitor='val_loss', verbose=1, save_best_only=True, mode='min', period=1), \n",
    "\n",
    "              ]\n",
    "         )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.load_weights('/tmp/lstm_model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = model.predict([X_val, num_val])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### 计算loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.025828115257083836"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_loss(y_val, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_sub = model.predict([te_data, mm_test])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 储存提交文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_id</th>\n",
       "      <th>prob0</th>\n",
       "      <th>prob1</th>\n",
       "      <th>prob2</th>\n",
       "      <th>prob3</th>\n",
       "      <th>prob4</th>\n",
       "      <th>prob5</th>\n",
       "      <th>sum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [file_id, prob0, prob1, prob2, prob3, prob4, prob5, sum]\n",
       "Index: []"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub = pd.read_csv('input/3rd_security_submit_sample.csv')\n",
    "\n",
    "sub.loc[:, ['prob0','prob1','prob2','prob3','prob4','prob5']] = y_sub\n",
    "\n",
    "sub = sub.round(7)\n",
    "\n",
    "sub['sum'] = sub[['prob0','prob1','prob2','prob3','prob4','prob5']].sum(1)\n",
    "\n",
    "sub['prob0'] = sub['prob0'] / sub['sum']\n",
    "sub['prob1'] = sub['prob1'] / sub['sum']\n",
    "sub['prob2'] = sub['prob2'] / sub['sum']\n",
    "sub['prob3'] = sub['prob3'] / sub['sum']\n",
    "sub['prob4'] = sub['prob4'] / sub['sum']\n",
    "sub['prob5'] = sub['prob5'] / sub['sum']\n",
    "\n",
    "sub.head()\n",
    "\n",
    "sub[(sub[['prob0','prob1','prob2','prob3','prob4','prob5']].sum(1) - 1) >= 1e-6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub[['file_id', 'prob0','prob1','prob2','prob3','prob4','prob5']].to_csv('rnn_02582.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save('rnn_y_pred_02582.npy', y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}