diff --git a/Attention_Improved_LSTM.ipynb b/Attention_Improved_LSTM.ipynb new file mode 100644 index 0000000..dc27ffa --- /dev/null +++ b/Attention_Improved_LSTM.ipynb @@ -0,0 +1,774 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "Attention-Improved-LSTM.ipynb", + "provenance": [], + "collapsed_sections": [], + "include_colab_link": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "4kPsSo5aMU4w" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "BQWwdUDbhmQQ", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 106 + }, + "outputId": "622de1c2-6981-4642-ec04-f28a508c0f06" + }, + "source": [ + "! pip install jsonlines " + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Collecting jsonlines\n", + " Downloading https://files.pythonhosted.org/packages/4f/9a/ab96291470e305504aa4b7a2e0ec132e930da89eb3ca7a82fbe03167c131/jsonlines-1.2.0-py2.py3-none-any.whl\n", + "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from jsonlines) (1.15.0)\n", + "Installing collected packages: jsonlines\n", + "Successfully installed jsonlines-1.2.0\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "yJTPJoZjYr4H" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "_p7CDRFuMDgN", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 35 + }, + "outputId": "61bd0dc9-99a8-4a7d-9e22-99e8d5ef82c7" + }, + "source": [ + "from google.colab import drive \n", + "drive.mount('/content/gdrive')" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Mounted at /content/gdrive\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "0CN9EfHpMQDp", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 35 + }, + "outputId": "9df98dd6-81ed-4b18-ac76-302459663535" + }, + "source": [ + "cd /content/gdrive/'My Drive'/'Toxic Spam'" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/content/gdrive/My Drive/Toxic Spam\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "UOKYhK3aMZHh" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "LwxOtHwGMCfi" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "wUQ0f12FLQ21" + }, + "source": [ + "# Libraries\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", + "import numpy as np\n", + "import re\n", + "import torch\n", + "import json\n", + "import torch.nn.functional as F\n", + "# Preliminaries\n", + "\n", + "from torchtext.data import Field, TabularDataset, BucketIterator\n", + "\n", + "# Models\n", + "\n", + "import torch.nn as nn\n", + "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n", + "\n", + "# Training\n", + "\n", + "import torch.optim as optim\n", + "\n", + "# Evaluation\n", + "\n", + "from sklearn.metrics import accuracy_score, classification_report, confusion_matrix,f1_score\n", + "import seaborn as sns" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "JJKehd18ORO0" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "PBo-ygVPeF2O" + }, + "source": [ + "data.head()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "zoQugFzSeGVD", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 35 + }, + "outputId": "48d403f7-79eb-445e-f87f-6e91df47fc36" + }, + "source": [ + "data_train=pd.read_pickle('./processed/finaltrain.pkl')\n", + "data_val=pd.read_pickle('./processed/finaldev.pkl')\n", + "print(len(data_train),len(data_val))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "6351 794\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "gFc_cxqqC2Gr", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "outputId": "7ccaf934-7dd1-4da7-aef9-9d1b3345629d" + }, + "source": [ + "###IGNORE THIS CELLLLL\n", + "# y=len(data)\n", + "# data=data.sample(frac=1)\n", + "# train_data= data[:int(8*y/10)]\n", + "# val_data=data[int(8*y/10):int(9*y/10)]\n", + "# test_data=data[int(9*y/10):]\n", + "# print(len(train_data),len(val_data),len(test_data))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "6351 794 794\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "OG-BV3d4SasL", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 72 + }, + "outputId": "819bb0e6-d83c-4297-bf79-a49c17839505" + }, + "source": [ + "X_train=data_train['token_final'].to_list()\n", + "Y_train=data_train['target_final'].to_list()\n", + "X_val=data_val['token_final'].tolist()\n", + "Y_val=data_val['target_final'].tolist()\n", + "print(X_train[0],Y_train[0])\n", + "print(X_val[0],Y_val[0])\n", + "\n", + "\n", + "train_data = []\n", + "for i in range(len(X_train)):\n", + " train_data.append({\"text\" : X_train[i], \"label\" : Y_train[i]})\n", + "\n", + "with jsonlines.open('./processed/train.json', mode='w') as f:\n", + " f.write_all(train_data)\n", + "\n", + "val_data = []\n", + "for i in range(len(X_val)):\n", + " val_data.append({\"text\" : X_val[i], \"label\" : Y_val[i]})\n", + "\n", + "with jsonlines.open('./processed/val.json', mode='w') as f:\n", + " f.write_all(val_data)\n" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "['only', 'a', 'fool', 'would', 'not', 'be', 'against', 'this', 'president', 'he', 'is', 'massively', 'and', 'obviously', 'unfit', 'for', 'office'] [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", + "['fucking', 'leftist', 'hebes', 'always', 'finding', 'the', 'dirty', 'way', 'to', 'find', 'dirt', 'and', 'make', 'it', 'salacious'] [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "HPCHVfQMd4BS" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "e_xbVnaBSIIW" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "e4GQrcARihBb" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "O_3617xt8yEZ" + }, + "source": [ + "class Corpus(object):\n", + " def __init__(self, input_folder, min_word_freq, batch_size,device):\n", + " # list all the fields\n", + " self.word_field = Field(lower=True)\n", + " self.label_field = Field(unk_token=None,pad_token=None)\n", + " # create dataset using built-in parser from torchtext\n", + " self.train_data, self.val_data = TabularDataset.splits(\n", + " path=input_folder,\n", + " train=\"train.json\",\n", + " validation=\"val.json\",\n", + " format=\"json\",\n", + " fields={'text':(\"text\", self.word_field),'label':(\"label\",self.label_field)}\n", + " )\n", + " # convert fields to vocabulary list\n", + " # \n", + " self.word_field.build_vocab(self.train_data.text, vectors='glove.twitter.27B.200d', unk_init=torch.Tensor.normal_,min_freq=min_word_freq)\n", + " self.label_field.build_vocab(self.train_data.label)\n", + " # create iterator for batch input\n", + " self.train_iter, self.val_iter = BucketIterator.splits(\n", + " datasets=(self.train_data, self.val_data),\n", + " batch_size=batch_size,\n", + " sort_within_batch = True,\n", + " sort_key = lambda x: len(x.text),\n", + " device=device\n", + " )\n", + " # prepare padding index to be ignored during model training/evaluation\n", + " self.word_pad_idx = self.word_field.vocab.stoi[self.word_field.pad_token]\n", + " # self.label_pad_idx = self.tag_field.vocab.stoi[self.label_field.pad_token]" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "00AZVg2_dH7Y", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "outputId": "4a57988b-da86-4957-e2f7-3e4f95d47a86" + }, + "source": [ + "device2 = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", + "corpus = Corpus(\n", + " input_folder=\"./processed/\",\n", + " min_word_freq=2, \n", + " batch_size=64,\n", + " device = device2\n", + ")\n", + "print(f\"Train set: {len(corpus.train_data)} sentences\")\n", + "print(f\"Val set: {len(corpus.val_data)} sentences\")" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Train set: 6351 sentences\n", + "Val set: 794 sentences\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "FjmIIMV_dqIc", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 35 + }, + "outputId": "c842ee79-7fb3-4059-c6fc-6fadcb5606c4" + }, + "source": [ + "print(len(corpus.word_field.vocab))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "8815\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "PdB7-yaNgKyX" + }, + "source": [ + "class BiLSTM(nn.Module):\n", + " def attention_net(self, a, b):\n", + " self.weights1 = torch.bmm(a,b.reshape(1, -1,self.hidden_dim*2).squeeze(0).unsqueeze(2)).squeeze(2)\n", + " self.weights2 = F.softmax(self.weights1).unsqueeze(2)\n", + " return torch.bmm(a.transpose(1,2), self.weights2).squeeze(2)\n", + " def __init__(self, input_dim, embedding_dim, hidden_dim, output_dim, lstm_layers,\n", + " emb_dropout, lstm_dropout, fc_dropout, word_pad_idx):\n", + " super().__init__()\n", + " self.embedding_dim = embedding_dim\n", + " self.word_pad_idx = word_pad_idx\n", + " self.hidden_dim=hidden_dim\n", + " # LAYER 1: Embedding\n", + " self.embedding = nn.Embedding(\n", + " num_embeddings=input_dim, \n", + " embedding_dim=embedding_dim, \n", + " padding_idx=word_pad_idx\n", + " )\n", + " self.emb_dropout = nn.Dropout(emb_dropout)\n", + " # LAYER 2: BiLSTM\n", + " self.lstm = nn.LSTM(\n", + " input_size=embedding_dim,\n", + " hidden_size=hidden_dim,\n", + " num_layers=lstm_layers,\n", + " bidirectional=True,\n", + " dropout=lstm_dropout if lstm_layers > 1 else 0\n", + " )\n", + " #LAYER 3: Attention\n", + " \n", + " # LAYER 4: Fully-connected\n", + " self.fc_dropout = nn.Dropout(fc_dropout)\n", + " self.fc = nn.Linear(hidden_dim * 2, output_dim) # times 2 for bidirectional\n", + "\n", + "\n", + " def forward(self, sentence):\n", + " # sentence = [sentence length, batch size]\n", + " # embedding_out = [sentence length, batch size, embedding dim]\n", + " embedding_out = self.emb_dropout(self.embedding(sentence))\n", + " # lstm_out = [sentence length, batch size, hidden dim * 2]\n", + " lstm_out, _ = self.lstm(embedding_out)\n", + " # key_padding_mask = torch.as_tensor(sentence == self.word_pad_idx).permute(1, 0)\n", + " attn_output = self.attention_net(lstm_out, lstm_out)\n", + "\n", + " # toxic_out = [sentence length, batch size, output dim]\n", + " toxic_out = self.fc(self.fc_dropout(attn_output))\n", + "\n", + " toxic_out2=torch.sigmoid(toxic_out)\n", + " return toxic_out2\n", + "\n", + " def init_weights(self):\n", + " # to initialize all parameters from normal distribution\n", + " # helps with converging during training\n", + " for name, param in self.named_parameters():\n", + " nn.init.normal_(param.data, mean=0, std=0.1)\n", + "\n", + " def init_embeddings(self, word_pad_idx,pretrained=None):\n", + " # initialize embedding for padding as zero\n", + " self.embedding.weight.data[word_pad_idx] = torch.zeros(self.embedding_dim)\n", + " if pretrained is not None:\n", + " self.embedding = nn.Embedding.from_pretrained(\n", + " embeddings=torch.as_tensor(pretrained),\n", + " padding_idx=word_pad_idx,\n", + " freeze=True\n", + " )\n", + "\n", + " def count_parameters(self):\n", + " return sum(p.numel() for p in self.parameters() if p.requires_grad)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "q2-WqibW_EdU", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 158 + }, + "outputId": "429b0434-704d-47b2-c3b9-edff2c36c821" + }, + "source": [ + "bilstm = BiLSTM(\n", + " input_dim=len(corpus.word_field.vocab),\n", + " embedding_dim=200,\n", + " hidden_dim=64,\n", + " output_dim=1,\n", + " lstm_layers=2,\n", + " emb_dropout=0,\n", + " lstm_dropout=0,\n", + " fc_dropout=0, \n", + " word_pad_idx=corpus.word_pad_idx\n", + ")\n", + "bilstm.init_weights()\n", + "# bilstm.init_embeddings(word_pad_idx=corpus.word_pad_idx)\n", + "bilstm.init_embeddings(\n", + " word_pad_idx=corpus.word_pad_idx,\n", + " pretrained=corpus.word_field.vocab.vectors\n", + ")\n", + "print(f\"The model has {bilstm.count_parameters():,} trainable parameters.\")\n", + "print(bilstm)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "The model has 235,649 trainable parameters.\n", + "BiLSTM(\n", + " (embedding): Embedding(8815, 200, padding_idx=1)\n", + " (emb_dropout): Dropout(p=0, inplace=False)\n", + " (lstm): LSTM(200, 64, num_layers=2, bidirectional=True)\n", + " (fc_dropout): Dropout(p=0, inplace=False)\n", + " (fc): Linear(in_features=128, out_features=1, bias=True)\n", + ")\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "IO3079aOeof5", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 35 + }, + "outputId": "da8ec101-f292-4560-d2b1-ac3d984ab47e" + }, + "source": [ + "print(len(corpus.word_field.vocab))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "6203\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GLZRrEZRb1bf" + }, + "source": [ + "# TRAINER " + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "D8rvd6lwb-bO" + }, + "source": [ + "import time" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "TJaQFF461Emv" + }, + "source": [ + "class Toxicspan(object):\n", + "\n", + " def __init__(self, model, data, optimizer_cls, loss_fn_cls):\n", + " self.model = model\n", + " self.data = data\n", + " self.optimizer = optimizer_cls(model.parameters())\n", + " self.loss_fn = loss_fn_cls()\n", + "\n", + " def accuracy(self, preds, y):\n", + " # rounded_preds=preds\n", + " rounded_preds = torch.round(preds)\n", + " correct = (rounded_preds == y).float()\n", + " acc = correct.sum() / len(correct)\n", + " return(acc)\n", + " def sigmoid(self,x):\n", + " return 1/(1 + np.exp(-np.array(x)))\n", + " def f1score(self,preds,y):\n", + " # rounded_preds = torch.round(preds)\n", + " return f1_score(y,preds,average='binary') \n", + "\n", + " def epoch(self):\n", + " epoch_loss = 0\n", + " epoch_acc = 0\n", + " self.model.train()\n", + " batch_f1=[]\n", + " true_labels=[]\n", + " pred_labels=[]\n", + " for batch in self.data.train_iter:\n", + " text = batch.text\n", + " true_label = batch.label\n", + " self.optimizer.zero_grad()\n", + " pred_label = self.model(text)\n", + " # to calculate the loss and accuracy, we flatten both prediction and true tags\n", + " # flatten pred_tags to [sent len, batch size, output dim]\n", + " pred_label = pred_label.view(-1)\n", + " # flatten true_tags to [sent len * batch size]\n", + " true_label = true_label.view(-1)\n", + " # print(true_label.shape,pred_label.shape)\n", + " batch_loss = self.loss_fn(pred_label, true_label.type(torch.FloatTensor))\n", + " batch_acc = self.accuracy(pred_label, true_label)\n", + " # batch_f1.append(f1_score(true_label,torch.round(pred_label).detach().numpy(),average='binary'))\n", + " true_labels.extend(true_label)\n", + " with torch.no_grad():\n", + " pred_labels.extend((torch.round(pred_label)))\n", + " batch_loss.backward()\n", + " self.optimizer.step()\n", + " epoch_loss += batch_loss.item()\n", + " epoch_acc += batch_acc.item()\n", + " # print(pred_labels[0],true_labels[0])\n", + " # print(true_label,torch.round(pred_label))\n", + " # print(type(pred_labels[0]),type(true_labels[0]))\n", + " # print(true_labels[:10],pred_labels[0:10])\n", + " # batch_f1=0\n", + " batch_f1=self.f1score(pred_labels,true_labels)\n", + " return epoch_loss / len(self.data.train_iter), epoch_acc / len(self.data.train_iter), batch_f1\n", + "\n", + " def evaluate(self, iterator):\n", + " epoch_loss = 0\n", + " epoch_acc = 0\n", + " true_labels=[]\n", + " pred_labels=[]\n", + " self.model.eval()\n", + " with torch.no_grad():\n", + " # similar to epoch() but model is in evaluation mode and no backprop\n", + " for batch in iterator:\n", + " text = batch.text\n", + " true_label = batch.label\n", + " pred_label = self.model(text)\n", + " pred_label = pred_label.view(-1)\n", + " true_label = true_label.view(-1)\n", + " batch_loss = self.loss_fn(pred_label, true_label.type(torch.FloatTensor))\n", + " batch_acc = self.accuracy(pred_label, true_label)\n", + " epoch_loss += batch_loss.item()\n", + " epoch_acc += batch_acc.item()\n", + " pred_labels.extend(torch.round(pred_label))\n", + " true_labels.extend(true_label)\n", + " # print(true_labels[:10],pred_labels[0:10])\n", + " batch_f1=self.f1score(pred_labels,true_labels)\n", + " print(confusion_matrix(true_labels,pred_labels))\n", + " return epoch_loss / len(iterator), epoch_acc / len(iterator), batch_f1\n", + "\n", + " # main training sequence\n", + " def train(self, n_epochs):\n", + " for epoch in range(n_epochs):\n", + " start=time.time()\n", + " train_loss, train_acc, train_f1 = self.epoch()\n", + " final=time.time()\n", + " print(f\"\\tTrn Loss: {train_loss:.3f} | Trn Acc: {train_acc * 100:.2f}% | Trn F1score: {train_f1:.5f}\")\n", + " print(\"Epoch time:\",final-start)\n", + " val_loss, val_acc,val_f1 = self.evaluate(self.data.val_iter)\n", + " print(f\"\\tVal Loss: {val_loss:.3f} | Val Acc: {val_acc * 100:.2f}% | Val F1 :{val_f1:.5f}\")\n", + " # test_loss, test_acc = self.evaluate(self.data.test_iter)\n", + " # print(f\"Test Loss: {test_loss:.3f} | Test Acc: {test_acc * 100:.2f}%\")\n", + " " + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "dKS0swe78uMi", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 366 + }, + "outputId": "66a2c243-7075-4895-b52c-36115f4d1ac9" + }, + "source": [ + "# this will continue training if the model has been trained before.\n", + "# to restart training, run the bilstm creation cell (2 cells above) once again.\n", + "import torch.optim as optim\n", + "toxic = Toxicspan(\n", + " model=bilstm,\n", + " data=corpus,\n", + " optimizer_cls=optim.Adam,\n", + " loss_fn_cls=nn.BCEWithLogitsLoss\n", + ")\n", + "toxic.train(10)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "error", + "ename": "RuntimeError", + "evalue": "ignored", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0mloss_fn_cls\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mBCEWithLogitsLoss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m )\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0mtoxic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, n_epochs)\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_epochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 85\u001b[0m \u001b[0mstart\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 86\u001b[0;31m \u001b[0mtrain_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_acc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_f1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 87\u001b[0m \u001b[0mfinal\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 88\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"\\tTrn Loss: {train_loss:.3f} | Trn Acc: {train_acc * 100:.2f}% | Trn F1score: {train_f1:.5f}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mepoch\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0mtrue_label\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m \u001b[0mpred_label\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 33\u001b[0m \u001b[0;31m# to calculate the loss and accuracy, we flatten both prediction and true tags\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 34\u001b[0m \u001b[0;31m# flatten pred_tags to [sent len, batch size, output dim]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 720\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 721\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 722\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 723\u001b[0m for hook in itertools.chain(\n\u001b[1;32m 724\u001b[0m \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, sentence)\u001b[0m\n\u001b[1;32m 39\u001b[0m \u001b[0mlstm_out\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlstm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membedding_out\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 40\u001b[0m \u001b[0;31m# key_padding_mask = torch.as_tensor(sentence == self.word_pad_idx).permute(1, 0)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 41\u001b[0;31m \u001b[0mattn_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattention_net\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlstm_out\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlstm_out\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 42\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[0;31m# toxic_out = [sentence length, batch size, output dim]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mattention_net\u001b[0;34m(self, a, b)\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mBiLSTM\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mModule\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mattention_net\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweights1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbmm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden_dim\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden_dim\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweights2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweights1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbmm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweights2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mRuntimeError\u001b[0m: Expected tensor to have size 1 at dimension 1, but got size 128 for argument #2 'batch2' (while checking arguments for bmm)" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "8HpwiyrmNT5Z" + }, + "source": [ + "\n", + "\n" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "5BLr7X43Zpim" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/Evaluation.ipynb b/Evaluation.ipynb new file mode 100644 index 0000000..c9012ab --- /dev/null +++ b/Evaluation.ipynb @@ -0,0 +1,139 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "Evaluation.ipynb", + "provenance": [], + "mount_file_id": "1XQ3jC6LqASONst5TRme8aLYo--O2yxOn", + "authorship_tag": "ABX9TyMTaDmmAuCXzAnnqLkkPBQ/", + "include_colab_link": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "r0acXEA7zadc", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "2b346d93-1ffa-4740-ad5e-05e254f99b54" + }, + "source": [ + "from google.colab import drive\n", + "drive.mount('/content/drive')" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Mounted at /content/drive\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "TGgzdrg-55m6" + }, + "source": [ + "test_path = 'drive/MyDrive/Toxic span/devfinal.csv' # enter the path to csv where the test spans are available \n", + "pred_path='drive/MyDrive/Toxic span/valpredicted3.csv' # enter the path where is predicted.csv" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "jtmn72VIrQZk", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "a9f1cf65-0340-4908-fa17-d81dc7684faf" + }, + "source": [ + "import pandas as pd\n", + "import pickle\n", + "\n", + "from ast import literal_eval\n", + "\n", + "test = pd.read_csv(test_path)\n", + "test.spans = test.spans.apply(literal_eval)\n", + "\n", + "prediction = pd.read_csv(pred_path)\n", + "prediction.spans = prediction.spans.apply(literal_eval)\n", + "\n", + "\n", + "spans = test.spans.tolist() #list of lists\n", + "predicted = prediction.spans.tolist() #list of lists \n", + "\n", + "\n", + "#set1: predicted, set2: ground\n", + "\n", + "def precision_(set1, set2):\n", + " return len(set1.intersection(set2))/len(set1)\n", + "\n", + "def recall_(set1, set2):\n", + " return len(set1.intersection(set2))/len(set2)\n", + "\n", + "def f1_(set1, set2):\n", + " if (len(set1) == 0 and len(set2) == 0):\n", + " return 1\n", + " elif (len(set1) == 0 or len(set2) == 0):\n", + " return 0\n", + " else:\n", + " p = precision_(set1, set2)\n", + " r = recall_(set1, set2)\n", + " if (p+r) == 0:\n", + " return 0\n", + " return 2*p*r/(p+r) \n", + "\n", + "f1s = []\n", + "for i in range(len(spans)):\n", + " f1s.append(f1_(set(predicted[i]), set(spans[i])))\n", + "\n", + "print(sum(f1s)/len(f1s))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0.6933597933052192\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "g6ZyfT77wVRW" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/Finalprediction.ipynb b/Finalprediction.ipynb new file mode 100644 index 0000000..bed044b --- /dev/null +++ b/Finalprediction.ipynb @@ -0,0 +1,1552 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.3" + }, + "colab": { + "name": "Finalprediction.ipynb", + "provenance": [], + "collapsed_sections": [], + "include_colab_link": true + }, + "accelerator": "GPU", + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "eb30fae1db3c4d7981f314efaed43744": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "state": { + "_view_name": "HBoxView", + "_dom_classes": [], + "_model_name": "HBoxModel", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.5.0", + "box_style": "", + "layout": "IPY_MODEL_64587bf0800945c1aef9ab70facf92ca", + "_model_module": "@jupyter-widgets/controls", + "children": [ + "IPY_MODEL_7e63d5a523d14e9db305c967af8bcdee", + "IPY_MODEL_07c98439f7c241f98d6d4170f5b34795" + ] + } + }, + "64587bf0800945c1aef9ab70facf92ca": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "7e63d5a523d14e9db305c967af8bcdee": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "state": { + "_view_name": "ProgressView", + "style": "IPY_MODEL_044e6439d72d458f9a097a4e166b3110", + "_dom_classes": [], + "description": "Downloading: 100%", + "_model_name": "FloatProgressModel", + "bar_style": "success", + "max": 213450, + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": 213450, + "_view_count": null, + "_view_module_version": "1.5.0", + "orientation": "horizontal", + "min": 0, + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_b24a8208cf0e426eb2005f1dc3f9bb59" + } + }, + "07c98439f7c241f98d6d4170f5b34795": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "state": { + "_view_name": "HTMLView", + "style": "IPY_MODEL_2c939107cb0b428f885fa8a47fdebabc", + "_dom_classes": [], + "description": "", + "_model_name": "HTMLModel", + "placeholder": "​", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": " 213k/213k [00:15<00:00, 13.9kB/s]", + "_view_count": null, + "_view_module_version": "1.5.0", + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_a9255aa4e9ee4f5092aa37d4f3427a22" + } + }, + "044e6439d72d458f9a097a4e166b3110": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "ProgressStyleModel", + "description_width": "initial", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "bar_color": null, + "_model_module": "@jupyter-widgets/controls" + } + }, + "b24a8208cf0e426eb2005f1dc3f9bb59": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "2c939107cb0b428f885fa8a47fdebabc": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "DescriptionStyleModel", + "description_width": "", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "_model_module": "@jupyter-widgets/controls" + } + }, + "a9255aa4e9ee4f5092aa37d4f3427a22": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + } + } + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2HFdqstxfhdQ" + }, + "source": [ + "# First do the preprocessing on the test dataset......\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "PptsRx2mZZQ9", + "outputId": "50392fb3-69e2-4bff-ea38-1bd871f9cc39" + }, + "source": [ + "from google.colab import drive\n", + "drive.mount('/content/drive')" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Mounted at /content/drive\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "hf09MtzB2XHG" + }, + "source": [ + "###PATHS\n", + "\n", + "test_path ='drive/MyDrive/Final/test.csv' #will be used to open test.csv\n", + "\n", + "model_path='drive/MyDrive/Final/semi_sup-6.pt' # will be used to load model\n", + "\n", + "save_path = 'drive/MyDrive/Final/predicted1.csv' # predicted.csv will store the predicted tags" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "6NNlEyqDpItD", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "8b3bdb5f-57cd-497d-b846-02a835ea7494" + }, + "source": [ + "pip install contractions" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Collecting contractions\n", + " Downloading https://files.pythonhosted.org/packages/ce/ad/d1c685967945a04f8596128b15a1ab56c51488f53312e953341af6ff22d1/contractions-0.0.43-py2.py3-none-any.whl\n", + "Collecting textsearch\n", + " Downloading https://files.pythonhosted.org/packages/42/a8/03407021f9555043de5492a2bd7a35c56cc03c2510092b5ec018cae1bbf1/textsearch-0.0.17-py2.py3-none-any.whl\n", + "Collecting Unidecode\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/d0/42/d9edfed04228bacea2d824904cae367ee9efd05e6cce7ceaaedd0b0ad964/Unidecode-1.1.1-py2.py3-none-any.whl (238kB)\n", + "\u001b[K |████████████████████████████████| 245kB 8.4MB/s \n", + "\u001b[?25hCollecting pyahocorasick\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/f4/9f/f0d8e8850e12829eea2e778f1c90e3c53a9a799b7f412082a5d21cd19ae1/pyahocorasick-1.4.0.tar.gz (312kB)\n", + "\u001b[K |████████████████████████████████| 317kB 9.2MB/s \n", + "\u001b[?25hBuilding wheels for collected packages: pyahocorasick\n", + " Building wheel for pyahocorasick (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for pyahocorasick: filename=pyahocorasick-1.4.0-cp36-cp36m-linux_x86_64.whl size=81700 sha256=53963fa8828eb478e469927c095ce4cc61247c49e4c61c840754c491a7b75458\n", + " Stored in directory: /root/.cache/pip/wheels/0a/90/61/87a55f5b459792fbb2b7ba6b31721b06ff5cf6bde541b40994\n", + "Successfully built pyahocorasick\n", + "Installing collected packages: Unidecode, pyahocorasick, textsearch, contractions\n", + "Successfully installed Unidecode-1.1.1 contractions-0.0.43 pyahocorasick-1.4.0 textsearch-0.0.17\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Y_tBb4AvAhVG" + }, + "source": [ + "import sys\n", + "import numpy as np\n", + "import pandas as pd" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "xvhpwznOh8ab" + }, + "source": [ + "\n" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "Ew-OORqDAhVK", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 204 + }, + "outputId": "6d13e298-7837-4a63-d3d1-3ba97a75e925" + }, + "source": [ + "# import data\n", + "from ast import literal_eval\n", + "pred = pd.read_csv(test_path)\n", + "pred.head()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
text
0I completely agree, Sylvia is a moron.
1If Kitty is giving them $17,000 a year to sit ...
2If someone does not know that drug use can kee...
3Maybe we can get those people to hand feed the...
4the work of soros' whores
\n", + "
" + ], + "text/plain": [ + " text\n", + "0 I completely agree, Sylvia is a moron.\n", + "1 If Kitty is giving them $17,000 a year to sit ...\n", + "2 If someone does not know that drug use can kee...\n", + "3 Maybe we can get those people to hand feed the...\n", + "4 the work of soros' whores" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 5 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Hn-ecbXYAhVM" + }, + "source": [ + "# corpus = dup of \"train\"\n", + "corpus = pred.copy()\n", + "\n", + "\n", + " " + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "felL0iqIAhVP" + }, + "source": [ + "\n", + "\n" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "85o2H1dcAhVR" + }, + "source": [ + "# stats of words and chars\n", + "length_of_words = corpus['text'].apply(lambda x: len(x.split()))\n", + "length_of_text = corpus['text'].apply(lambda x: len(x))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "HA_dH5rGAhVT" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "J9pYaMKnAhVV" + }, + "source": [ + "\n", + "from nltk.tokenize import TreebankWordTokenizer as twt\n", + "\n", + "def tokens(string):\n", + " tokens = twt().tokenize(string)\n", + " return tokens \n", + "\n", + "def spans_of_tokens(string):\n", + " list_of_spans = list(twt().span_tokenize(string))\n", + " return list_of_spans\n", + "\n", + "corpus[\"tokenize\"] = corpus[\"text\"].apply(lambda x: tokens(x.replace('\"', \"'\")))\n", + "corpus[\"spans_of_tokens\"] = corpus[\"text\"].apply(lambda x: spans_of_tokens(x.replace('\"', \"'\")))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 204 + }, + "id": "xl-5EOLQmOS7", + "outputId": "e46f3382-76d0-47c1-a1b7-6c3894a49234" + }, + "source": [ + "corpus.head()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
texttokenizespans_of_tokens
0I completely agree, Sylvia is a moron.[I, completely, agree, ,, Sylvia, is, a, moron...[(0, 1), (2, 12), (13, 18), (18, 19), (20, 26)...
1If Kitty is giving them $17,000 a year to sit ...[If, Kitty, is, giving, them, $, 17,000, a, ye...[(0, 2), (3, 8), (9, 11), (12, 18), (19, 23), ...
2If someone does not know that drug use can kee...[If, someone, does, not, know, that, drug, use...[(0, 2), (3, 10), (11, 15), (16, 19), (20, 24)...
3Maybe we can get those people to hand feed the...[Maybe, we, can, get, those, people, to, hand,...[(0, 5), (6, 8), (9, 12), (13, 16), (17, 22), ...
4the work of soros' whores[the, work, of, soros, ', whores][(0, 3), (4, 8), (9, 11), (12, 17), (17, 18), ...
\n", + "
" + ], + "text/plain": [ + " text ... spans_of_tokens\n", + "0 I completely agree, Sylvia is a moron. ... [(0, 1), (2, 12), (13, 18), (18, 19), (20, 26)...\n", + "1 If Kitty is giving them $17,000 a year to sit ... ... [(0, 2), (3, 8), (9, 11), (12, 18), (19, 23), ...\n", + "2 If someone does not know that drug use can kee... ... [(0, 2), (3, 10), (11, 15), (16, 19), (20, 24)...\n", + "3 Maybe we can get those people to hand feed the... ... [(0, 5), (6, 8), (9, 12), (13, 16), (17, 22), ...\n", + "4 the work of soros' whores ... [(0, 3), (4, 8), (9, 11), (12, 17), (17, 18), ...\n", + "\n", + "[5 rows x 3 columns]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 9 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "INLUvdbPAhVZ" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "rYiUPNYVAhVb" + }, + "source": [ + "# zipped together -> token, their spans\n", + "temp = []\n", + "for i in range(len(corpus)):\n", + " temp.append(list(zip(corpus.tokenize[i], corpus.spans_of_tokens[i])))\n", + "\n", + "for i in range(len(temp)): \n", + " temp[i] = [list(ele) for ele in temp[i]] \n", + "\n", + "corpus[\"zipped\"] = temp\n", + "# del temp\n" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "S3qaCeDPuvAX", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "c2c3f670-76e7-4c46-b36e-c7eb9f1e5987" + }, + "source": [ + "corpus.zipped[0]" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[['I', (0, 1)],\n", + " ['completely', (2, 12)],\n", + " ['agree', (13, 18)],\n", + " [',', (18, 19)],\n", + " ['Sylvia', (20, 26)],\n", + " ['is', (27, 29)],\n", + " ['a', (30, 31)],\n", + " ['moron', (32, 37)],\n", + " ['.', (37, 38)]]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 11 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "v2pMVdNvAhVd" + }, + "source": [ + "#################### cleaning functions ####################\n", + "# emoji removing function\n", + "def deEmojify(text):\n", + " regrex_pattern = re.compile(pattern = \"[\"\n", + " u\"\\U0001F600-\\U0001F64F\" # emoticons\n", + " u\"\\U0001F300-\\U0001F5FF\" # symbols & pictographs\n", + " u\"\\U0001F680-\\U0001F6FF\" # transport & map symbols\n", + " u\"\\U0001F1E0-\\U0001F1FF\" # flags (iOS)\n", + " \"]+\", flags = re.UNICODE)\n", + " return regrex_pattern.sub(r'',text)\n", + "\n", + "#puntuation remover\n", + "# from string import punctuation\n", + "# def strip_punctuation(text):\n", + "# return ''.join(c for c in text if c not in punctuation)\n", + "\n", + "#convert accented text\n", + "import unicodedata\n", + "def convert_accented_chars(text):\n", + " text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf-8', 'ignore')\n", + " return text\n", + "\n", + "#expand contraction\n", + "import contractions\n", + "import re \n", + "\n", + "# function to expand contractions\n", + "def expand_contractions(text):\n", + " return(contractions.fix(text))\n", + "# function to remove digits\n", + "def remove_nums(text):\n", + " return re.sub('[0-9]+', '', text)\n", + "\n", + "# remove special chars\n", + "def remove_spe_chars(text):\n", + " return re.sub('\\.+', '.', text)\n", + "#################### cleaning functions #################### " + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "jZjCBK7-AhVf" + }, + "source": [ + "#cleaning tokens (in zipped)\n", + "def clean_string(string):\n", + " # string = string.lower()#lowercasing\n", + " string = expand_contractions(string) #expand contraction\n", + " string = remove_nums(string)#remove digits\n", + " #numbers into words(could be done if required)# not require though\n", + " string = convert_accented_chars(string)#convert accented\n", + " string = deEmojify(string)#remove emoji\n", + " string = remove_spe_chars(string)#combine all multiple full stops occurring together\n", + " \n", + " return string\n", + "\n", + "def cleaning(iterable):\n", + " for i in range(len(iterable)):\n", + " iterable[i][0] = clean_string(iterable[i][0]) \n", + " \n", + " return iterable\n" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "BKt9m9NkAhVh" + }, + "source": [ + "#remove empty tokens (in zipped)\n", + "def remove_empty(iterable):\n", + " iterable = list(filter(lambda x: x[0] != '', iterable))\n", + " return iterable" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "-aosYsVfAhVk" + }, + "source": [ + "# clean\n", + "corpus.zipped = corpus.zipped.apply(lambda x: cleaning(x))\n", + "# remove nulls\n", + "corpus.zipped = corpus.zipped.apply(lambda x: remove_empty(x))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "uJnf9ZncAhVn", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "ed17bb51-666d-4563-8ee7-67be16719986" + }, + "source": [ + "corpus.zipped[0]" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[['I', (0, 1)],\n", + " ['completely', (2, 12)],\n", + " ['agree', (13, 18)],\n", + " [',', (18, 19)],\n", + " ['Sylvia', (20, 26)],\n", + " ['is', (27, 29)],\n", + " ['a', (30, 31)],\n", + " ['moron', (32, 37)],\n", + " ['.', (37, 38)]]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 16 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "ER2DJwZ8AhVq" + }, + "source": [ + "\n", + "\n", + "def redeem_spans(lis):\n", + " arr = []\n", + " for item in lis:\n", + " arr.append(item[1])\n", + " return arr\n", + "\n", + "def reddem_token(lis):\n", + " arr = []\n", + " for item in lis:\n", + " arr.append(item[0])\n", + " return arr" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "3_WmjchbAhVs" + }, + "source": [ + "\n", + "corpus['span_final'] = corpus.zipped.apply(lambda x : redeem_spans(x))\n", + "\n", + "corpus['token_final'] = corpus.zipped.apply(lambda x : reddem_token(x))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "dm49b6mrtlM7", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 589 + }, + "outputId": "6b0566f5-217e-4820-85f8-3e73624ab75b" + }, + "source": [ + "corpus" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
texttokenizespans_of_tokenszippedspan_finaltoken_final
0I completely agree, Sylvia is a moron.[I, completely, agree, ,, Sylvia, is, a, moron...[(0, 1), (2, 12), (13, 18), (18, 19), (20, 26)...[[I, (0, 1)], [completely, (2, 12)], [agree, (...[(0, 1), (2, 12), (13, 18), (18, 19), (20, 26)...[I, completely, agree, ,, Sylvia, is, a, moron...
1If Kitty is giving them $17,000 a year to sit ...[If, Kitty, is, giving, them, $, 17,000, a, ye...[(0, 2), (3, 8), (9, 11), (12, 18), (19, 23), ...[[If, (0, 2)], [Kitty, (3, 8)], [is, (9, 11)],...[(0, 2), (3, 8), (9, 11), (12, 18), (19, 23), ...[If, Kitty, is, giving, them, $, ,, a, year, t...
2If someone does not know that drug use can kee...[If, someone, does, not, know, that, drug, use...[(0, 2), (3, 10), (11, 15), (16, 19), (20, 24)...[[If, (0, 2)], [someone, (3, 10)], [does, (11,...[(0, 2), (3, 10), (11, 15), (16, 19), (20, 24)...[If, someone, does, not, know, that, drug, use...
3Maybe we can get those people to hand feed the...[Maybe, we, can, get, those, people, to, hand,...[(0, 5), (6, 8), (9, 12), (13, 16), (17, 22), ...[[Maybe, (0, 5)], [we, (6, 8)], [can, (9, 12)]...[(0, 5), (6, 8), (9, 12), (13, 16), (17, 22), ...[Maybe, we, can, get, those, people, to, hand,...
4the work of soros' whores[the, work, of, soros, ', whores][(0, 3), (4, 8), (9, 11), (12, 17), (17, 18), ...[[the, (0, 3)], [work, (4, 8)], [of, (9, 11)],...[(0, 3), (4, 8), (9, 11), (12, 17), (17, 18), ...[the, work, of, soros, ', whores]
.....................
789True.....for once UR right.\\nBut the thing is,...[True, ..., ..for, once, UR, right., But, the,...[(0, 4), (4, 7), (7, 12), (13, 17), (18, 20), ...[[True, (0, 4)], [., (4, 7)], [.for, (7, 12)],...[(0, 4), (4, 7), (7, 12), (13, 17), (18, 20), ...[True, ., .for, once, UR, right., But, the, th...
790Naw. The best way to remove the terrorists' po...[Naw., The, best, way, to, remove, the, terror...[(0, 4), (5, 8), (9, 13), (14, 17), (18, 20), ...[[Naw., (0, 4)], [The, (5, 8)], [best, (9, 13)...[(0, 4), (5, 8), (9, 13), (14, 17), (18, 20), ...[Naw., The, best, way, to, remove, the, terror...
791imagine if this dumb witch every cared about m...[imagine, if, this, dumb, witch, every, cared,...[(0, 7), (8, 10), (11, 15), (16, 20), (21, 26)...[[imagine, (0, 7)], [if, (8, 10)], [this, (11,...[(0, 7), (8, 10), (11, 15), (16, 20), (21, 26)...[imagine, if, this, dumb, witch, every, cared,...
792Why thank you for your vey civil remark. I ag...[Why, thank, you, for, your, vey, civil, remar...[(0, 3), (4, 9), (10, 13), (14, 17), (18, 22),...[[Why, (0, 3)], [thank, (4, 9)], [you, (10, 13...[(0, 3), (4, 9), (10, 13), (14, 17), (18, 22),...[Why, thank, you, for, your, vey, civil, remar...
793I hope he does, the bomb the crap out of Bruss...[I, hope, he, does, ,, the, bomb, the, crap, o...[(0, 1), (2, 6), (7, 9), (10, 14), (14, 15), (...[[I, (0, 1)], [hope, (2, 6)], [he, (7, 9)], [d...[(0, 1), (2, 6), (7, 9), (10, 14), (14, 15), (...[I, hope, he, does, ,, the, bomb, the, crap, o...
\n", + "

794 rows × 6 columns

\n", + "
" + ], + "text/plain": [ + " text ... token_final\n", + "0 I completely agree, Sylvia is a moron. ... [I, completely, agree, ,, Sylvia, is, a, moron...\n", + "1 If Kitty is giving them $17,000 a year to sit ... ... [If, Kitty, is, giving, them, $, ,, a, year, t...\n", + "2 If someone does not know that drug use can kee... ... [If, someone, does, not, know, that, drug, use...\n", + "3 Maybe we can get those people to hand feed the... ... [Maybe, we, can, get, those, people, to, hand,...\n", + "4 the work of soros' whores ... [the, work, of, soros, ', whores]\n", + ".. ... ... ...\n", + "789 True.....for once UR right.\\nBut the thing is,... ... [True, ., .for, once, UR, right., But, the, th...\n", + "790 Naw. The best way to remove the terrorists' po... ... [Naw., The, best, way, to, remove, the, terror...\n", + "791 imagine if this dumb witch every cared about m... ... [imagine, if, this, dumb, witch, every, cared,...\n", + "792 Why thank you for your vey civil remark. I ag... ... [Why, thank, you, for, your, vey, civil, remar...\n", + "793 I hope he does, the bomb the crap out of Bruss... ... [I, hope, he, does, ,, the, bomb, the, crap, o...\n", + "\n", + "[794 rows x 6 columns]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 19 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "nRlcwFzqAhVu" + }, + "source": [ + "final_data = corpus[['text', 'span_final', 'token_final']].copy()\n" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "6u9c1xSCAhVx" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "XcG7ckueAhVz" + }, + "source": [ + "use_tokens=final_data['token_final'].tolist()\n", + "use_spans=final_data['span_final'].tolist()\n", + "use_text=final_data['text'].tolist()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "Ei5OBtqT33Q_" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LVN5xanq35fm" + }, + "source": [ + "# Load the model and predict\n", + "\n", + "---\n", + "\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "NygFtcIUyuCK", + "outputId": "cde64cb5-8702-428f-fce6-e633a914d8cf" + }, + "source": [ + "#model1\n", + "\n", + "!pip install transformers==2.6.0\n", + "!pip install seqeval\n", + "!pip install urllib3 --upgrade\n", + "\n", + "\n", + "#Importing libraries\n", + "\n", + "import pandas as pd\n", + "import numpy as np\n", + "import pickle\n", + "from tqdm import tqdm, trange\n", + "\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n", + "from torchtext import data\n", + "import transformers\n", + "from transformers import BertTokenizer, BertConfig, BertForTokenClassification, AdamW\n", + "from transformers import get_linear_schedule_with_warmup\n", + "\n", + "from keras.preprocessing.sequence import pad_sequences\n", + "from seqeval.metrics import accuracy_score\n", + "from sklearn.metrics import f1_score, confusion_matrix\n", + "\n", + "np.random.seed(42)\n", + "torch.manual_seed(12)\n", + "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", + " " + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Collecting transformers==2.6.0\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/4c/a0/32e3a4501ef480f7ea01aac329a716132f32f7911ef1c2fac228acc57ca7/transformers-2.6.0-py3-none-any.whl (540kB)\n", + "\u001b[K |████████████████████████████████| 542kB 8.0MB/s \n", + "\u001b[?25hCollecting sacremoses\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n", + "\u001b[K |████████████████████████████████| 890kB 14.5MB/s \n", + "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers==2.6.0) (2.23.0)\n", + "Collecting boto3\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/4d/cd/513fff674c22507caf5a983ac1aacf87fc207535ada17d720199b51b6cc3/boto3-1.16.36-py2.py3-none-any.whl (130kB)\n", + "\u001b[K |████████████████████████████████| 133kB 34.6MB/s \n", + "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==2.6.0) (2019.12.20)\n", + "Collecting sentencepiece\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/e5/2d/6d4ca4bef9a67070fa1cac508606328329152b1df10bdf31fb6e4e727894/sentencepiece-0.1.94-cp36-cp36m-manylinux2014_x86_64.whl (1.1MB)\n", + "\u001b[K |████████████████████████████████| 1.1MB 23.1MB/s \n", + "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers==2.6.0) (1.18.5)\n", + "Collecting tokenizers==0.5.2\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/d1/3f/73c881ea4723e43c1e9acf317cf407fab3a278daab3a69c98dcac511c04f/tokenizers-0.5.2-cp36-cp36m-manylinux1_x86_64.whl (3.7MB)\n", + "\u001b[K |████████████████████████████████| 3.7MB 44.0MB/s \n", + "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers==2.6.0) (4.41.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers==2.6.0) (3.0.12)\n", + "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.6.0) (1.15.0)\n", + "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.6.0) (7.1.2)\n", + "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.6.0) (0.17.0)\n", + "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.6.0) (3.0.4)\n", + "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.6.0) (2.10)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.6.0) (2020.12.5)\n", + "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.6.0) (1.24.3)\n", + "Collecting botocore<1.20.0,>=1.19.36\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/9c/27/f8757c8d3d11a2332677e2be978f2a524ab13d07d3766e2fff18693e6f3d/botocore-1.19.36-py2.py3-none-any.whl (7.1MB)\n", + "\u001b[K |████████████████████████████████| 7.1MB 34.4MB/s \n", + "\u001b[?25hCollecting jmespath<1.0.0,>=0.7.1\n", + " Downloading https://files.pythonhosted.org/packages/07/cb/5f001272b6faeb23c1c9e0acc04d48eaaf5c862c17709d20e3469c6e0139/jmespath-0.10.0-py2.py3-none-any.whl\n", + "Collecting s3transfer<0.4.0,>=0.3.0\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/69/79/e6afb3d8b0b4e96cefbdc690f741d7dd24547ff1f94240c997a26fa908d3/s3transfer-0.3.3-py2.py3-none-any.whl (69kB)\n", + "\u001b[K |████████████████████████████████| 71kB 10.1MB/s \n", + "\u001b[?25hRequirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.20.0,>=1.19.36->boto3->transformers==2.6.0) (2.8.1)\n", + "Building wheels for collected packages: sacremoses\n", + " Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893261 sha256=200cab31eff90d44082d2c028a3f33f27dbfb1ba584db2af5beaa125a9c783cd\n", + " Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n", + "Successfully built sacremoses\n", + "\u001b[31mERROR: botocore 1.19.36 has requirement urllib3<1.27,>=1.25.4; python_version != \"3.4\", but you'll have urllib3 1.24.3 which is incompatible.\u001b[0m\n", + "Installing collected packages: sacremoses, jmespath, botocore, s3transfer, boto3, sentencepiece, tokenizers, transformers\n", + "Successfully installed boto3-1.16.36 botocore-1.19.36 jmespath-0.10.0 s3transfer-0.3.3 sacremoses-0.0.43 sentencepiece-0.1.94 tokenizers-0.5.2 transformers-2.6.0\n", + "Collecting seqeval\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/9d/2d/233c79d5b4e5ab1dbf111242299153f3caddddbb691219f363ad55ce783d/seqeval-1.2.2.tar.gz (43kB)\n", + "\u001b[K |████████████████████████████████| 51kB 4.5MB/s \n", + "\u001b[?25hRequirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.6/dist-packages (from seqeval) (1.18.5)\n", + "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.6/dist-packages (from seqeval) (0.22.2.post1)\n", + "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.21.3->seqeval) (0.17.0)\n", + "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.4.1)\n", + "Building wheels for collected packages: seqeval\n", + " Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for seqeval: filename=seqeval-1.2.2-cp36-none-any.whl size=16171 sha256=78b26afc49433ce992ec4bdcff29e32f59e8dcbd940aa610b1f165db88bcc6f7\n", + " Stored in directory: /root/.cache/pip/wheels/52/df/1b/45d75646c37428f7e626214704a0e35bd3cfc32eda37e59e5f\n", + "Successfully built seqeval\n", + "Installing collected packages: seqeval\n", + "Successfully installed seqeval-1.2.2\n", + "Collecting urllib3\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/f5/71/45d36a8df68f3ebb098d6861b2c017f3d094538c0fb98fa61d4dc43e69b9/urllib3-1.26.2-py2.py3-none-any.whl (136kB)\n", + "\u001b[K |████████████████████████████████| 143kB 9.1MB/s \n", + "\u001b[31mERROR: requests 2.23.0 has requirement urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1, but you'll have urllib3 1.26.2 which is incompatible.\u001b[0m\n", + "\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n", + "\u001b[?25hInstalling collected packages: urllib3\n", + " Found existing installation: urllib3 1.24.3\n", + " Uninstalling urllib3-1.24.3:\n", + " Successfully uninstalled urllib3-1.24.3\n", + "Successfully installed urllib3-1.26.2\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.6/dist-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.26.2) or chardet (3.0.4) doesn't match a supported version!\n", + " RequestsDependencyWarning)\n" + ], + "name": "stderr" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Ai7qv26gHxoG" + }, + "source": [ + "#Class to take in tokenizer and model along with input data and calculate the span average f1 score\n", + "class SpanAvgF1():\n", + " def __init__(self, model, tokenizer, X, spans, maxlen=500):\n", + " self.model = model\n", + " self.model.cuda()\n", + " self.tokenizer = tokenizer\n", + " self.maxlen = maxlen\n", + " self.X = X\n", + " self.spans = spans\n", + " \n", + " self.prepare_data()\n", + "\n", + " self.X = torch.tensor(self.X)\n", + " self.attention_mask = torch.tensor(self.attention_mask)\n", + "\n", + " data = TensorDataset(self.X, self.attention_mask)\n", + " self.dataloader = DataLoader(data, batch_size=16, shuffle=False)\n", + " \n", + " def tokenize_data(self, x, s):\n", + " sentence = []\n", + " spans = []\n", + " for i in range(len(x)):\n", + " word = x[i]\n", + " # label = y[i]\n", + " tokenized_word = self.tokenizer.tokenize(word)\n", + " sentence.extend(tokenized_word)\n", + " curr = s[i][0]\n", + " spans.append([curr, curr+len(tokenized_word[0])])\n", + " curr += len(tokenized_word[0])\n", + " for j in range(len(tokenized_word)-1):\n", + " spans.append([curr, curr+len(tokenized_word[j+1])-2])\n", + " curr += len(tokenized_word[j+1])-2\n", + " spans[-1][-1] = s[i][1]\n", + "\n", + " return(sentence, spans)\n", + "\n", + " def get_attention_mask(self, x):\n", + " return([[(i!=0) for i in text] for text in x])\n", + "\n", + " def prepare_data(self):\n", + " for i in range(len(self.X)):\n", + " self.X[i], self.spans[i] = self.tokenize_data(self.X[i], self.spans[i])\n", + " self.X = pad_sequences([tokenizer.encode(text) for text in self.X], maxlen = self.maxlen, dtype='long', value=0.0, truncating='post', padding = 'post')\n", + " # self.y = pad_sequences(self.y, maxlen=self.maxlen, value=2, dtype='long', truncating='post', padding='post')\n", + " self.attention_mask = self.get_attention_mask(self.X)\n", + " \n", + " def get_text_lengths(self, masks):\n", + " lengths = []\n", + " for mask in masks:\n", + " lengths.append(torch.sum(mask).item())\n", + " return(lengths)\n", + "\n", + " def evaluate(self):\n", + " predictions , true_labels = [], []\n", + " for batch in self.dataloader:\n", + " batch = tuple(t.to(device) for t in batch)\n", + " b_input_id, b_input_mask = batch\n", + " # b_input_id, b_input_mask, b_labels = batch\n", + " self.model.eval()\n", + "\n", + " with torch.no_grad():\n", + " outputs = self.model(b_input_id, token_type_ids=None, attention_mask=b_input_mask)\n", + " logits = outputs[0].detach().cpu().numpy()\n", + " predictions.extend([list(p) for p in np.argmax(logits, axis=2)])\n", + " true_labels.extend(b_input_mask)\n", + " \n", + " self.pred_tags = [[p_i for p_i, l_i in zip(p, l) if l_i != 0][1:-1] for p, l in zip(predictions, true_labels)]\n", + " # else:\n", + " # self.pred_tags = predictions\n", + " # self.pred_tags = predictions\n", + " return self.pred_tags" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "39SXP1rdXCS3" + }, + "source": [ + "X = use_tokens.copy()\n", + "spans = use_spans.copy()\n" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "RYVZx2Gg-itJ" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "lzPPIIVRA3gg" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "2x-Ym2k1XEII", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 66, + "referenced_widgets": [ + "eb30fae1db3c4d7981f314efaed43744", + "64587bf0800945c1aef9ab70facf92ca", + "7e63d5a523d14e9db305c967af8bcdee", + "07c98439f7c241f98d6d4170f5b34795", + "044e6439d72d458f9a097a4e166b3110", + "b24a8208cf0e426eb2005f1dc3f9bb59", + "2c939107cb0b428f885fa8a47fdebabc", + "a9255aa4e9ee4f5092aa37d4f3427a22" + ] + }, + "outputId": "c8ffc2b1-cb5e-4a75-b8fb-f16a1dfef9b8" + }, + "source": [ + "\n", + "modelorg = torch.load(model_path, map_location=torch.device('cpu'))\n", + "model= modelorg\n", + "tokenizer = BertTokenizer.from_pretrained('bert-base-cased', do_lower_case = False)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "eb30fae1db3c4d7981f314efaed43744", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=213450.0, style=ProgressStyle(descripti…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "2-2ILU5iXVMC" + }, + "source": [ + "model=modelorg\n", + "metric = SpanAvgF1(model, tokenizer, X, spans)\n", + "out=metric.evaluate()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "00zcLxmqT9Px" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "M31cFXVaU1f5" + }, + "source": [ + "predicted_spans=[]\n", + "for i in range(len(spans)):\n", + " s = [0 for k in range(len(use_text[i]))]\n", + " prev = 0\n", + " for j in range(len(spans[i])):\n", + " for k in range(spans[i][j][0], spans[i][j][1]):\n", + " s[k] = out[i][j]\n", + " if(prev==1 and out[i][j]==1):\n", + " for l in range(spans[i][j-1][1], spans[i][j][0]):\n", + " s[l] = 1\n", + " prev = out[i][j]\n", + " predicted_spans.append(s)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "11vIQb94ffXC" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "JtNq_q5OlU57" + }, + "source": [ + "final_offsets=[]\n", + "for x in predicted_spans:\n", + " lis=[]\n", + " for i in range(len(x)):\n", + " if x[i]==1:\n", + " lis.append(i)\n", + " final_offsets.append(lis)\n" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "plmxbyxAplVv" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "kkVPPbBbsHY8" + }, + "source": [ + "\n", + "data = {'spans':final_offsets,'text':use_text}\n", + " \n", + "df = pd.DataFrame(data)\n", + "\n", + "df.to_csv(save_path,index=None)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "PkXewEVFGp0X" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "k8AaScIDGrRj" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/FusionBERT.ipynb b/FusionBERT.ipynb new file mode 100644 index 0000000..14c2ef4 --- /dev/null +++ b/FusionBERT.ipynb @@ -0,0 +1,1723 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "FusionBERT.ipynb", + "provenance": [], + "mount_file_id": "1cL2WAgEa4gTVJEXkQWrjv_Vqce5ZvpaK", + "authorship_tag": "ABX9TyN7RDfLUBeJsdzX/Afecqhr", + "include_colab_link": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "8yJ1nLR5N0eJ", + "outputId": "a05e7702-4214-4e23-f689-ebca7098955f" + }, + "source": [ + "!pip install transformers==2.6.0\n", + "!pip install seqeval" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Requirement already satisfied: transformers==2.6.0 in /usr/local/lib/python3.6/dist-packages (2.6.0)\n", + "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers==2.6.0) (0.0.43)\n", + "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers==2.6.0) (1.16.36)\n", + "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==2.6.0) (2019.12.20)\n", + "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers==2.6.0) (1.18.5)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers==2.6.0) (2.23.0)\n", + "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.6/dist-packages (from transformers==2.6.0) (0.1.94)\n", + "Requirement already satisfied: tokenizers==0.5.2 in /usr/local/lib/python3.6/dist-packages (from transformers==2.6.0) (0.5.2)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers==2.6.0) (3.0.12)\n", + "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers==2.6.0) (4.41.1)\n", + "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.6.0) (1.15.0)\n", + "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.6.0) (7.1.2)\n", + "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.6.0) (0.17.0)\n", + "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers==2.6.0) (0.10.0)\n", + "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers==2.6.0) (0.3.3)\n", + "Requirement already satisfied: botocore<1.20.0,>=1.19.36 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers==2.6.0) (1.19.36)\n", + "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.6.0) (2.10)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.6.0) (2020.12.5)\n", + "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.6.0) (3.0.4)\n", + "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.6.0) (1.24.3)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.20.0,>=1.19.36->boto3->transformers==2.6.0) (2.8.1)\n", + "Requirement already satisfied: seqeval in /usr/local/lib/python3.6/dist-packages (1.2.2)\n", + "Requirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.6/dist-packages (from seqeval) (1.18.5)\n", + "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.6/dist-packages (from seqeval) (0.22.2.post1)\n", + "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.21.3->seqeval) (0.17.0)\n", + "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.4.1)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "VXRdGjhiRfHj" + }, + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import pickle\n", + "from tqdm import tqdm, trange\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n", + "from torchtext import data\n", + "import transformers\n", + "from transformers import BertTokenizer, BertConfig, BertModel, AdamW, BertForTokenClassification, BertPreTrainedModel, BertConfig\n", + "from transformers import RobertaTokenizer, RobertaModel, RobertaConfig, RobertaForTokenClassification\n", + "\n", + "from keras.preprocessing.sequence import pad_sequences\n", + "from tensorflow.keras.utils import to_categorical\n", + "from seqeval.metrics import accuracy_score\n", + "from sklearn.metrics import f1_score, confusion_matrix" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "5JmDqUwMRmiy" + }, + "source": [ + "np.random.seed(42)\n", + "torch.manual_seed(12)\n", + "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "JcN5VNDrRpT1" + }, + "source": [ + "train_path = 'drive/My Drive/Ensemble/train.pkl'\n", + "val_path = 'drive/My Drive/Ensemble/val.pkl'\n", + "\n", + "with open(train_path, 'rb') as f:\n", + " train_data = pickle.load(f)\n", + " f.close()\n", + "\n", + "with open(val_path, 'rb') as f:\n", + " val_data = pickle.load(f)\n", + " f.close()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "aITkl6nOS6jl" + }, + "source": [ + "X_train = train_data['token_final']\n", + "X_val = val_data['token_final']\n", + "Y_train = train_data['target_final']\n", + "Y_val = val_data['target_final']" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "5bxiWMjJS9Vl" + }, + "source": [ + "tokenizer = BertTokenizer.from_pretrained('bert-base-cased', do_lower_case = False)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "2a1tijHXTCBm" + }, + "source": [ + "def tokenize_bert(x, y):\n", + " sentence = []\n", + " labels = [0]\n", + " for word, label in zip(x, y):\n", + " tokenized_word = tokenizer.tokenize(word)\n", + " sentence.extend(tokenized_word)\n", + " labels.extend([label for i in range(len(tokenized_word))])\n", + " labels.append(0)\n", + " return(sentence, labels)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "6gfMvn5CTCuG" + }, + "source": [ + " len_train = len(X_train)\n", + " len_val = len(X_val)\n", + "\n", + " for i in range(len_train):\n", + " X_train[i], Y_train[i] = tokenize_bert(X_train[i], Y_train[i])\n", + "\n", + " for i in range(len_val):\n", + " X_val[i], Y_val[i] = tokenize_bert(X_val[i], Y_val[i])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "2iTLQ747Ulk4" + }, + "source": [ + "ones = 0\n", + "zeros = 0\n", + "total = 0\n", + "for y in Y_train:\n", + " ones += np.sum(np.array(y))\n", + " zeros += len(y) - np.sum(np.array(y))\n", + " total += len(y)\n", + "for y in Y_val:\n", + " ones += np.sum(np.array(y))\n", + " zeros += len(y) - np.sum(np.array(y))\n", + " total += len(y)\n", + "class_weights = torch.tensor([zeros/zeros, zeros/ones], dtype=torch.float32)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "nz-CRa2TV1d1" + }, + "source": [ + "CLASSES = {'0':0, '1':1, '[PAD]':2}\n", + "MAX_LEN = 500\n", + "BATCH_SIZE = 4\n", + "#Convert tokens to token_ids for bert_model, add special tokens to the sequences, pad the sequences\n", + "X_train_id = pad_sequences([tokenizer.encode(text) for text in X_train], maxlen = MAX_LEN, dtype='long', value=0.0, truncating='post', padding = 'post')\n", + "Y_train_id = pad_sequences(Y_train, maxlen=MAX_LEN, value=CLASSES['[PAD]'], dtype='long', truncating='post', padding='post')\n", + "X_val_id = pad_sequences([tokenizer.encode(text) for text in X_val], maxlen = MAX_LEN, dtype='long', value=0.0, truncating='post', padding = 'post')\n", + "Y_val_id = pad_sequences(Y_val, maxlen=MAX_LEN, value=CLASSES['[PAD]'], dtype='long', truncating='post', padding='post')" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "dXYf5_EgV319" + }, + "source": [ + "def get_attention_mask(x):\n", + " return([[(i!=0) for i in text] for text in x])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "KVwkJBzYV5pG" + }, + "source": [ + "attention_mask_train = get_attention_mask(X_train_id)\n", + "attention_mask_val = get_attention_mask(X_val_id)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "E1RD40VOV7yc" + }, + "source": [ + "X_train_id = torch.tensor(X_train_id)\n", + "Y_train_id = torch.tensor(Y_train_id)\n", + "X_val_id = torch.tensor(X_val_id)\n", + "Y_val_id = torch.tensor(Y_val_id)\n", + "attention_mask_train = torch.tensor(attention_mask_train)\n", + "attention_mask_val = torch.tensor(attention_mask_val)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "IxJrV6PbV-NN" + }, + "source": [ + "train_data = TensorDataset(X_train_id, attention_mask_train, Y_train_id)\n", + "train_sampler = RandomSampler(train_data)\n", + "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=BATCH_SIZE)\n", + "\n", + "val_data = TensorDataset(X_val_id, attention_mask_val, Y_val_id)\n", + "val_sampler = RandomSampler(val_data)\n", + "val_dataloader = DataLoader(val_data, sampler=val_sampler, batch_size=BATCH_SIZE)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "qtNkeeBkWA68" + }, + "source": [ + "class BertEnsemble(BertPreTrainedModel):\n", + " def __init__(self,config,*args,**kwargs):\n", + " super().__init__(config)\n", + " self.num_labels = config.num_labels\n", + " self.bert_model_1 = BertModel.from_pretrained('bert-base-cased', output_attentions = False, output_hidden_states = False)\n", + " self.bert_model_2 = BertModel.from_pretrained('bert-base-cased', output_attentions = False, output_hidden_states = False)\n", + " self.dropout = nn.Dropout(config.hidden_dropout_prob)\n", + " self.classifier = nn.Linear(config.hidden_size, config.num_labels)\n", + " self.init_weights()\n", + " def forward(\n", + " self,\n", + " input_ids=None,\n", + " attention_mask=None,\n", + " token_type_ids=None,\n", + " position_ids=None,\n", + " head_mask=None,\n", + " inputs_embeds=None,\n", + " labels=None,\n", + " # output_attentions=None,\n", + " # output_hidden_states=None,\n", + " # return_dict=None,\n", + " ):\n", + " outputs = self.bert_model_1(\n", + " input_ids,\n", + " attention_mask=attention_mask,\n", + " token_type_ids=token_type_ids,\n", + " position_ids=position_ids,\n", + " head_mask=head_mask,\n", + " inputs_embeds=inputs_embeds,\n", + " # output_attentions=output_attentions,\n", + " # output_hidden_states=output_hidden_states,\n", + " # return_dict=return_dict,\n", + " )\n", + " outputs_2 = self.bert_model_2(\n", + " input_ids,\n", + " attention_mask=attention_mask,\n", + " token_type_ids=token_type_ids,\n", + " position_ids=position_ids,\n", + " head_mask=head_mask,\n", + " inputs_embeds=inputs_embeds,\n", + " # output_attentions=output_attentions,\n", + " # output_hidden_states=output_hidden_states,\n", + " # return_dict=return_dict,\n", + " )\n", + " sequence_output = outputs[0]\n", + " sequence_output2 = outputs_2[0]\n", + " sequence_output = self.dropout(sequence_output)\n", + " sequence_output2 = self.dropout(sequence_output2)\n", + " logits = self.classifier(sequence_output+sequence_output2)\n", + "\n", + " loss = None\n", + " if labels is not None:\n", + " loss_fct = nn.CrossEntropyLoss()\n", + " # Only keep active parts of the loss\n", + " if attention_mask is not None:\n", + " active_loss = attention_mask.view(-1) == 1\n", + " active_logits = logits.view(-1, self.num_labels)\n", + " active_labels = torch.where(\n", + " active_loss, labels.view(-1), torch.tensor(loss_fct.ignore_index).type_as(labels)\n", + " )\n", + " loss = loss_fct(active_logits, active_labels)\n", + " else:\n", + " loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))\n", + "\n", + " \n", + " output = (logits,) + outputs[2:]\n", + " return ((loss,) + output) if loss is not None else output\n", + "\n", + " " + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "IOEnpd93WXhK", + "outputId": "0eee1a22-318a-4dce-9bfe-1a648c67f0d0" + }, + "source": [ + "if torch.cuda.is_available(): \n", + " device = torch.device(\"cuda\")\n", + " print('There are %d GPU(s) available.' % torch.cuda.device_count())\n", + " print('We will use the GPU:', torch.cuda.get_device_name(0))\n", + "else:\n", + " print('No GPU available, using the CPU instead.')\n", + " device = torch.device(\"cpu\")\n", + "\n", + "config = BertConfig()\n", + "model = BertEnsemble(config)\n", + "model.to(device)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "There are 1 GPU(s) available.\n", + "We will use the GPU: Tesla T4\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "BertEnsemble(\n", + " (bert_model_1): BertModel(\n", + " (embeddings): BertEmbeddings(\n", + " (word_embeddings): Embedding(28996, 768, padding_idx=0)\n", + " (position_embeddings): Embedding(512, 768)\n", + " (token_type_embeddings): Embedding(2, 768)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (encoder): BertEncoder(\n", + " (layer): ModuleList(\n", + " (0): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (1): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (2): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (3): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (4): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (5): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (6): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (7): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (8): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (9): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (10): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (11): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " )\n", + " (pooler): BertPooler(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (activation): Tanh()\n", + " )\n", + " )\n", + " (bert_model_2): BertModel(\n", + " (embeddings): BertEmbeddings(\n", + " (word_embeddings): Embedding(28996, 768, padding_idx=0)\n", + " (position_embeddings): Embedding(512, 768)\n", + " (token_type_embeddings): Embedding(2, 768)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (encoder): BertEncoder(\n", + " (layer): ModuleList(\n", + " (0): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (1): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (2): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (3): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (4): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (5): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (6): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (7): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (8): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (9): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (10): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (11): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " )\n", + " (pooler): BertPooler(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (activation): Tanh()\n", + " )\n", + " )\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " (classifier): Linear(in_features=768, out_features=2, bias=True)\n", + ")" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 16 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "6RSpX_1pXPgl" + }, + "source": [ + "train_loss, val_loss = [], []\n", + "train_acc, val_acc = [], []\n", + "train_f1, val_f1 = [], []" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "wnKkveybXcH5" + }, + "source": [ + "FINE_TUNING = True\n", + "if FINE_TUNING:\n", + " param_optimizer = list(model.named_parameters())\n", + " no_decay = ['bias', 'gamma', 'beta']\n", + " optimizer_grouped_parameters = [\n", + " {'params' : [p for n,p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay_rate' : 0.01},\n", + " {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],'weight_decay_rate': 0.0}]\n", + "else:\n", + " param_optimizer = list(model.classifier.named_parameters())\n", + " optimizer_grouped_parameters = [{\"params\": [p for n, p in param_optimizer]}]\n", + "\n", + "optimizer = AdamW(optimizer_grouped_parameters, lr=3e-5, eps=1e-8)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "4wk54KdmXeO-" + }, + "source": [ + "from transformers import get_linear_schedule_with_warmup\n", + "epochs = 4\n", + "max_grad_norm = 1.0\n", + "\n", + "total_steps = len(train_dataloader) * epochs\n", + "\n", + "scheduler = get_linear_schedule_with_warmup(\n", + " optimizer,\n", + " num_warmup_steps=0,\n", + " num_training_steps=total_steps\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "NSMHnrZYXfuK", + "outputId": "8bd94afc-d184-499c-dde3-19291f18404f" + }, + "source": [ + "#Training code\n", + "\n", + "train_loss, val_loss = [], []\n", + "train_acc, val_acc = [], []\n", + "train_f1, val_f1 = [], []\n", + "\n", + "l = 100\n", + "criterion = nn.CrossEntropyLoss(weight=class_weights)\n", + "criterion = criterion.to(device)\n", + "for epoch in range(epochs):\n", + " print(f\"Epoch {epoch}\")\n", + " model.train()\n", + " t_loss, t_acc = 0, 0\n", + " predictions, true_labels = [], []\n", + " for step, batch in enumerate(tqdm(train_dataloader, desc='Train')):\n", + " batch = tuple(t.to(device) for t in batch)\n", + " b_input_id, b_input_mask, b_labels = batch\n", + " model.zero_grad()\n", + " outputs = model(b_input_id, token_type_ids=None, attention_mask=b_input_mask, labels = b_labels)\n", + " # loss = outputs[0]\n", + "\n", + " # Code for using weighted cross-entropy loss\n", + " active_loss = b_input_mask.view(-1) == 1\n", + " active_logits = outputs[1].view(-1, 2)\n", + " active_labels = torch.where(active_loss, b_labels.view(-1), torch.tensor(criterion.ignore_index).type_as(b_labels))\n", + " loss = criterion(active_logits, active_labels)\n", + "\n", + " loss.backward()\n", + " t_loss += loss.item()\n", + " torch.nn.utils.clip_grad_norm_(parameters=model.parameters(), max_norm=max_grad_norm)\n", + " optimizer.step()\n", + " scheduler.step()\n", + "\n", + " logits = outputs[1].detach().cpu().numpy()\n", + " label_ids = b_labels.to('cpu').numpy()\n", + "\n", + " predictions.extend([list(p) for p in np.argmax(logits, axis=2)])\n", + " true_labels.extend(label_ids)\n", + " \n", + " print(f\"Train Loss : {t_loss/len(train_dataloader)}\")\n", + " train_loss.append(t_loss/len(train_dataloader))\n", + " pred_tags = [[p_i for p_i, l_i in zip(p,l) if l_i!=2] for p,l in zip(predictions, true_labels)]\n", + " pred_tags = [p_i for p in pred_tags for p_i in p[1:-1]]\n", + " # pred_tags = [p_i for p, l in zip(predictions, true_labels)\n", + " # for p_i, l_i in zip(p, l) if l_i != 2]\n", + " # valid_tags = [l_i for l in true_labels\n", + " # for l_i in l if l_i != 2]\n", + " valid_tags = [[l_i for l_i in l if l_i!=2] for l in true_labels]\n", + " valid_tags = [l_i for l in valid_tags for l_i in l[1:-1]]\n", + " train_acc.append(accuracy_score(pred_tags, valid_tags))\n", + " train_f1.append(f1_score(pred_tags, valid_tags))\n", + " print(\"Train Accuracy: {}\".format(accuracy_score(pred_tags, valid_tags)))\n", + " print(\"Train F1-Score: {}\".format(f1_score(pred_tags, valid_tags)))\n", + " print()\n", + "\n", + " model.eval()\n", + " v_loss, v_accuracy = 0, 0\n", + " predictions , true_labels = [], []\n", + " for batch in tqdm(val_dataloader, desc=\"Val\"):\n", + " batch = tuple(t.to(device) for t in batch)\n", + " b_input_id, b_input_mask, b_labels = batch\n", + " with torch.no_grad():\n", + " outputs = model(b_input_id, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n", + " \n", + " # loss = outputs[0]\n", + "\n", + " active_loss = b_input_mask.view(-1) == 1\n", + " active_logits = outputs[1].view(-1, 2)\n", + " active_labels = torch.where(active_loss, b_labels.view(-1), torch.tensor(criterion.ignore_index).type_as(b_labels))\n", + " loss = criterion(active_logits, active_labels)\n", + " \n", + " logits = outputs[1].detach().cpu().numpy()\n", + " label_ids = b_labels.to('cpu').numpy()\n", + "\n", + " v_loss += loss.item()\n", + " predictions.extend([list(p) for p in np.argmax(logits, axis=2)])\n", + " true_labels.extend(label_ids)\n", + " \n", + " v_loss = v_loss/len(val_dataloader)\n", + " val_loss.append(v_loss)\n", + " if(v_loss < l):\n", + " l = v_loss\n", + " print(\"Model Checkpoint\")\n", + " torch.save(model, f'drive/MyDrive/Ensemble/ensemble{epoch}.pt')\n", + " \n", + " print(f\"Validation Loss : {v_loss}\")\n", + " pred_tags = [[p_i for p_i,l_i in zip(p,l) if l_i!=2] for p,l in zip(predictions, true_labels)]\n", + " pred_tags = [p_i for p in pred_tags for p_i in p[1:-1]]\n", + " # pred_tags = [p_i for p, l in zip(predictions, true_labels)\n", + " # for p_i, l_i in zip(p, l) if l_i != 2]\n", + " # valid_tags = [l_i for l in true_labels\n", + " # for l_i in l if l_i != 2]\n", + " valid_tags = [[l_i for l_i in l if l_i!=2] for l in true_labels]\n", + " valid_tags = [l_i for l in valid_tags for l_i in l[1:-1]]\n", + " print(\"Validation Accuracy: {}\".format(accuracy_score(pred_tags, valid_tags)))\n", + " print(\"Validation F1-Score: {}\".format(f1_score(pred_tags, valid_tags)))\n", + " val_acc.append(accuracy_score(pred_tags, valid_tags))\n", + " val_f1.append(f1_score(pred_tags, valid_tags))\n", + " print()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "\rTrain: 0%| | 0/1588 [00:00\"Open" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "GEmLho-W2bKt" + }, + "source": [ + "import pickle" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "y90zB68i4Bmt" + }, + "source": [ + "path = 'drive/MyDrive/Toxic Spam/augmented.pkl'" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "Bs7qEsf14jQ2" + }, + "source": [ + "with open(path,'rb') as f:\n", + " aug_data=pickle.load(f)\n" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "7H_CRpGK4v0R", + "outputId": "848516cb-0710-47d0-9d67-883bf44bd438" + }, + "source": [ + "print(aug_data)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + " token_final target_final\n", + "0 [Only, a, fool, would, not, obviously, preside... [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", + "1 [cops, are, dumbasses, any, male, from, is, a,... [0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, ...\n", + "2 [Well, stated, 'Donald, dumb-azz, be, Vladimir... [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n", + "3 [So, ,, the, is, on, white, supremacists, whil... [0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, ...\n", + "4 [is, ,, another, sycophant, and, lick, The, on... [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n", + "... ... ...\n", + "4559 [Yeah, ,, your, idiot, let, 's, be, stupid, ra... [0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, ...\n", + "4560 [do, You, Beav, wonder, n't, ;, you, make, a, ... [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, ...\n", + "4561 [As, for, Clinton, order, it, 's, well, known,... [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n", + "4562 [Because, the, with, steaming, pile, of, crap,... [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, ...\n", + "4563 [Does, this, have, anything, of, to, have, wit... [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n", + "\n", + "[4564 rows x 2 columns]\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "seh34N9a5G02" + }, + "source": [ + "train_path = 'drive/MyDrive/Ensemble/train.pkl'\n", + "val_path = 'drive/MyDrive/Ensemble/val.pkl'\n", + "\n", + "with open(train_path, 'rb') as f:\n", + " train_data = pickle.load(f)\n", + " f.close()\n", + "\n", + "with open(val_path, 'rb') as f:\n", + " val_data = pickle.load(f)\n", + " f.close()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "bGQTboDO6Yu1" + }, + "source": [ + "import pandas as pd" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "KpEXiS5Q7y64", + "outputId": "7b9aca12-b765-4b57-feb1-3d8bac8c7398" + }, + "source": [ + "final_train_data = pd.concat([train_data,aug_data],ignore_index=True)\n", + "print(len(final_train_data))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "10915\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "hVXBUS5k6nao" + }, + "source": [ + "X_train = final_train_data['token_final']\n", + "X_val = val_data['token_final']\n", + "Y_train = final_train_data['target_final']\n", + "Y_val = val_data['target_final']" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "c8J8uXfp7RJO", + "outputId": "e71c3ddd-58bd-4fd4-aa37-1042d3adb0ed" + }, + "source": [ + "print(len(X_train),len(Y_train))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "10915 10915\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "4zlkajM_7fm_" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "hsTHGPjn8gEW", + "outputId": "ecc2b924-38e3-4f48-d8a5-79393bdccb82" + }, + "source": [ + "import torch\n", + "if torch.cuda.is_available(): \n", + " device = torch.device(\"cuda\")\n", + " print('There are %d GPU(s) available.' % torch.cuda.device_count())\n", + " print('We will use the GPU:', torch.cuda.get_device_name(0))\n", + "else:\n", + " print('No GPU available, using the CPU instead.')\n", + " device = torch.device(\"cpu\")" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "There are 1 GPU(s) available.\n", + "We will use the GPU: Tesla T4\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "iZq54ywM8oVP", + "outputId": "d7887a1b-3568-4da9-c3ee-bfc13b819ea6" + }, + "source": [ + "!pip install transformers==4.0.0\n", + "!pip install seqeval\n", + "!pip install urllib3 --upgrade" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Collecting transformers==4.0.0\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/99/84/7bc03215279f603125d844bf81c3fb3f2d50fe8e511546eb4897e4be2067/transformers-4.0.0-py3-none-any.whl (1.4MB)\n", + "\u001b[K |████████████████████████████████| 1.4MB 22.0MB/s \n", + "\u001b[?25hCollecting tokenizers==0.9.4\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/0f/1c/e789a8b12e28be5bc1ce2156cf87cb522b379be9cadc7ad8091a4cc107c4/tokenizers-0.9.4-cp36-cp36m-manylinux2010_x86_64.whl (2.9MB)\n", + "\u001b[K |████████████████████████████████| 2.9MB 55.2MB/s \n", + "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers==4.0.0) (2.23.0)\n", + "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers==4.0.0) (1.18.5)\n", + "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers==4.0.0) (4.41.1)\n", + "Collecting sacremoses\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n", + "\u001b[K |████████████████████████████████| 890kB 56.4MB/s \n", + "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers==4.0.0) (20.7)\n", + "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==4.0.0) (2019.12.20)\n", + "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers==4.0.0) (0.8)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers==4.0.0) (3.0.12)\n", + "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==4.0.0) (2.10)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==4.0.0) (2020.12.5)\n", + "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==4.0.0) (3.0.4)\n", + "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==4.0.0) (1.24.3)\n", + "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==4.0.0) (1.15.0)\n", + "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==4.0.0) (7.1.2)\n", + "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==4.0.0) (0.17.0)\n", + "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers==4.0.0) (2.4.7)\n", + "Building wheels for collected packages: sacremoses\n", + " Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893261 sha256=866b33898ba8bc4058113b467aba4c3e724422a1b666747fc5adb87459c2b017\n", + " Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n", + "Successfully built sacremoses\n", + "Installing collected packages: tokenizers, sacremoses, transformers\n", + "Successfully installed sacremoses-0.0.43 tokenizers-0.9.4 transformers-4.0.0\n", + "Collecting seqeval\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/9d/2d/233c79d5b4e5ab1dbf111242299153f3caddddbb691219f363ad55ce783d/seqeval-1.2.2.tar.gz (43kB)\n", + "\u001b[K |████████████████████████████████| 51kB 6.7MB/s \n", + "\u001b[?25hRequirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.6/dist-packages (from seqeval) (1.18.5)\n", + "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.6/dist-packages (from seqeval) (0.22.2.post1)\n", + "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.4.1)\n", + "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.21.3->seqeval) (0.17.0)\n", + "Building wheels for collected packages: seqeval\n", + " Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for seqeval: filename=seqeval-1.2.2-cp36-none-any.whl size=16171 sha256=58418ca51099c0126a3ac2b1684d7761b5026a0de933e5482109fdcf14aef2da\n", + " Stored in directory: /root/.cache/pip/wheels/52/df/1b/45d75646c37428f7e626214704a0e35bd3cfc32eda37e59e5f\n", + "Successfully built seqeval\n", + "Installing collected packages: seqeval\n", + "Successfully installed seqeval-1.2.2\n", + "Collecting urllib3\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/f5/71/45d36a8df68f3ebb098d6861b2c017f3d094538c0fb98fa61d4dc43e69b9/urllib3-1.26.2-py2.py3-none-any.whl (136kB)\n", + "\u001b[K |████████████████████████████████| 143kB 21.6MB/s \n", + "\u001b[31mERROR: requests 2.23.0 has requirement urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1, but you'll have urllib3 1.26.2 which is incompatible.\u001b[0m\n", + "\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n", + "\u001b[?25hInstalling collected packages: urllib3\n", + " Found existing installation: urllib3 1.24.3\n", + " Uninstalling urllib3-1.24.3:\n", + " Successfully uninstalled urllib3-1.24.3\n", + "Successfully installed urllib3-1.26.2\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "qPcqhtHcHqRH", + "outputId": "7e1b26fa-9e9b-4a1b-bd0a-3f60fa924647" + }, + "source": [ + "#Importing librariesCLASSES = {'0':0, '1':1, '[PAD]':2}\n", + "\n", + "import pandas as pd\n", + "import numpy as np\n", + "import pickle\n", + "from tqdm import tqdm, trange\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import random\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n", + "from torchtext import data\n", + "import transformers\n", + "from transformers import RobertaTokenizer, RobertaForTokenClassification, AdamW\n", + "from transformers import get_linear_schedule_with_warmup\n", + "\n", + "from keras.preprocessing.sequence import pad_sequences\n", + "from seqeval.metrics import accuracy_score\n", + "from sklearn.metrics import f1_score, confusion_matrix\n", + "\n", + "np.random.seed(42)\n", + "torch.manual_seed(12)\n", + "# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.6/dist-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.26.2) or chardet (3.0.4) doesn't match a supported version!\n", + " RequestsDependencyWarning)\n" + ], + "name": "stderr" + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 12 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "2RmShAMOH0Mt", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 115, + "referenced_widgets": [ + "2542e4f9d1774d60906a2b9084563fb9", + "ea9ca28039e1405791089eaf8d49363c", + "5697e19c0e00493fa31c242574e577a3", + "16625af134d94413ba02f0775e1c8401", + "d73b232bac0a42fdbd6301f12b8a1a37", + "9c52c5c154c44e5eaca685d4433f1970", + "41a0dae3f8d747199a579eb863bb7b8d", + "fa6440bb32324e988d5b98ccb878f1db", + "29263e9ba655448b826536e32bb55633", + "b3f1a9c56f4e4a97bd717fe21520b588", + "61ff1f515a034abab3a943881592c161", + "f7642c4ecbc74d309992b362d30b9e54", + "3438956d97a843b28fc67c8ecece31dd", + "478b34d3c8bb448393afdcf5b15354b8", + "414ebd5060814463b8e2a5ad1ddca239", + "efc43628efb14b419105916365dedb35" + ] + }, + "outputId": "7e55b4d7-a2a2-4d9a-bb54-c434334b6dc9" + }, + "source": [ + "tokenizer = RobertaTokenizer.from_pretrained(\"roberta-base\")" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2542e4f9d1774d60906a2b9084563fb9", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=898823.0, style=ProgressStyle(descripti…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "29263e9ba655448b826536e32bb55633", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=456318.0, style=ProgressStyle(descripti…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "nLMh1-ThI7Cx" + }, + "source": [ + "def tokenize_roberta(x, y):\n", + " sentence = []\n", + " labels = [0]\n", + " for word, label in zip(x, y):\n", + " tokenized_word = tokenizer.tokenize(word)\n", + " sentence.extend(tokenized_word)\n", + " labels.extend([label for i in range(len(tokenized_word))])\n", + " labels.append(0)\n", + " return(sentence, labels)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "rY-e8b0AJWYI", + "outputId": "582f81fc-e728-4107-9a0e-a21291928bc8" + }, + "source": [ + "print(tokenize_roberta(X_train[0],Y_train[0]))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "(['Only', 'a', 'f', 'ool', 'would', 'not', 'be', 'against', 'this', 'president', 'He', 'is', 'mass', 'ively', 'and', 'ob', 'viously', 'un', 'fit', 'for', 'office'], [0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "BYBppUyJJcrg" + }, + "source": [ + " len_train = len(X_train)\n", + " len_val = len(X_val)\n", + "\n", + " for i in range(len_train):\n", + " X_train[i], Y_train[i] = tokenize_roberta(X_train[i], Y_train[i])\n", + "\n", + " for i in range(len_val):\n", + " X_val[i], Y_val[i] = tokenize_roberta(X_val[i], Y_val[i])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "Mw1pZf3yJpXI" + }, + "source": [ + "#Calculating class weights for training on weighted cross-entropy\n", + "ones = 0\n", + "zeros = 0\n", + "total = 0\n", + "for y in Y_train:\n", + " ones += np.sum(np.array(y))\n", + " zeros += len(y) - np.sum(np.array(y))\n", + " total += len(y)\n", + "for y in Y_val:\n", + " ones += np.sum(np.array(y))\n", + " zeros += len(y) - np.sum(np.array(y))\n", + " total += len(y)\n", + "class_weights = torch.tensor([zeros/zeros, zeros/ones], dtype=torch.float32)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "FpkFHm1cJzq_" + }, + "source": [ + "#We use the class 2 to pad the labels tensors and it also allows us to find the mask easily\n", + "CLASSES = {'0':0, '1':1, '[PAD]':2}\n", + "MAX_LEN = 500\n", + "BATCH_SIZE = 4\n", + "#Convert tokens to token_ids for bert_model, add special tokens to the sequences, pad the sequences\n", + "X_train_id = pad_sequences([tokenizer.encode(text) for text in X_train], maxlen = MAX_LEN, dtype='long', value=0.0, truncating='post', padding = 'post')\n", + "Y_train_id = pad_sequences(Y_train, maxlen=MAX_LEN, value=CLASSES['[PAD]'], dtype='long', truncating='post', padding='post')\n", + "X_val_id = pad_sequences([tokenizer.encode(text) for text in X_val], maxlen = MAX_LEN, dtype='long', value=0.0, truncating='post', padding = 'post')\n", + "Y_val_id = pad_sequences(Y_val, maxlen=MAX_LEN, value=CLASSES['[PAD]'], dtype='long', truncating='post', padding='post')" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "0hLvj1gFJ6CL" + }, + "source": [ + "def get_attention_mask(x):\n", + " return([[(i!=0) for i in text] for text in x])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "LZnkLIsRKPTT" + }, + "source": [ + "#Generate mask for training so that padding tokens are not considered while training\n", + "attention_mask_train = get_attention_mask(X_train_id)\n", + "attention_mask_val = get_attention_mask(X_val_id)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "HnBbrnFbKKbQ" + }, + "source": [ + "X_train_id = torch.tensor(X_train_id)\n", + "Y_train_id = torch.tensor(Y_train_id)\n", + "X_val_id = torch.tensor(X_val_id)\n", + "Y_val_id = torch.tensor(Y_val_id)\n", + "attention_mask_train = torch.tensor(attention_mask_train)\n", + "attention_mask_val = torch.tensor(attention_mask_val)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "U28cNYQTKLqA" + }, + "source": [ + "#Initialize dataloaders for train and validation data which will give batches of data, labels, masks\n", + "train_data = TensorDataset(X_train_id, attention_mask_train, Y_train_id)\n", + "train_sampler = RandomSampler(train_data)\n", + "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=BATCH_SIZE)\n", + "\n", + "val_data = TensorDataset(X_val_id, attention_mask_val, Y_val_id)\n", + "val_sampler = RandomSampler(val_data)\n", + "val_dataloader = DataLoader(val_data, sampler=val_sampler, batch_size=BATCH_SIZE)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "zH1LBUsfKcXR" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000, + "referenced_widgets": [ + "94b01044d6fd4b45a7804f91043d36b9", + "ed5223ef9ca54ef09112dbd58cdc7934", + "a93adb498c0341a69e6173d7523dd439", + "1918691a5e0f4c51b3814802d25dbda2", + "d3adc874585245dca671f7882dc5eea2", + "73b12eb6dec04a8d837b11ff43103a93", + "56a300e0c3ab4aeeaade7250072b0e7c", + "a5e7f75e49b541f4bf0d3c39ce72bbdb", + "d33de9a76d7941f4af3cda8ee65eace7", + "a6814af315bd4401a0f745454746bdd5", + "6f3ced07670545ee828a71768d7922ed", + "38fdb942bec3438b9b3561cfa318c572", + "2eeafff492584ee98908e6d3e29b68ad", + "c51f80e031554d77806480ccf9db9865", + "9a2c9fe22cf44d868d46131e30d910d8", + "92aa8713f78d41a6883d223a2b957462" + ] + }, + "id": "rXShwwiRKuz6", + "outputId": "1d1c7f21-4352-4da3-bc28-44a1068180c4" + }, + "source": [ + "from transformers import RobertaForTokenClassification\n", + "model = RobertaForTokenClassification.from_pretrained('roberta-base', num_labels=2)\n", + "model.cuda()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "94b01044d6fd4b45a7804f91043d36b9", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=481.0, style=ProgressStyle(description_…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d33de9a76d7941f4af3cda8ee65eace7", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=501200538.0, style=ProgressStyle(descri…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForTokenClassification: ['lm_head.bias', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n", + "- This IS expected if you are initializing RobertaForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing RobertaForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of RobertaForTokenClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.weight', 'classifier.bias']\n", + "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" + ], + "name": "stderr" + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "RobertaForTokenClassification(\n", + " (roberta): RobertaModel(\n", + " (embeddings): RobertaEmbeddings(\n", + " (word_embeddings): Embedding(50265, 768, padding_idx=1)\n", + " (position_embeddings): Embedding(514, 768, padding_idx=1)\n", + " (token_type_embeddings): Embedding(1, 768)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (encoder): RobertaEncoder(\n", + " (layer): ModuleList(\n", + " (0): RobertaLayer(\n", + " (attention): RobertaAttention(\n", + " (self): RobertaSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): RobertaSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): RobertaIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): RobertaOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (1): RobertaLayer(\n", + " (attention): RobertaAttention(\n", + " (self): RobertaSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): RobertaSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): RobertaIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): RobertaOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (2): RobertaLayer(\n", + " (attention): RobertaAttention(\n", + " (self): RobertaSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): RobertaSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): RobertaIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): RobertaOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (3): RobertaLayer(\n", + " (attention): RobertaAttention(\n", + " (self): RobertaSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): RobertaSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): RobertaIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): RobertaOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (4): RobertaLayer(\n", + " (attention): RobertaAttention(\n", + " (self): RobertaSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): RobertaSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): RobertaIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): RobertaOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (5): RobertaLayer(\n", + " (attention): RobertaAttention(\n", + " (self): RobertaSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): RobertaSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): RobertaIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): RobertaOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (6): RobertaLayer(\n", + " (attention): RobertaAttention(\n", + " (self): RobertaSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): RobertaSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): RobertaIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): RobertaOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (7): RobertaLayer(\n", + " (attention): RobertaAttention(\n", + " (self): RobertaSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): RobertaSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): RobertaIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): RobertaOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (8): RobertaLayer(\n", + " (attention): RobertaAttention(\n", + " (self): RobertaSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): RobertaSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): RobertaIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): RobertaOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (9): RobertaLayer(\n", + " (attention): RobertaAttention(\n", + " (self): RobertaSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): RobertaSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): RobertaIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): RobertaOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (10): RobertaLayer(\n", + " (attention): RobertaAttention(\n", + " (self): RobertaSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): RobertaSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): RobertaIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): RobertaOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (11): RobertaLayer(\n", + " (attention): RobertaAttention(\n", + " (self): RobertaSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): RobertaSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): RobertaIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): RobertaOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " )\n", + " )\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " (classifier): Linear(in_features=768, out_features=2, bias=True)\n", + ")" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 23 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "danH-mlbK-VJ" + }, + "source": [ + "#Initialize AdamW optimizer (add weight decay for regularization) \n", + "FINE_TUNING = True\n", + "if FINE_TUNING:\n", + " param_optimizer = list(model.named_parameters())\n", + " no_decay = ['bias', 'gamma', 'beta']\n", + " optimizer_grouped_parameters = [\n", + " {'params' : [p for n,p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay_rate' : 0.01},\n", + " {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],'weight_decay_rate': 0.0}]\n", + "else:\n", + " param_optimizer = list(model.classifier.named_parameters())\n", + " optimizer_grouped_parameters = [{\"params\": [p for n, p in param_optimizer]}]\n", + "\n", + "optimizer = AdamW(optimizer_grouped_parameters, lr=1e-5, eps=1e-8)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "fwHZsFm7MGlf" + }, + "source": [ + "#Initialize scheduler (helps to gradually decrease the learning rate)\n", + "epochs = 3\n", + "max_grad_norm = 1.0\n", + "\n", + "total_steps = len(train_dataloader) * epochs\n", + "\n", + "scheduler = get_linear_schedule_with_warmup(\n", + " optimizer,\n", + " num_warmup_steps=0,\n", + " num_training_steps=total_steps\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "DZYCn2jYMI90" + }, + "source": [ + "criterion = nn.CrossEntropyLoss(weight=class_weights)\n", + "criterion = criterion.to(device)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "wvTWthxJMMi7", + "outputId": "df7fab4e-7fd8-430c-f9af-ee2c2fcf1343" + }, + "source": [ + "#Training code\n", + "\n", + "train_loss, val_loss = [], []\n", + "train_acc, val_acc = [], []\n", + "train_f1, val_f1 = [], []\n", + "\n", + "l = 100\n", + "\n", + "for epoch in range(epochs):\n", + " print(f\"Epoch {epoch}\")\n", + " model.train()\n", + " t_loss, t_acc = 0, 0\n", + " predictions, true_labels = [], []\n", + " for step, batch in enumerate(tqdm(train_dataloader, desc='Train')):\n", + " batch = tuple(t.to(device) for t in batch)\n", + " b_input_id, b_input_mask, b_labels = batch\n", + " model.zero_grad()\n", + " outputs = model(b_input_id, token_type_ids=None, attention_mask=b_input_mask, labels = b_labels)\n", + " # loss = outputs[0]\n", + "\n", + " # Code for using weighted cross-entropy loss\n", + " active_loss = b_input_mask.view(-1) == 1\n", + " active_logits = outputs[1].view(-1, 2)\n", + " active_labels = torch.where(active_loss, b_labels.view(-1), torch.tensor(criterion.ignore_index).type_as(b_labels))\n", + " loss = criterion(active_logits, active_labels)\n", + "\n", + " loss.backward()\n", + " t_loss += loss.item()\n", + " torch.nn.utils.clip_grad_norm_(parameters=model.parameters(), max_norm=max_grad_norm)\n", + " optimizer.step()\n", + " scheduler.step()\n", + "\n", + " logits = outputs[1].detach().cpu().numpy()\n", + " label_ids = b_labels.to('cpu').numpy()\n", + "\n", + " predictions.extend([list(p) for p in np.argmax(logits, axis=2)])\n", + " true_labels.extend(label_ids)\n", + " \n", + " print(f\"Train Loss : {t_loss/len(train_dataloader)}\")\n", + " train_loss.append(t_loss/len(train_dataloader))\n", + " pred_tags = [[p_i for p_i, l_i in zip(p,l) if l_i!=2] for p,l in zip(predictions, true_labels)]\n", + " pred_tags = [p_i for p in pred_tags for p_i in p[1:-1]]\n", + " # pred_tags = [p_i for p, l in zip(predictions, true_labels)\n", + " # for p_i, l_i in zip(p, l) if l_i != 2]\n", + " # valid_tags = [l_i for l in true_labels\n", + " # for l_i in l if l_i != 2]\n", + " valid_tags = [[l_i for l_i in l if l_i!=2] for l in true_labels]\n", + " valid_tags = [l_i for l in valid_tags for l_i in l[1:-1]]\n", + " train_acc.append(accuracy_score(pred_tags, valid_tags))\n", + " train_f1.append(f1_score(pred_tags, valid_tags))\n", + " print(\"Train Accuracy: {}\".format(accuracy_score(pred_tags, valid_tags)))\n", + " print(\"Train F1-Score: {}\".format(f1_score(pred_tags, valid_tags)))\n", + " print()\n", + "\n", + " model.eval()\n", + " v_loss, v_accuracy = 0, 0\n", + " predictions , true_labels = [], []\n", + " for batch in tqdm(val_dataloader, desc=\"Val\"):\n", + " batch = tuple(t.to(device) for t in batch)\n", + " b_input_id, b_input_mask, b_labels = batch\n", + " with torch.no_grad():\n", + " outputs = model(b_input_id, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n", + " \n", + " # loss = outputs[0]\n", + "\n", + " active_loss = b_input_mask.view(-1) == 1\n", + " active_logits = outputs[1].view(-1, 2)\n", + " active_labels = torch.where(active_loss, b_labels.view(-1), torch.tensor(criterion.ignore_index).type_as(b_labels))\n", + " loss = criterion(active_logits, active_labels)\n", + " \n", + " logits = outputs[1].detach().cpu().numpy()\n", + " label_ids = b_labels.to('cpu').numpy()\n", + "\n", + " v_loss += loss.item()\n", + " predictions.extend([list(p) for p in np.argmax(logits, axis=2)])\n", + " true_labels.extend(label_ids)\n", + " \n", + " v_loss = v_loss/len(val_dataloader)\n", + " val_loss.append(v_loss)\n", + " if(v_loss < l):\n", + " l = v_loss\n", + " print(\"Model Checkpoint\")\n", + " torch.save(model, f'drive/MyDrive/Toxic Spam/1612augroberta{epoch}.pt')\n", + " \n", + " print(f\"Validation Loss : {v_loss}\")\n", + " pred_tags = [[p_i for p_i,l_i in zip(p,l) if l_i!=2] for p,l in zip(predictions, true_labels)]\n", + " pred_tags = [p_i for p in pred_tags for p_i in p[1:-1]]\n", + " # pred_tags = [p_i for p, l in zip(predictions, true_labels)\n", + " # for p_i, l_i in zip(p, l) if l_i != 2]\n", + " # valid_tags = [l_i for l in true_labels\n", + " # for l_i in l if l_i != 2]\n", + " valid_tags = [[l_i for l_i in l if l_i!=2] for l in true_labels]\n", + " valid_tags = [l_i for l in valid_tags for l_i in l[1:-1]]\n", + " print(\"Validation Accuracy: {}\".format(accuracy_score(pred_tags, valid_tags)))\n", + " print(\"Validation F1-Score: {}\".format(f1_score(pred_tags, valid_tags)))\n", + " val_acc.append(accuracy_score(pred_tags, valid_tags))\n", + " val_f1.append(f1_score(pred_tags, valid_tags))\n", + " print()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "\rTrain: 0%| | 0/2729 [00:00\"Open" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "k_ASTuqZ6daT" + }, + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import pickle\n", + "from tqdm import tqdm, trange\n", + "\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n", + "from torchtext import data" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "ZYFjW-0i3LO6" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "fIU23g6_D1zp", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "outputId": "1ae23f24-879f-4180-e8ce-d6be0449e81c" + }, + "source": [ + "\n", + "!pip install transformers==2.6.0\n", + "!pip install seqeval\n", + "!pip install urllib3 --upgrade" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Collecting transformers==2.6.0\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/4c/a0/32e3a4501ef480f7ea01aac329a716132f32f7911ef1c2fac228acc57ca7/transformers-2.6.0-py3-none-any.whl (540kB)\n", + "\u001b[K |████████████████████████████████| 542kB 9.3MB/s \n", + "\u001b[?25hCollecting tokenizers==0.5.2\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/d1/3f/73c881ea4723e43c1e9acf317cf407fab3a278daab3a69c98dcac511c04f/tokenizers-0.5.2-cp36-cp36m-manylinux1_x86_64.whl (3.7MB)\n", + "\u001b[K |████████████████████████████████| 3.7MB 16.0MB/s \n", + "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers==2.6.0) (1.18.5)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers==2.6.0) (2.23.0)\n", + "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers==2.6.0) (4.41.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers==2.6.0) (3.0.12)\n", + "Collecting sacremoses\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n", + "\u001b[K |████████████████████████████████| 890kB 51.1MB/s \n", + "\u001b[?25hCollecting sentencepiece\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/e5/2d/6d4ca4bef9a67070fa1cac508606328329152b1df10bdf31fb6e4e727894/sentencepiece-0.1.94-cp36-cp36m-manylinux2014_x86_64.whl (1.1MB)\n", + "\u001b[K |████████████████████████████████| 1.1MB 49.6MB/s \n", + "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==2.6.0) (2019.12.20)\n", + "Collecting boto3\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/25/39/87f03b1599e4bfe3e8eaa57ff2ae4c2e99e813ad94e29e9b065df6c8f1e8/boto3-1.16.20-py2.py3-none-any.whl (129kB)\n", + "\u001b[K |████████████████████████████████| 133kB 55.8MB/s \n", + "\u001b[?25hRequirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.6.0) (2020.6.20)\n", + "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.6.0) (1.24.3)\n", + "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.6.0) (2.10)\n", + "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.6.0) (3.0.4)\n", + "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.6.0) (1.15.0)\n", + "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.6.0) (7.1.2)\n", + "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.6.0) (0.17.0)\n", + "Collecting s3transfer<0.4.0,>=0.3.0\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/69/79/e6afb3d8b0b4e96cefbdc690f741d7dd24547ff1f94240c997a26fa908d3/s3transfer-0.3.3-py2.py3-none-any.whl (69kB)\n", + "\u001b[K |████████████████████████████████| 71kB 11.0MB/s \n", + "\u001b[?25hCollecting botocore<1.20.0,>=1.19.20\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/7f/c7/118b4be030bd181b451442e00116aedb445664d7e363b826c9c9bf842456/botocore-1.19.20-py2.py3-none-any.whl (6.8MB)\n", + "\u001b[K |████████████████████████████████| 6.8MB 57.4MB/s \n", + "\u001b[?25hCollecting jmespath<1.0.0,>=0.7.1\n", + " Downloading https://files.pythonhosted.org/packages/07/cb/5f001272b6faeb23c1c9e0acc04d48eaaf5c862c17709d20e3469c6e0139/jmespath-0.10.0-py2.py3-none-any.whl\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.20.0,>=1.19.20->boto3->transformers==2.6.0) (2.8.1)\n", + "Building wheels for collected packages: sacremoses\n", + " Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893257 sha256=cbad80042e5d2ccb5558f5a8c6f8e4b6f82cd9b631648177b8fa5c97522df2be\n", + " Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n", + "Successfully built sacremoses\n", + "\u001b[31mERROR: botocore 1.19.20 has requirement urllib3<1.27,>=1.25.4; python_version != \"3.4\", but you'll have urllib3 1.24.3 which is incompatible.\u001b[0m\n", + "Installing collected packages: tokenizers, sacremoses, sentencepiece, jmespath, botocore, s3transfer, boto3, transformers\n", + "Successfully installed boto3-1.16.20 botocore-1.19.20 jmespath-0.10.0 s3transfer-0.3.3 sacremoses-0.0.43 sentencepiece-0.1.94 tokenizers-0.5.2 transformers-2.6.0\n", + "Collecting seqeval\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/9d/2d/233c79d5b4e5ab1dbf111242299153f3caddddbb691219f363ad55ce783d/seqeval-1.2.2.tar.gz (43kB)\n", + "\u001b[K |████████████████████████████████| 51kB 4.0MB/s \n", + "\u001b[?25hRequirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.6/dist-packages (from seqeval) (1.18.5)\n", + "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.6/dist-packages (from seqeval) (0.22.2.post1)\n", + "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.4.1)\n", + "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.21.3->seqeval) (0.17.0)\n", + "Building wheels for collected packages: seqeval\n", + " Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for seqeval: filename=seqeval-1.2.2-cp36-none-any.whl size=16171 sha256=54a26ea3bae1986c0a304a413824a89768baa75b42cb8e5e8fbe019762ffcb60\n", + " Stored in directory: /root/.cache/pip/wheels/52/df/1b/45d75646c37428f7e626214704a0e35bd3cfc32eda37e59e5f\n", + "Successfully built seqeval\n", + "Installing collected packages: seqeval\n", + "Successfully installed seqeval-1.2.2\n", + "Collecting urllib3\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/f5/71/45d36a8df68f3ebb098d6861b2c017f3d094538c0fb98fa61d4dc43e69b9/urllib3-1.26.2-py2.py3-none-any.whl (136kB)\n", + "\u001b[K |████████████████████████████████| 143kB 8.9MB/s \n", + "\u001b[31mERROR: requests 2.23.0 has requirement urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1, but you'll have urllib3 1.26.2 which is incompatible.\u001b[0m\n", + "\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n", + "\u001b[?25hInstalling collected packages: urllib3\n", + " Found existing installation: urllib3 1.24.3\n", + " Uninstalling urllib3-1.24.3:\n", + " Successfully uninstalled urllib3-1.24.3\n", + "Successfully installed urllib3-1.26.2\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.colab-display-data+json": { + "pip_warning": { + "packages": [ + "urllib3" + ] + } + } + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "ZMVZqu3xkXb6", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "ddd9c251-a282-4f9e-fa03-634b9edb87a0" + }, + "source": [ + "from google.colab import drive \n", + "drive.mount('/content/gdrive')" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Mounted at /content/gdrive\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "yIrWNgXi_YQI", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "45a3c77c-ef60-4f6f-f116-bf007b22c306" + }, + "source": [ + "cd gdrive/'My Drive'" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/content/gdrive/My Drive\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "WQJcjsK_EVSF", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "a75829d6-14d3-43ca-f62d-312d3f7e4560" + }, + "source": [ + "print(torch.__version__)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "1.6.0+cu101\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "rcIEZj8ECihS", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "252be197-6752-444e-d593-8fde8d702d07" + }, + "source": [ + "|" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Mounted at /content/gdrive\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "KmLA0BblCja8" + }, + "source": [ + "train_path = '/content/drive/MyDrive/ToxicSpan_CS669V/processed/finaltrain.pkl'\n", + "val_path = '/content/drive/MyDrive/ToxicSpan_CS669V/processed/finaldev.pkl'\n", + "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "fGqBi7YrCtrM" + }, + "source": [ + "with open(train_path, 'rb') as f:\n", + " train_data = pickle.load(f)\n", + " f.close()\n", + "\n", + "with open(val_path, 'rb') as f:\n", + " val_data = pickle.load(f)\n", + " f.close()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "a0UeXFYvCxuk" + }, + "source": [ + "import transformers\n", + "from transformers import XLNetTokenizer, XLNetForTokenClassification, AdamW\n", + "from transformers import get_linear_schedule_with_warmup\n", + "\n", + "from keras.preprocessing.sequence import pad_sequences\n", + "from seqeval.metrics import accuracy_score\n", + "from sklearn.metrics import f1_score, confusion_matrix\n", + "\n", + "torch.manual_seed(42)\n", + "np.random.seed(42)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "eGVFGvHMDxCq" + }, + "source": [ + "X_train = train_data['token_final']\n", + "X_val = val_data['token_final']\n", + "Y_train = train_data['target_final']\n", + "Y_val = val_data['target_final']" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "VSzspvoZEhM5" + }, + "source": [ + "MAX_LEN = 300\n", + "BATCH_SIZE = 16" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "rtmiaBI9EmYR", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 67, + "referenced_widgets": [ + "f3cd39b0f0044138be5f132d5e1a7990", + "aa0d0be4a43749f080d1d909a8066e33", + "b281ef27d4c64f989296260633854189", + "a5a404fb53a44492b62dcc6eed5dfbeb", + "fa9881ec5cd343af93f767db79649b91", + "0118091b589644d591b1e34a1fd1e01a", + "4925b99fc4164037bc74bd86cdeaa17d", + "3782464e953c4996b3c3c9ef91376e28" + ] + }, + "outputId": "89400abb-3b9a-4599-ab27-768ff3a8a593" + }, + "source": [ + "\n", + "### tokenizer\n", + "\n", + "tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased', do_lower_case = False)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f3cd39b0f0044138be5f132d5e1a7990", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=798011.0, style=ProgressStyle(descripti…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "mGdfMUvSEw25" + }, + "source": [ + "def tokenize_xlnet(x, y):\n", + " sentence = []\n", + " labels = []\n", + " for word, label in zip(x, y):\n", + " tokenized_word = tokenizer.tokenize(word)\n", + " sentence.extend(tokenized_word)\n", + " labels.extend([label for i in range(len(tokenized_word))])\n", + " return(sentence, labels)\n", + "\n", + "def tokenize_xlnet(x, y):\n", + " sentence = []\n", + " labels = []\n", + " for word, label in zip(x, y):\n", + " tokenized_word = tokenizer.tokenize(word)\n", + " sentence.extend(tokenized_word)\n", + " labels.extend([label for i in range(len(tokenized_word))])\n", + " return(sentence, labels)\n", + "\n", + "\n", + "len_train = len(X_train)\n", + "len_val = len(X_val)\n", + "\n", + "for i in range(len_train):\n", + " X_train[i], Y_train[i] = tokenize_xlnet(X_train[i], Y_train[i])\n", + "\n", + "for i in range(len_val):\n", + " X_val[i], Y_val[i] = tokenize_xlnet(X_val[i], Y_val[i])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "i_7tuayJFXxe" + }, + "source": [ + "CLASSES = {'0':0, '1':1, '[PAD]':2}" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "GEGP_6XTFEtD" + }, + "source": [ + "ones = 0\n", + "zeros = 0\n", + "total = 0\n", + "for y in Y_train:\n", + " ones += np.sum(np.array(y))\n", + " zeros += len(y) - np.sum(np.array(y))\n", + " total += len(y)\n", + "for y in Y_val:\n", + " ones += np.sum(np.array(y))\n", + " zeros += len(y) - np.sum(np.array(y))\n", + " total += len(y)\n", + "# print(ones, zeros)\n", + "class_weights = torch.tensor([zeros/zeros, zeros/ones], dtype=torch.float32)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "2bJxRXfMFOkU" + }, + "source": [ + "X_train_id = pad_sequences([tokenizer.convert_tokens_to_ids(text) for text in X_train], maxlen = MAX_LEN, dtype='long', value=0.0, truncating='post', padding = 'post')\n", + "Y_train_id = pad_sequences(Y_train, maxlen=MAX_LEN, value=CLASSES['[PAD]'], dtype='long', truncating='post', padding='post')\n", + "X_val_id = pad_sequences([tokenizer.convert_tokens_to_ids(text) for text in X_val], maxlen = MAX_LEN, dtype='long', value=0.0, truncating='post', padding = 'post')\n", + "Y_val_id = pad_sequences(Y_val, maxlen=MAX_LEN, value=CLASSES['[PAD]'], dtype='long', truncating='post', padding='post')\n", + "\n", + "\n", + "\n", + "X_train_id = pad_sequences([tokenizer.convert_tokens_to_ids(text) for text in X_train], maxlen = MAX_LEN, dtype='long', value=0.0, truncating='post', padding = 'post')\n", + "Y_train_id = pad_sequences(Y_train, maxlen=MAX_LEN, value=CLASSES['[PAD]'], dtype='long', truncating='post', padding='post')\n", + "X_val_id = pad_sequences([tokenizer.convert_tokens_to_ids(text) for text in X_val], maxlen = MAX_LEN, dtype='long', value=0.0, truncating='post', padding = 'post')\n", + "Y_val_id = pad_sequences(Y_val, maxlen=MAX_LEN, value=CLASSES['[PAD]'], dtype='long', truncating='post', padding='post')" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "_YnGi4WZujSd", + "outputId": "5446215a-de50-4d31-8f42-1f573392e25d" + }, + "source": [ + "X_train[0]" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "['▁only',\n", + " '▁a',\n", + " '▁fool',\n", + " '▁would',\n", + " '▁not',\n", + " '▁be',\n", + " '▁against',\n", + " '▁this',\n", + " '▁president',\n", + " '▁he',\n", + " '▁is',\n", + " '▁massive',\n", + " 'ly',\n", + " '▁and',\n", + " '▁obviously',\n", + " '▁unfit',\n", + " '▁for',\n", + " '▁office']" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 20 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "lPSrJbVpFRNk" + }, + "source": [ + "def get_attention_mask(x):\n", + " return([[int(i!=0) for i in text] for text in x])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "EpHSyDLOFqFN" + }, + "source": [ + "attention_mask_train = get_attention_mask(X_train_id)\n", + "attention_mask_val = get_attention_mask(X_val_id)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "Eji4LP8PFf8F" + }, + "source": [ + "X_train_id = torch.tensor(X_train_id)\n", + "Y_train_id = torch.tensor(Y_train_id)\n", + "X_val_id = torch.tensor(X_val_id)\n", + "Y_val_id = torch.tensor(Y_val_id)\n", + "attention_mask_train = torch.tensor(attention_mask_train)\n", + "attention_mask_val = torch.tensor(attention_mask_val)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "YwMyK0CoFj5d" + }, + "source": [ + "train_data = TensorDataset(X_train_id, attention_mask_train, Y_train_id)\n", + "train_sampler = RandomSampler(train_data)\n", + "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=BATCH_SIZE)\n", + "\n", + "val_data = TensorDataset(X_val_id, attention_mask_val, Y_val_id)\n", + "val_sampler = RandomSampler(val_data)\n", + "val_dataloader = DataLoader(val_data, sampler=val_sampler, batch_size=BATCH_SIZE)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "2Q26PbGlFrdv", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 116, + "referenced_widgets": [ + "3e34c5538b2f4dbf9dc06b7ed3994ff6", + "fa01bca5b1934cf6bf15b07c30b4bec1", + "c1a1d8cf32ba4630aaffb8843abde5d7", + "f6e632c00ce24f5992650b00e21e337b", + "48f4da3196d746b08ddd5425bdb811fe", + "3756d5c919a54bc998ecb67bdb07eda0", + "3d40d12971354c3ebac7f3c4cc73ecd8", + "4440472b8111448a85dccfc50e33b9d4", + "20368c3b7a6f4f1a8dfe15c91490b39f", + "2f1c9109ec614cc1896344d9fb0c19b3", + "82a95a7c47754909931cb09e5720b200", + "0496b557a3944b7193f76a9acbb4bea6", + "eb787c25ca324d98b95ae2f46543416b", + "d57f37417b2641849b35ae023d28fb61", + "787078509d8d4f548ec8c505a2fcd495", + "1dc50573890e4b10a95e3cdd3e433647" + ] + }, + "outputId": "309cc2e6-bbf9-470e-8333-d08d05ac404b" + }, + "source": [ + "model = XLNetForTokenClassification.from_pretrained(\n", + " 'xlnet-base-cased',\n", + " num_labels = 2,\n", + " output_attentions = False,\n", + " output_hidden_states = False\n", + ")" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3e34c5538b2f4dbf9dc06b7ed3994ff6", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=760.0, style=ProgressStyle(description_…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "20368c3b7a6f4f1a8dfe15c91490b39f", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=467042463.0, style=ProgressStyle(descri…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "eWA3cYWEGEaE", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "baf5081d-9c7b-47dd-b674-899bb22ef30a" + }, + "source": [ + "model.cuda()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "XLNetForTokenClassification(\n", + " (transformer): XLNetModel(\n", + " (word_embedding): Embedding(32000, 768)\n", + " (layer): ModuleList(\n", + " (0): XLNetLayer(\n", + " (rel_attn): XLNetRelativeAttention(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (ff): XLNetFeedForward(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (layer_1): Linear(in_features=768, out_features=3072, bias=True)\n", + " (layer_2): Linear(in_features=3072, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (1): XLNetLayer(\n", + " (rel_attn): XLNetRelativeAttention(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (ff): XLNetFeedForward(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (layer_1): Linear(in_features=768, out_features=3072, bias=True)\n", + " (layer_2): Linear(in_features=3072, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (2): XLNetLayer(\n", + " (rel_attn): XLNetRelativeAttention(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (ff): XLNetFeedForward(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (layer_1): Linear(in_features=768, out_features=3072, bias=True)\n", + " (layer_2): Linear(in_features=3072, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (3): XLNetLayer(\n", + " (rel_attn): XLNetRelativeAttention(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (ff): XLNetFeedForward(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (layer_1): Linear(in_features=768, out_features=3072, bias=True)\n", + " (layer_2): Linear(in_features=3072, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (4): XLNetLayer(\n", + " (rel_attn): XLNetRelativeAttention(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (ff): XLNetFeedForward(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (layer_1): Linear(in_features=768, out_features=3072, bias=True)\n", + " (layer_2): Linear(in_features=3072, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (5): XLNetLayer(\n", + " (rel_attn): XLNetRelativeAttention(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (ff): XLNetFeedForward(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (layer_1): Linear(in_features=768, out_features=3072, bias=True)\n", + " (layer_2): Linear(in_features=3072, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (6): XLNetLayer(\n", + " (rel_attn): XLNetRelativeAttention(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (ff): XLNetFeedForward(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (layer_1): Linear(in_features=768, out_features=3072, bias=True)\n", + " (layer_2): Linear(in_features=3072, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (7): XLNetLayer(\n", + " (rel_attn): XLNetRelativeAttention(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (ff): XLNetFeedForward(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (layer_1): Linear(in_features=768, out_features=3072, bias=True)\n", + " (layer_2): Linear(in_features=3072, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (8): XLNetLayer(\n", + " (rel_attn): XLNetRelativeAttention(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (ff): XLNetFeedForward(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (layer_1): Linear(in_features=768, out_features=3072, bias=True)\n", + " (layer_2): Linear(in_features=3072, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (9): XLNetLayer(\n", + " (rel_attn): XLNetRelativeAttention(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (ff): XLNetFeedForward(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (layer_1): Linear(in_features=768, out_features=3072, bias=True)\n", + " (layer_2): Linear(in_features=3072, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (10): XLNetLayer(\n", + " (rel_attn): XLNetRelativeAttention(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (ff): XLNetFeedForward(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (layer_1): Linear(in_features=768, out_features=3072, bias=True)\n", + " (layer_2): Linear(in_features=3072, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (11): XLNetLayer(\n", + " (rel_attn): XLNetRelativeAttention(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (ff): XLNetFeedForward(\n", + " (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (layer_1): Linear(in_features=768, out_features=3072, bias=True)\n", + " (layer_2): Linear(in_features=3072, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (classifier): Linear(in_features=768, out_features=2, bias=True)\n", + ")" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 19 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "n14Heq0XlvTw", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "1c2ad0d9-6494-4e4c-9380-790ad1ebf95f" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "There are 1 GPU(s) available.\n", + "We will use the GPU: Tesla T4\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "m_Uyv6vPF7jO" + }, + "source": [ + "FINE_TUNING = True\n", + "if FINE_TUNING:\n", + " param_optimizer = list(model.named_parameters())\n", + " no_decay = ['bias', 'gamma', 'beta']\n", + " optimizer_grouped_parameters = [\n", + " {'params' : [p for n,p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay_rate' : 0.01},\n", + " {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],'weight_decay_rate': 0.0}]\n", + "else:\n", + " param_optimizer = list(model.classifier.named_parameters())\n", + " optimizer_grouped_parameters = [{\"params\": [p for n, p in param_optimizer]}]\n", + "\n", + "optimizer = AdamW(optimizer_grouped_parameters, lr=3e-5, eps=1e-8)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "qZ8pFE8uGNzS" + }, + "source": [ + "epochs = 3\n", + "max_grad_norm = 1.0\n", + "\n", + "total_steps = len(train_dataloader) * epochs\n", + "\n", + "scheduler = get_linear_schedule_with_warmup(\n", + " optimizer,\n", + " num_warmup_steps=0,\n", + " num_training_steps=total_steps\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "MgDM2sFYGSpo" + }, + "source": [ + "criterion = nn.CrossEntropyLoss(weight=class_weights)\n", + "criterion = criterion.to(device)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "gou8IL87Dvaq" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "HlBJkdf8GVH0" + }, + "source": [ + "def get_text_lengths(masks):\n", + " lengths = []\n", + " for mask in masks:\n", + " lengths.append(torch.sum(mask).item())\n", + " return(lengths)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "oZlzQh83GaI7", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "3fa1217d-e68b-4bf5-99b0-13695ac8a070" + }, + "source": [ + "train_loss, val_loss = [], []\n", + "train_acc, val_acc = [], []\n", + "train_f1, val_f1 = [], []\n", + "\n", + "l = 100\n", + "\n", + "for epoch in trange(epochs, desc = 'Epoch'):\n", + " print(epoch)\n", + " model.train()\n", + " t_loss, t_acc = 0, 0\n", + " predictions, true_labels = [], []\n", + " for step, batch in enumerate(train_dataloader):\n", + " batch = tuple(t.to(device) for t in batch)\n", + " b_input_id, b_input_mask, b_labels = batch\n", + " optimizer.zero_grad()\n", + " outputs = model(b_input_id, token_type_ids=None, attention_mask=b_input_mask, labels = b_labels)\n", + " active_loss = b_input_mask.view(-1) == 1\n", + " active_logits = outputs[1].view(-1, 2)\n", + " active_labels = torch.where(active_loss, b_labels.view(-1), torch.tensor(criterion.ignore_index).type_as(b_labels))\n", + " loss = criterion(active_logits, active_labels)\n", + " loss.backward()\n", + " t_loss += loss.item()\n", + " torch.nn.utils.clip_grad_norm_(parameters=model.parameters(), max_norm=max_grad_norm)\n", + " optimizer.step()\n", + " scheduler.step()\n", + "\n", + " logits = outputs[1].detach().cpu().numpy()\n", + " label_ids = b_labels.to('cpu').numpy()\n", + "\n", + " predictions.extend([list(p) for p in np.argmax(logits, axis=2)])\n", + " true_labels.extend(label_ids)\n", + " \n", + " print(f\"Train Loss : {t_loss/len(train_dataloader)}\")\n", + " train_loss.append(t_loss/len(train_dataloader))\n", + " pred_tags = [p_i for p, l in zip(predictions, true_labels)\n", + " for p_i, l_i in zip(p, l) if l_i != 2]\n", + " valid_tags = [l_i for l in true_labels\n", + " for l_i in l if l_i != 2]\n", + " train_acc.append(accuracy_score(pred_tags, valid_tags))\n", + " train_f1.append(f1_score(pred_tags, valid_tags))\n", + " print(\"Train Accuracy: {}\".format(accuracy_score(pred_tags, valid_tags)))\n", + " print(\"Train F1-Score: {}\".format(f1_score(pred_tags, valid_tags)))\n", + " print()\n", + "\n", + " model.eval()\n", + " v_loss, v_accuracy = 0, 0\n", + " predictions , true_labels = [], []\n", + " for batch in val_dataloader:\n", + " batch = tuple(t.to(device) for t in batch)\n", + " b_input_id, b_input_mask, b_labels = batch\n", + " with torch.no_grad():\n", + " outputs = model(b_input_id, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n", + " \n", + " active_loss = b_input_mask.view(-1) == 1\n", + " active_logits = outputs[1].view(-1, 2)\n", + " active_labels = torch.where(active_loss, b_labels.view(-1), torch.tensor(criterion.ignore_index).type_as(b_labels))\n", + " loss = criterion(active_logits, active_labels)\n", + " \n", + " logits = outputs[1].detach().cpu().numpy()\n", + " label_ids = b_labels.to('cpu').numpy()\n", + "\n", + " v_loss += loss.item()\n", + " predictions.extend([list(p) for p in np.argmax(logits, axis=2)])\n", + " true_labels.extend(label_ids)\n", + " \n", + " v_loss = v_loss/len(val_dataloader)\n", + " val_loss.append(v_loss)\n", + " # if(v_loss < l):\n", + " # l = v_loss\n", + " # print(\"Model Checkpoint\")\n", + " # torch.save(model, f'./savedmodel/xlnet-{epoch}.pt')\n", + " \n", + " print(f\"Validation Loss : {v_loss}\")\n", + " pred_tags = [p_i for p, l in zip(predictions, true_labels)\n", + " for p_i, l_i in zip(p, l) if l_i != 2]\n", + " valid_tags = [l_i for l in true_labels\n", + " for l_i in l if l_i != 2]\n", + " print(\"Validation Accuracy: {}\".format(accuracy_score(pred_tags, valid_tags)))\n", + " print(\"Validation F1-Score: {}\".format(f1_score(pred_tags, valid_tags)))\n", + " val_acc.append(accuracy_score(pred_tags, valid_tags))\n", + " val_f1.append(f1_score(pred_tags, valid_tags))\n", + " print()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "\n", + "\n", + "Epoch: 0%| | 0/3 [00:00\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtorch\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mversion\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mNameError\u001b[0m: name 'version' is not defined" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "urz2F-y2m8r0" + }, + "source": [ + "torch.save(model, './savedmodels/xlnet3epochs.pt')" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "1XvV0h9gbbfE", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "e66be1a4-fdb8-421f-e43a-f9ea8a2f68e0" + }, + "source": [ + "cd Toxic\\ Spam" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/content/gdrive/My Drive/Toxic Spam\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "KWxxLoFMbwok", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "30ea888e-815a-4869-8e66-b4681724888f" + }, + "source": [ + "ls" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Attention-Improved-LSTM.ipynb LSTM.ipynb test_split.csv xlnet2.ipynb\n", + "final.pkl \u001b[0m\u001b[01;34mprocessed\u001b[0m/ train.json xlnet.ipynb\n", + "Ibilstm1.pt processed.csv train_split.csv\n", + "Improved-LSTM.ipynb Roberta.ipynb val.json\n", + "losses1.json \u001b[01;34msavedmodels\u001b[0m/ val_split.csv\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "L1kN02-zc8sh", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 495 + }, + "outputId": "178a612c-de1b-46e9-80ae-5f2ba100eff7" + }, + "source": [ + "import seaborn as sns\n", + "import matplotlib.pyplot as plt\n", + "epoch=[]\n", + "for i in range(3):\n", + " epoch.append(i+1)\n", + "sns.set(font_scale=1.4)\n", + "\n", + "plt.figure(figsize=(7,7))\n", + "dic={'Train_Loss':train_loss,'Val_Loss': val_loss}\n", + "x=pd.DataFrame(dic)\n", + "ax = sns.lineplot(data=x)\n", + "ax.set(ylim=(0,1),xlabel='No. of epochs',ylabel='Loss',title='XLNET')\n", + "ax.set_xticks(range(3)) # <--- s\n", + "ax.set_xticklabels(['1','2','3'])" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[Text(0, 0, '1'), Text(0, 0, '2'), Text(0, 0, '3')]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 71 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcwAAAHNCAYAAACTh0sjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXxU9b3/8deZNdskmWwEEpJA2DcFUSMgIIIU0FJxuXTzUq1LU9FKcaO/XteKbV0utuWqV61abZVaxSu2KCgSd0VkR4FAICyBkIRMAkkmM3N+f0wYSMMygYTJ8n4+Hjx0vmfJJ/GYN9/v+Z7vMUzTNBEREZETskS6ABERkfZAgSkiIhIGBaaIiEgYFJgiIiJhUGCKiIiEQYEpIiISBgWmiIhIGBSYIm3cXXfdxaBBgygsLGyybcGCBfTt25c33ngDgHHjxnHddded9Hx9+/ZlypQpBAKBJtuHDh3KXXfdFfq8c+dO+vbte9w/jzzyCJ9//vkJ9zn6z86dO0/zJyISGbZIFyAiJ3bHHXewbNkyfv3rX/Pyyy9jGAYApaWl/P73vycvL4/LL7+82efdsmUL//rXv5gyZUpY+0+ePJmxY8c2ae/Tpw+pqan87ne/a9Q+d+5c0tLSmgR4UlJSs2sVaQsUmCJtXFJSEnfeeSd33303f//737n66qsBePDBB6mtreW+++5r9jntdjvZ2dnMnz+fSZMmYbGcfLCpf//+TJ069bjb/33bvHnzSE1NPeExIu2JhmRF2oFp06Zx/vnn88gjj7B//36WLVvG4sWLuemmm8jJyWn2+QzDID8/ny1btvDPf/6z5QsW6YAUmCLtxH333UdNTQ333nsv9913H7m5uVx//fWnfL5JkybRq1cv5s+ff8x7mf+upqaG8vLyJn/q6+tPuQaR9kSBKdJO9OjRgxtvvJElS5ZQUlLC/fffj8PhOOXzWSwWfv7zn1NYWBhWL3P+/PlccMEFTf58+umnp1yDSHuie5gi7Yjb7Q79c+DAgad9vu985zvMnz+fP/3pT0yePPmE9zKvvPLKY04Q6t+//2nXIdIeqIcp0k7s3buXxx57jN69e1NRUcEf/vCH0z6nxWIhPz+frVu38vbbb59w3+zsbEaMGNHkz+EQF+noFJgi7cRvfvMbfD4f8+fPZ9q0abzwwgt8++23p33eSZMm0bt377DvZYp0VgpMkXbg/fff55133iE/P5+srCxuv/12XC4X99xzD6f7DvjDM2a3bt3KokWLWqhikY5HgSnSxh08eJD777+fPn36cO211wLBe5h33HEHX3/9NQsWLDjtrzFp0iT69OmjXqbICWjSj0gbN2/ePEpKSnjsscew2+2h9mnTpvH666/z6KOPMn78eJKTk4HgUnbz589vcp7evXszYcKEY36Nw73MX/ziF8etY+PGjbz55ptN2jMyMhg+fHhzvy2RdkeBKdKGrVu3jpdeeomrr76aYcOGNdl+3333MXXqVObOncsjjzwCQFFREfPmzWuy7+TJk48bmBCcMdunTx82bdp0zO3//Oc/j/n4ycSJExWY0ikY5uneADkN27dv59lnn2X16tVs3ryZnj17hn0PZeHChTz55JPs2rWLrKwsfv7znzN58uRWrlhERDqriPYwN2/ezPLlyznrrLMIBAJhT15YvHgxd955JzfccAMjR45k6dKlzJo1i9jYWMaMGdPKVYuISGcU0R5mIBAIPSh91113sW7durB6mIcnKBw97HTttdfi8Xh47bXXWq1eERHpvCI6SzacNyT8u+LiYrZu3dpkxZFLL72UtWvXUl5e3lLliYiIhLS7x0q2bt0KQG5ubqP2Xr16NdouIiLSktpdYFZWVgIQHx/fqD0hIaHRdhERkZbU7gJTREQkEtrdc5iHe5Iej4fU1NRQ++Ge5eHt4aqoOEggcHrznpKT4ygrqz6tc4hEkq5h6Qha4jq2WAzc7thjbmt3gdmzZ08geK/y6PuYhYWFjbaHKxAwTzswD59HpD3TNSwdQWtex+1uSLZ79+707NmzyYojixYtYvDgwSQlJUWoMhER6cgi2sOsqalh+fLlAOzatYvq6moWL14MwODBg8nIyGDOnDksXLiQDRs2hI675ZZbuO2228jKymLEiBG89957fPzxxzz11FMR+T5ERKTji2hglpWVceuttzZqO/x57ty5TJs2jUAggN/vb7TPpEmTqK2t5cknn+TZZ58lKyuLRx99VKv8iIhIq4noSj9tQVlZ9WmPeaemuigtrWqhikTOPF3D0hG0xHVssRgkJ8cdc1u7m/RzptXUHKS6uhK/v/64++zbZ9E7BDsBq9VOXFwC0dHHnkEnIh2bAvME6uu9VFVVkJiYgt3uxDCMY+5ns1nw+RSYHZlpmtTX13HgwH5sNjt2uyPSJYnIGdbuZsmeSVVVB4iLS8DhiDpuWErnYBgGDkcUsbEJVFcfiHQ5IhIBCswT8Pm8OJ3RkS5D2pCoqGjq672RLkNEIkCBeQKBgB+LxRrpMqQNsVisBAL+k+8oIh2OAvMkNBQrR9P1INJ5KTBFRETCoFmyHdyoUcNPus9PfnI911134ymd/9lnn+KVV15iyZIPT+n44/nNb+7lm2828Je/LGjR84qInCoFZgf35JN/bvT5ppt+wpVX/gfjx38n1JaWlnbK57/ssu8xYsSoUz5eRKS9UGB2cIMGDW7SlpaWfsz2w+rqanE6o8I6f1paF9LSupxyfSIi7YXuYXZy//znW4waNZz169fxy1/ewvjxo3jssd8BsGDBX/npT69h4sQxXHrpeGbNmsnWrYWNjn/22aeYMOHC0OeVK1cwatRwvvzyc+6//9dMmDCaadOm8MwzT7b4akirV3/Nz352HePGjWTSpHHcd9//o6xsf6N9/vrXF5k+/XLGjRvBlCkXM3PmjRQVbQt7u4jIYephCgD33juHKVO+yw9+8GMcjuAqNnv37mXatKtIT+9KTU0Nb731Bj/72bW8/PI/SElJOeH5fv/7hxg/fiIPPfR7vvzyM55//hm6d89i4sTJLVLvN99s5Be/yGfIkLO5//6HqKqq4qmn/sStt+bz7LN/wel0snjx2zz11J/46U9vYuDAwRw6dJC1a9dw8OBBgJNuFxE5mgKzmT5eu4eP1uxp1GYYcKaWsB81pCsjB3dt8fNedtn3uOaaaxu1zZx5W+jf/X4/552Xx+WXT2bp0sVMn/6jE55v9OiLuOGGfADOPfd8vvjicz744L0WC8wXX3wOtzuJRx55ArvdDkBWVg433jiD9957l8mTL2PDhnXk5vbmxz/+Sei4UaOOvNHmZNtFRI6mIVkBYNSo0U3a1q1by223/ZwpUy5mzJjzueiiCzhwoILi4h0nPd/55+c1+pyT04N9+/a1WL1r1nzNhReOCYUlwMCBg+jatRurV38NQJ8+/di8+VueeOJRVq1aSX194wX0T7ZdRORo6mE208jBTXt4HWHxdbc7udHnkpISZs26mT59+vLLX95NSkoKDoeD//f/7sTrPfnScC5XfKPPdrsdr7euxeqtqqoiKSm5SbvbnYTH4wFg8uTLqKmp4f/+73UWLPgbMTGxTJw4mfz8W4iOjj7pdhGRoykwBWi6gs3nn39CTc0hHnroEeLjj4Tf4TCKNJcrnoqK8ibtFRXl9OjREwCLxcJVV03nqqums3//fj74YCnz5z9BTEwMP/vZzJNuFxE5moZk5Zjq6uowDAOb7cjfqT76aDmHDrWNCTFDhpxNQcEH+Hy+UNvGjevZs2c3Z501tMn+KSkpXHnldPr3H0hR0dZmbxcRUQ9Tjumcc84F4KGH7mXq1CsoLt7BSy89T2Ki+4zVcPDgQZYtW9qkfciQs7nmmmv52c+uZfbsW7jqqu9TVeXhqaf+RE5OTy6++BIAfve73xAX52LgwMHEx8ezYcM61q9fy803/yKs7SLS9gWqy/CufRfneVe2+tdSYMox5eb24le/upfnnnuaO++cRW5uLvfc8xseeeShM1bDvn17+fWv72rS/tvfPs7IkRfy+ON/4qmn/sSvf30XDoeDvLwRzJx5G06nE4DBg8/irbcWsmjRm9TW1tK1a1duuulmrrxyeljbRaTtCtR48H79FvUbloEBtpxhkJ7Uql/TMM0z9UBE21RWVk0gcOwfQUnJdtLTs096jo4w6UfCF+510Z6kprooLa2KdBkiJ2XWHcS7ZjHete+Cvx5731E4hk3FEpfcItexxWKQnBx3zG3qYYqISLvhL9mM9+u3sPU8D+fwaVgS08/Y11Zgyhl39ESdYzl6opGIdG6m30f9N8sJlBUTNXoG1qyziLnqIazubme8Fv1mkjNu7Ni8E27/6KMVZ6gSEWmrzEAA35ZPqftqIWZVKdb0Ppg+L4bNEZGwBAWmRMAzz7wY6RJEpA3z7VhF3ed/J1CxC0tyNlGTZmHNHNzkefEzTYEpZ1y/fgMiXYKItEFmIIBhseDfWwgBP1Hj87H1GI5htI0lAxSYIiISUf59hdR9+Q+sGQNwnn0pjqGX4TjnexgWa6RLa0SBKSIiEeEv34n3y3/g2/41RpQLW+75ABg2R4QrOzYFpoiInFGmv57a5c/h2/IZ2KNwDJ+GY/AlGPaoSJd2QgpMERE5I8zaaoyoOAyrHdNbg+OsSTjOmowRdeyFAtoaBaaIiLQqs7Ya7+p/4l23lOhJt2Hr1p/oibdGfNZrcykwRUSkVZjeGrzrluBd/S+or8XWKw9LXArQ9JWC7UHbmKsrrebOO2/jqqu+e9ztixe/zahRw9mwYd1Jz3XllZfx2GO/Dftr33zzDdxxh978IdIZ+fdt5eArd+Bd8Tq2jP7EXPkA0eNuxBKfGunSTpkCs4O75JJJ7Nmzm7VrVx9z+7vvLiYzszsDBgw6w5WJSEdjBvz4928HwOLuhrVbP2K+92uiL7kFa1JmhKs7fQrMDm7UqNHExMSyZMniJtsqKsr56qsvmDDhOxGoTEQ6CtMMUF/4BQf//isO/d9DBGqrMOxRRI//Oda03EiX12J0D7ODczqjGDPmIpYte49bbvllo4XN339/CX6/n7y8kcydez8rV35FWdl+UlNTufDCsVx//U04na07zbug4AOef/5/KSraRmxsHGPGjCM//xZiYmKA4ELtTz89n/fee5fy8jJcrnj69OnLr351L2530km3i0jrMU0Tf/Fa6r78B4Gy7VjcGTjH3YDhbB+zXptLgdkJXHLJd/jXvxaxYsUX5OWNCLUvWfIO/fsPICYmhrg4FzNn3obL5WLXrmL+/OdnKCnZw4MPhn/Psrk++mg5v/rV7YwdezHXX5/Pnj27eeqpP1JcvIN58+YD8NJLz/PGG6/xs5/NpEePnng8laxY8SV1dXVhbReR1uP98h94Vy3CcKUSNfZ6bL0uwLB03IFLBeYpOPTW3EafDcPANE1iLrsbgNpPXiZQtqPJcc4LfoA1JZv6bz+kftNHTbbb+4zC3vdC/Pu3U/fpX5tstyRnETXih82ud9iwc0lOTmHp0ndCgbl79y7WrVvDLbf8kh49ejJz5m2h/QcPPouEhETmzLmdysoDJCQkNvtrhuO5556mX78BPPDAw6G2hIRE7rnnblauXMGwYcHJSOeddz7Tpl0V2mfMmHGhfz/ZdhFpWf792yHgw5qWi63X+Rixbuz9xmBYO36cdNy/CkiI1Wpl/PhLKCj4INTzWrJkcajdNE0WLPgrP/rRVYwbN5KxY/O4++7ZmKZJcXFxq9R06NAhNm/exLhxExq1jx07DqvVypo1qwDo06cfn376Cc8++xQbNqzD7/c32v9k20WkZQQOlFCzdD6HXr+Hui9eA8Ca1B3HwIs7RViCepin5HBP8jCbzYLPFwh9Plkv0N73Qux9LzzudmtKdpOvcbomTJjEq6/+lY8//pBx48azdOk7nHPOuSQlJfPqqy/zxz/+N9///o8555xzcblcFBVt46GH7sPrbZ2hzerqKkzTJDk5uVG71WolISERj6cSgGuuuRaLxcLixW/z5z//LwkJCUydegXXXXcjVqv1pNtF5PQEqsvwfvVmcFTMag8ujD6kc04UVGB2Ev369Sc7O4clSxbTvXsW27Zt5Qc/uAaAZcveY9So0eTn3xLaf9++va1aT1ycC8MwKC8va9Tu9/uprDxAfHwCAA6Hg2uvvYFrr72B3bt38c47/+S5554mLS2N733vypNuF5FTZ5oBDr39O8yqMuwDL8Zx9qVYYhIiXVbEaEi2E5kw4Tt8/vknvPHG33E6nYwZcxEAdXW12O2N3w7w7rtNH0NpSTExMfTu3Yf331/aqH358mX4/X6GDDm7yTHdumXwk59cT2pqGkVF25q9XUROzvQeom7FGwSqyzEMC1GjryX2Px4masQPO3VYgnqYncqECd/hmWee5K23FjJu3HhiYmIBOPfc81mw4G/8/e+vkJ2dw7JlS9m8eVOLfM2ysjKWLVvapP288/K49tobuPvu2dxzzxwmTbqUkpLdPPnkHznnnPMYNmw4AHff/Uv69OlH7959iYmJ4fPPP2Xfvr0MH35eWNtFJDymz0v9+qXUrXob6g5ixLpx9B+LrWvfSJfWZigwO5GMjEwGDRrCunVrmDBhUqh9xozrqays5M9//l8CgQCjRo3mjjvmcNttPz/tr/nttxv59a/vatL+4ouvMGrUGB588Hc8//z/cvfdvyQ2No7x479Dfv7M0H6DB5/NsmVLePXVl/H7/WRmdudXv7qXUaPGhLVdRE7MDPio/6YA78r/wzx0AGv3wTjPvQJrSk6kS2tzDNM0zUgXEUllZdUEAsf+EZSUbCc9Pfuk5/j3ST/SsYV7XbQnqakuSkurIl2GRECgsoSDC+ZgTcvFcd6V7bpH2RLXscVikJx87IUX1MMUEelETNPEv30V9Zs+Imp8PpaEdGKuuA+LO7NdvkHkTFJgyinx+XzH3WYYhh7pEGmDfLs3UvfFawT2FWIkdMGsLseIT8Wa1D3SpbULCkxptj17dp/wlWHp6V157bW3zmBFInIi/v3bqft8Af5d6zFik3CO/gn2PiMxLIqA5tBPS5otJSWVZ5558bjb//0RFRGJDNMMYBgWAgf2ECjbgTPv+9gHXIRh0/+jp0KBKc1mt9vp129ApMsQkeMIeEqp+2ohGBA99npsuedhyzoLwxEd6dLaNQWmiEgHETh0AO/Kt6j/5gMwLDgGTcA0TQzDAgrL06bAPIngxaaZYxLUyZ/CkjasbsUbeFf/CwJ+7P0uxDFsKpZYd6TL6lAUmCdgtdqor/ficDgjXYq0EfX1Xqyd5M0M0vaZ9bVgsWFYbZjeGmw9zsF5zvewJHSJdGkdkv7PP4G4uEQOHCglMTEVu92hnmYnZpom9fVeDhwoxeXS39olskx/PfUbP8D79Vs4hn0Xx8DxOC/4vn5HtTIF5glERwfXWq2s3I/ff/znDi0WC4GAVvrp6KxWGy6XO3RdiJxpZsCPb/Mn1H21ELO6DGvXflhTewIoLM8ABeZJREfHnvQXpJYVE5HWFjh0gJpFvyVwYA+W1B5Ejf4J1oyBCsozSIEpItJGmaZJoGwH1pRsjOgELKk9cJx7BbaccxSUEaDAFBFpg/wlm6n78jX8e74l5vJ7sKb2IPqiGyJdVqemwBQRaUP8ZcXBoNyxGiM6HueIH2FJyox0WYICU0Skzajf8hm17z8Fjmgc512JY+AEDLsea2srFJgiIhEUqC4ncGA3tsxB2LoPxjHsMhyDJ2I4NRu7rVFgiohEQKC2Cu+qt6lfvxTDEUvsDx/FcMbiHD4t0qXJcSgwRUTOINNbg3ftO3jXLAZfHbbeI3GeM1Wv2moH9F9IROQMqv3gGXxFX2HrMRzH8GlY3d0iXZKEKaKBWVRUxAMPPMDKlStxOp1MmTKF2bNnEx194lX1Dx06xPz581m8eDGlpaV06dKF7373u9xwww04HHrPm4i0HWbAR/23H2FNysTapReOc76HY+ilWFN7RLo0aaaIBabH4+Gaa66hW7duzJs3j/LycubOnUt5eTmPP/74CY+99957Wbp0Kbfddhu9e/dmzZo1PPHEE3g8HubMmXOGvgMRkeMzzQC+wi+oW/EGpmcv9gEXY+3SC2ty90iXJqcoYoH5yiuv4PF4WLhwIUlJSQBYrVZmz55Nfn4+vXv3PuZxPp+PxYsX89Of/pQf//jHAOTl5bF7924WLVqkwBSRiDJNE3/xauq+/AeBsmIsSd2JmvgLrFlnRbo0OU2WSH3hgoIC8vLyQmEJMHHiRBwOBwUFBcc9zjRN/H4/LperUXt8fLzeVSgikefzUvvBs5j1XqLG3UTMFfdhyz5bS9l1ABELzMLCQnr16tWozeFwkJWVxdatW497nN1uZ+rUqfzlL39h9erVHDx4kM8++4wFCxbwwx/+sLXLFhFpwl9aRM27fyBQW4VhdxJ96Z3EXv0b7L3yMIyI/ZqVFhbRe5jx8fFN2uPj46msrDzhsffffz/33HMPV199dahtxowZ3HzzzS1ep4jI8fgrduNd8Tq+bSswnHEEyndh6dYPq5ay65Da5WMljz76KMuXL+fBBx8kJyeHVatW8ac//YmUlBSuv/76Zp0rOTmuRWpKTXWdfCeRNkzXcPh8njLKl7/CobUfYNgdJF54NYnnX4bFGRPp0jq91ryOIxaY8fHxeDyeJu0ej4eePXse97hNmzbx3HPPMX/+fC6++GIAzj33XHw+H0888QTf//73iYsLPwTLyqoJBE7v3qfehyntna7h8JimiWEY+Pfv4dD6j7APmoBj6KX4o1yUefyAfoaR1BLXscViHLcjFbHB9dzcXAoLCxu1eb1eduzYccLA3LJlCwD9+/dv1D5gwAC8Xi979+5t+WJFpFMz6w5S98Vr1Cx6GNMMYE3JJu5HjxN1wfexRKln3llELDBHjx7NZ599RkVFRahtyZIleL1exowZc9zjMjIyAFi/fn2j9nXr1mEYBt26adUMEWkZZn0ddasWUf232/GuWoQR44b6OgAtjt4JRWxIdvr06bz00kvk5+eTn59PWVkZDz/8MJMnT240e3bOnDksXLiQDRs2ADBo0CCGDBnCPffcQ1lZGdnZ2axZs4ann36aK6644qSrBImIhKN+8yfUffYqZk0l1qyzcJ57BdbkrEiXJREU0XuYL7zwAg8++CAzZ84MLY13++23N9ovEAjg9/tDn61WK08++STz5s3j6aefZv/+/XTt2pVrr72WG2+88Ux/GyLSgZiBAPi9GPYoTJ8XS2I6jgk3Y0s/9kIq0rkYZid/2l+TfkR0DZumia/oK7xfvo61Wz+iRl2DaQYAQwsOtCOtPemnXT5WIiLSEkzTxL9rfXAZu9JtWBK7Ys0YAKAFB6QJBaaIdEpmwE/Nvx7Dv2s9RlwyUWOuw9Z7BIbFGunSpI1SYIpIp+Iv34UlsSuGxYo1JRtb9tnY+4/FsNojXZq0cQpMEekUAp591K14Hd+Wz4ka+1PsfUbiPP/qkx8o0kCBKSIdWuBgBd6V/0f9NwVgseI4ezI2vWpLToECU0Q6LP/+7Rx680EwA9j7j8Ux7DIsMYmRLkvaKQWmiHQoprcG3+4N2HPOwZLUHcfgS7D3G4slPjXSpUk7p8AUkQ7B9Hmp37AM76pFmHXVWL//CJa4ZJznXRXp0qSDUGCKSLtmBvzUb/oI71dvYh4sx5oxEOe5V2CJS450adLBKDBFpF3zrvw/vCvfxJLWk6iLrsfWrf/JDxI5BQpMEWlXTNPEX7wG01+Pvcdw7AMuwpqSgzX7bC1jJ61KgSki7YZvz7d4v/wH/pJNWNP7YO8xHEtMIpacoZEuTToBBaaItHn+/dup+/I1/MVrMWIScY66Bnu/0ZEuSzoZBaaItGmmaVL74QsEPHtxnn819oEXY9ickS5LOiEFpoi0OYHqMrxfLcQ+4GKsqTlEX3QDRkw8hiMm0qVJJ6bAFJE2I1Djwfv1W9RvWAaApUsvrKk5WBLTI1yZiAJTRNoA01uDd/U/8a59F/z12PuOwjFsqp6llDZFgSkiEWOaZvBRkIAf7/ql2LLOwjn8ciyJXSNdmkgTCkwROeNMv4/6b5ZTv3EZMd/9FUZUHHHTf48RFRfp0kSOS4EpImeMGQjg2/IpdV8txKwqxZreB7O2CsMRrbCUNk+BKSJnhG/Pt9R99BcCFTuxJGcT9Z1ZWLsP1uo80m4oMEWkVZneGgxHNBgGBHxEjc/H1mM4hmGJdGkizaLAFJFW4d9XSN2X/4CAn+hL78KW3gfrVQ9hWBSU0j4pMEWkRfnLd+Jd8Tq+opUYUS4cQy8D0wTDUFhKu6bAFJEWU/vJX6lftwTsUTiGT8MxaEJwOFakA1BgishpCRw6gGG1YzhjsSSk4ThrEo6zJmvWq3Q4CkwROSVmbXVwdZ51S3EMGo/z/KtxDBwf6bJEWo0CU0Saxayvxbv2Xbxr/gXeWmy98rD3HxvpskRanQJTRMJm1h3k4IK7MWs82HKG4Rg+DWtSZqTLEjkjFJinKVBZwo4FdxDAClYbWOwYVhuWhC5EjbkOgNoPX8D012NYbWC1g8WGYbXhOPtSDLsTX/EaAlVlR7Zbg9stKTlYYhIJ1HgwDx0ItlsOb7eDzYFhc0T4JyAdnRnw4ytaGXx20hmLfdAEbBkDsKblRro0kTNKgXm6rHaiMvtRe6gG/D5Mf33wnwF/aBd/6VbMmioIHNmO34djyHcAJ/UbP8BXtLLJqaMm3Iylx3B8Wz6j7tO/NtluH3gxUSN/jL+8mENvPBAK2sOha3VnEn3JTABq3pmHGfBhWBqHsvOCH2DYo6jf8imByr2NQhmrDVvGQCyuFAKeUgKVJUfC+vBfDqJdWKLjg9+vrw4sweP0UHr7Z5oBfNtW4P3ydQKVJURPno0tcxDOoZdFujSRiFBgniZLXDKpU2+ltLTquPvETruvSZtpmqF/jxp7PWZ9bUPQHglUS3waALbsszFi3RDwNQplS3J3AAxnHPaB40LHHT6HEes+8vX89Zh1BzH9PvDXYzacy5n3fQB8hV/g2/510+/vkluxuFLwFX1F3WevNNluH3QJUSN+QKC8mEOv33vUgVaw2rGmZBNz2d0AHHzjvuD30BumZ6YAACAASURBVNALPxzc0RfnY9ideNctIVCxq1EvHKsde+55WBLS8ZfvJLC/qCGUj5zD4krBEp+G6fNiVpc1hL298V8OLNbj/0eURkzTxL9zLXVf/oPA/u1Y3BlEXTITa8bASJcmElEKzAg5ev1MwxF9wmfVLPFpofA85vZYN1F500/49WImzz7h9uiJt2KagYbQrQ8Ga8AXejTA1isPa1puQ9A2bPf7sCQE6zJiEnHmTQ8FcrA37cMSk3CkzoR0qK890gv31UFddXDJNMBfug1/8dqGr93wFwfAmpKNJSEd3441eL9Y0KR2+5DvEJU3nUD5Tg4tvL/pzyclh9hp9wJQ/fJtEPAfFaTBf8Z8924Mm5O6FW8QKNvRqBeOxY59wDisSRn4927Bt+ebRr1ww2rHkpSJNTkL01uDf39R41641QZ2J5aYRCD4lxcstja7hqpv2wpql/4Jw5VK1NjrsfW6QAsOiKDAlKMYhgVsjuC90X/bZolJhIZf+MdiiUlsGGI+vuhxN554+0U3NPpsmmYw3BqGdx0DLsLec/gxQjlYlxGfStS4GxsPjft9WKJdoXPaepwLPm+jnnwwwII9ULOumkDV/ka9cNNfj63HOUAGvt0b8X75jya1O86egjU5i0DFLmoW/bbpzyetJ7Hf+y8Aqp//WfBrN/TCjYZh7Njpv8OwOaj96EX8pUWN72lbbDiGfRdrSja+nevw7Vh9VCg3hHaXXtjSexOorcK/c32jXjhWG4YzFqs7A4BAdRlYrKFeeF1JKd5N63H0H4steyhRo6/F1ntE8HgRARSY0oYZhhEMhMOfT9YTj3Jh6XXBCc8ZNeIHJ94+8scn3O44ewqOwRMb9cLx14MjJliDuxvRl97ZqBeOvx7DGXvkHOd8D3xHAp/D97UbvlfDGYvhjAmeo74Wag8Ps9cDECgrpv7bjxr1wgEcQy8LBmbFbmrff7JJ7dYuvYmZ+isADv7tdjADoW3VgBEdj733BRg2J/Z+o0/4cxDpjAzz6JtpnVBZWTWBwOn9CFJTXSe8hynSWkK98IZesmFzYPrqCFSVNeqF46/HsEdh7dIL0zTxbfqoUS/cleiiruuwRsEu0t60xO9ii8UgOfnYq1SphynSjoV64Uf3xG1OrO5uJzzG3vfCRm0J+kufyEnpTr6IiEgYFJgiIiJhUGCKiIiEQYEpIiISBgWmiIhIGBSYIiIiYVBgioiIhEGBKSIiEgYFpoiISBgUmCIiImFQYIqIiIRBgSkiIhIGBaaIiEgYFJgiIiJhUGCKiIiEQYEpIiISBgWmiIhIGBSYIiIiYVBgioiIhEGBKSIiEgYFpoiISBgUmCIiImFQYIqIiIRBgSkiIhIGBaaIiEgYFJgiIiJhiGhgFhUVcd111zF06FDy8vJ44IEHqKmpCevYqqoqfvOb3zB69GgGDRrEuHHjmDdvXitXLCIinZUtUl/Y4/FwzTXX0K1bN+bNm0d5eTlz586lvLycxx9//ITHHjp0iB/96EcYhsHtt99OWloaxcXFlJSUnKHqRUSks4lYYL7yyit4PB4WLlxIUlISAFarldmzZ5Ofn0/v3r2Pe+zTTz9NVVUVb731FrGxsQCcf/75Z6RuERHpnCI2JFtQUEBeXl4oLAEmTpyIw+GgoKDghMe+9tprXHnllaGwFBERaW0RC8zCwkJ69erVqM3hcJCVlcXWrVuPe9zOnTspLS3F7XZz0003MXjwYIYPH84dd9xBZWVla5ctIiKdVMQC0+PxEB8f36Q9Pj7+hMG3f/9+AH73u98RGxvLU089xZ133klBQQGzZs1qtXpFRKRzi9g9zFMVCAQAyM7O5pFHHsEwDABcLhe33nora9asYciQIWGfLzk5rkXqSk11tch5RCJF17B0BK15HUcsMOPj4/F4PE3aPR4PPXv2PO5xCQkJAFxwwQWhsDz8GWDz5s3NCsyysmoCATPs/Y8lNdVFaWnVaZ1DJJJ0DUtH0BLXscViHLcjFbEh2dzcXAoLCxu1eb1eduzYccLA7N69Ow6H47jb6+rqWqxGERGRwyIWmKNHj+azzz6joqIi1LZkyRK8Xi9jxow57nEOh4ORI0fyySefYJpHeoYff/wxAIMGDWq9okVEpNOKWGBOnz4dl8tFfn4+H374IQsXLuSBBx5g8uTJjWbPzpkzhwEDBjQ69uabb6awsJBZs2bx4Ycf8uqrr3LfffcxatSoZg3HioiIhCui9zBfeOEFHnzwQWbOnInT6WTKlCncfvvtjfYLBAL4/f5GbYMGDeKZZ57h0UcfJT8/n7i4OCZPnszs2bPP5LcgIiKdiGEePa7ZCWnSj4iuYekYOuykHxERkfZEgSkiIhIGBaaIiEgYFJgiIiJhUGCKiIiEQYEpIiISBgWmiIhIGBSYIiIiYVBgioiIhEGBKSIiEgYFpoiISBgUmCIiImFQYIqIiIRBgSkiIhIGBaaIiEgYFJgiIiJhUGCKiIiEodmBWVhYyNKlSxu1ffnll1x33XVcddVVPP/88y1Vm4iISJtha+4BDz/8MIZhMH78eABKSkq44YYbcDqdJCcn89vf/paEhAQuv/zyFi9WREQkUprdw9ywYQPnnntu6PObb76JaZq8+eabvP3224wdO5aXX365RYsUERGJtGYHpsfjISkpKfR5+fLl5OXl0aVLFwDGjh1LUVFRixUoIiLSFjQ7MJOTk9m1axcAlZWVrFmzhhEjRoS2e71eTNNsuQpFRETagGbfwxw5ciQvvfQSLpeLzz//HICLL744tH3z5s107dq15SoUERFpA5odmLNmzWLbtm389re/xW63c/vtt5ORkQFAXV0d77zzDpdddlmLFyoiIhJJhnmK46dVVVU4nU4cDkeorba2lqKiItLT00lMTGyxIltTWVk1gcDpDSGnprooLa1qoYpEzjxdw9IRtMR1bLEYJCfHHXNbs3uYh7lcrkafTdPENE369et3qqcUERFps5o96Wfp0qU89thjjdqeffZZhg4dyrBhw8jPz6empqbFChQREWkLmh2YTz/9NKWlpaHP69at45FHHmHIkCFcffXVFBQU8Mwzz7RokSIiIpHW7CHZ7du3c+mll4Y+L1q0iMTERJ555hkcDgd2u523336bmTNntmihIiIikdTsHmZtbS3R0dGhzx999BEXXnhhaPJPv379KCkpabkKRURE2oBmB2Z6ejpr164FoKioiC1btjBy5MjQ9oqKCpxOZ8tVKCIi0gY0e0h26tSp/OEPf2Dfvn1s2bKFhIQExo0bF9q+du1aevTo0aJFioiIRFqzA/PGG2/E6/WyfPlyunbtysMPPxx6xOTAgQOsWLGCGTNmtHSdIiIiEXXKCxd0FFq4QETXsHQMbXbhAoDy8nJ27twJQGZmZqO3mIiIiHQkpxSYK1as4OGHH2b9+vWN2gcPHsydd97JOeec0yLFiYiItBXNDswVK1bwk5/8hNjYWGbMmEFubi4AhYWFvPnmm8yYMYPnn39eoSkiIh1Ks+9h/vjHP6a0tJS//e1vuN3uRtsOHDjA9OnTSUtL48UXX2zRQluL7mGK6BqWjqG172E2+znMdevWcfXVVzcJS4DExESuuuqq0HOaIiIiHUWzA9NiseD1eo+73ev1YrE0+7QiIiJtWrOTbdiwYbz88ssUFxc32VZcXMzLL7/M8OHDW6Q4ERGRtqLZk35++ctf8oMf/IDJkyczbtw4cnJyANi2bRvLli3D4XAwa9aslq5TREQkopodmP369ePvf/87jz/+OAUFBbzzzjsAREdHc9FFFzFjxozQQuwiIiIdxSk9h5mbm8sf//hHAoEA5eXlACQlJWGxWPif//kfnnjiCTZu3NiihYqIiETSaa30Y7FYSElJaalaRERE2ixNZxUREQmDAlNERCQMCkwREZEwhHUPc82aNWGfsKSk5JSLERERaavCCsyrr74awzDCOqFpmmHvKyIi0l6EFZhz585t7TpERETatLAC8/LLL2/tOkRERNo0TfoREREJgwJTREQkDApMERGRMCgwRUREwqDAFBERCYMCU0REJAwKTBERkTAoMEVERMKgwBQREQmDAlNERCQMCkwREZEwKDBFRETCENHALCoq4rrrrmPo0KHk5eXxwAMPUFNT06xzLFmyhL59+3LppZe2UpUntnZrGe+vKKaiqi4iX19ERM6MsN5W0ho8Hg/XXHMN3bp1Y968eZSXlzN37lzKy8t5/PHHwzpHTU0NDz30ECkpKa1c7fEtXbGTtVvLAOiaHEP/bDf9s5Pol51IbJQ9YnWJiEjLilhgvvLKK3g8HhYuXEhSUhIAVquV2bNnk5+fT+/evU96jvnz55OZmUlGRgbr1q1r7ZKP6darhlDlDfDJqp1sLKrgo7V7eH/lLgwDsru46J/jZkB2Er0zE3DYrRGpUURETl/EArOgoIC8vLxQWAJMnDiROXPmUFBQcNLALCws5C9/+QsLFizgueeea+1yj8tiGPTKTCTBaWXS+dn4/AEKd1WycXsFG7ZX8O4Xxfzrsx3YrAa9MhKCPdCcJHp0dWG16BayiEh7EbHALCws5IorrmjU5nA4yMrKYuvWrSc9/v777+fKK6+kT58+rVXiKbFZLfTNctM3y833LoSaOh+bdx5gQ1EFG7dX8MaH23jjw21EOaz07Z5I/5wkBmS7yUiNxTCMSJcvIiLHEdF7mPHx8U3a4+PjqaysPOGxb7/9Nps2beIPf/jDadeRnBx32ucASE11HXdbVqabi/N6AFBZXceaLftZvbmUNZv3s/q9zQAkxjkZ0iuFIb1TOat3CunJsS1Sl0i4TnQNi7QXrXkdRywwT1V1dTUPP/wws2bNOmbgNldZWTWBgHla50hNdVFaWhX2/v0y4umXEc9/jM1lf2UNGxt6n6s3l1KwahcAKQlRDMgJTiDqn+0mPtZxWjWKnEhzr2GRtqglrmOLxThuRypigRkfH4/H42nS7vF46Nmz53GPe/LJJ0lMTGTChAmh4+vr6wkEAng8HqKionA42k+4pCREc+FZ0Vx4VjdM02T3/oNs2F7BxqIKvvxmHwWr9wCQmRobDM8cN327JxLtbHd/1xERadci9ls3NzeXwsLCRm1er5cdO3Ywbdq04x63detWNm3axPnnn99k27nnnsvdd9/NjBkzWrrcM8IwDDJS48hIjWPC8O74AwGKSqpCPdBlX+9iyYpiLIZBj24u+mcH73/mZiRgt2kCkYhIa4pYYI4ePZr/+Z//oaKiArfbDQQXIfB6vYwZM+a4x/3iF7/gP//zPxu1Pf3002zbto25c+eSnZ3dqnWfSVaLhdxuCeR2S+DSETl46/1sOTwDt6iCtz8tYtEnRThsFnpnJtA/Jzh8m93FhcWiCUQiIi3JME3z9G7gnSKPx8Oll15KRkYG+fn5lJWV8fDDD3PBBRc0Wrhgzpw5LFy4kA0bNhz3XHfddRfr1q1j0aJFza4jEvcwW8qh2nq+3XEgOIS7vYLd+w8CEBtlo2+Wm/7ZbgbkuElPitEMXDkh3cOUjqBD38N84YUXePDBB5k5cyZOp5MpU6Zw++23N9ovEAjg9/sjVGXbFhNlZ2ifVIb2SQXgQHUdGxvuf27cXs7KTaUAuF1O+mW5GyYRuUmKj4pk2SIi7VLEephtRXvuYZ6IaZrsOxCcgbthewXfbK+guqYegC5JMQzIDoZnv2w3cdFawq+za4vXsEhzddgeprQuwzDo4o6hizuGsUMzCJgmO/dVhxZQ+GRdCcu+3oUBZIWW8HPTu3siTi3hJyLShHqYHbSHeTI+f4Ctuz0NQ7jlFO724A+YWC0GuRkJwR5ojpseXeOxWTUDt6Nrj9ewyL9r7R6mArOTBua/q/P62bTzQMMQbjnFe6sxAefhJfwahnAz0+KwaAJRh9MRrmERDcnKGeF0WBncM5nBPZMBqK6p55uGBeQ3FpWzpjD4CjNXjJ1+WcHe54CcJNISoyNZtojIGaPAlGOKi7YzvF8aw/ulAVDuqW24/1nOhu3BVYgguIRfv2x3aBJRQpwzkmWLiLQaBaaEJSk+ilFDujJqSFdM02RP2aGGBRTKWfltKR+tCS7hl5ES2/AKMzd9u7uJidIlJiIdg+5h6h7maQsETLbvrWJDUTkbt1eweWcl9b4AFsMgp6sruIBCtptemQnYbZqB2xZ19mtYOgZN+mllCsyWV+/zs2WXh43by9lYVMG2PVUETBO7zUKvjITQW1hy0rWEX1uha1g6Ak36kXbHbrOGZtUyOvgS7eASfsEe6D+WbwW2Eu200S+rYQZuThLdkrWEn4i0XQpMaXXRThtn907h7N4pAFQe9IZ6nxu3V/D15v0AJMQ5QkE7IDuJ5AQt4ScibYcCU864hFgHeQPSyRuQDtCwhF+w97l+Wzmfrd8LQJo7mgHZwcdXtISfiESaAlMiLi0xmrSzMxhzdnAJv12lB9lYFHx85dMNe/lg1W4MoHuXOAY0vES7T2YiTocmEInImaNJP5r006b5/AGK9lQF738WVVC4uxKfv2EJv27xwWdAc5Lo2U1L+J0OXcPSEWiWbCtTYLYvdfV+NoeW8KtgR0lVcAk/u5Xe3ROCPdBsN927aAm/5tA1LB2BZsmKHMVptzKoRzKDehxZwu/bHYeX8KtgwdYtQHClon5ZifTPSWJAtps0d7Rm4IrIaVFgSrsWF23nnL5pnNM3uIRfRVVdaAGFjdsrWPFt8CXayfHOhiX8gvdAE7WEn4g0kwJTOhS3y8nIwV0ZOTi4hN/eiprQBKJVm/fz8doSALomx4TCs19WIjFRmoErIiemwJQOyzAM0pNiSE+K4aJhmQQCJjv2VYXuf364ZjfvrdyJYUBOuosBOcH7n70yEnDoJdoi8m806UeTfjqtel+ArbsrG97CUsG2PcGXaNusFnpnJoQWkc9Jd2G1dOwZuLqGpSPQpB+RVmK3Weib5aZvlpvLCS7ht6n4QMNbWCp4vWArFEC000rf7u5QgGakxGoCkUgnpMAUaRDttHFWrxTO6hVcws9z0Ms3OypC7wFdtSW4hF987NFL+LlJ0Uu0RToFBabIccTHOjivfxfO698FgP0HatiwvYJvtgfvgX6+IbiEX2piFP2zkxiQ46Zftpv4GEckyxaRVqLAFAlTSmI0oxOjGX1WN0zTZPf+g6HnP7/8Zi8Fq3cDkJka1/AKMzd9uicS7dT/ZiIdgSb9aNKPtAB/IEBRScMM3KJytuzy4PMHsFoMenSNDw7f5rjp2S0Bu63tTSDSNSwdgZbGa2UKTGkN3no/m3dVNrzCrJyikipMExx2C30yE+mfE1xEoa0s4adrWDoCzZIVaYccdisDc5IYmJME5HKotp5vdhxeA7ecvy8rBAqJjbI1rEAUfIl2Fy3hJ9JmKTBFzoCYKDvD+qQyrE8qEFzCLzh5KLiM31cNS/i5Xc6G8HTTPzsJt0tL+Im0FQpMkQhwu5xcMCidCwalY5om+yqCM3A3bq9gdWEZH687soTf4UdY+mW7idUSfiIRo8AUiTDDMOiSFEOXpBguGhp8ifbOfdWhFYg+XlvC+yt3YQBZ6a5QD7R3ZiJOLeEncsZo0o8m/Ugb5/MH2LrbE3wDS1E5hbsPL+FnkNstITSBKKer65Rfoq1rWDoCzZJtZQpMaW9qvT42FVeycXs5G4sq2LGvGoAoh5U+3RMZkO1mQE4SGanhL+Gna1g6As2SFZFGohw2huQmMyQ3+BLtqkPehhm4wdeYrSksAyA+xh6cgdvwFpZULeEncloUmCLtnCvGwbn90ji3X/Al2mWVtaHZtxuLKvhi4z4AUhKiGlYgCgZofKyW8BNpDg3JakhWOjDTNNlddij4CEtROd/sOEBNnQ+AzNTYUHiOHJbJwaraCFcrcnp0D7OVKTClMwkETLbvrWJDUbAHunlnJfW+AIYBae4YstLiyOoSR/c0F1ld4kiIdWghBWk3FJitTIEpnVm9z8+WXR52lR3im6JyduytYn/lkZ5mfIyd7l1cZKXF0T0tju5dXKQnRXf4F2pL+6RJPyLSauw2K/2z3YwenhX6RXOotp7ifdXs2FdN8d5qduyrYsmKYnx+s+EYC5mpsaFeaPe0ODJT4/RWFunwdIWLSCMxUXb6Zrnpm+UOtfn8AUrKDrFjXxU79lZTvK+ar77dF3qlGUCaOzrYE23okWZ1cZEYpyFd6TgUmCJyUjarhcy0ODLT4hgxKNhmmiYVVXUNPdEqduyrZsfealY0rIsLEBdtp3vDfdGsNBfdu8SRnhRzygssiESSAlNETolhGCTFR5EUH8XZvVJC7TV1Por3BXuhO/ZWUbyvmve+2oXPHwCC4ZuRGhu6L5rVxUVmahwxUfp1JG2brlARaVHRTht9uifSp3tiqM0fODyke+S+6Neb9/Phmj2hfVITo0K90Kw0F93T4kiKd2pIV9oMBaaItDqrxUJGahwZqXFcMDDYZpomB6q9FDfcFz08tLtyUymH563HRtlCvdDD/+yarCFdiQwFpohEhGEYuF1O3C4nQ3IbD+nuKj0YDNKG+6LLvt5Fve/wkK5Bt+TYUE/08EzdGL36TFqZAlNE2pRop41emQn0ykwItfkDAfaW17BjX1XDkG41awvL+HhtSWiflISo4LOiDT3RrLQ4khOiNKQrLUaBKSJtntVioVtKLN1SYskbcKS9srquoRda1TDJqJpVm/eHhnSjnbaGR12O9Ea7pcRqSFdOiQJTRNqthDgng+OcDO6ZHGqr8/rZub861BMt3ltFwerdeOuDQ7pWi0HX5NiGR12Cz412T4sjLlpDunJiCkwR6VCcDiu53RLI7XZkSDcQMNlbceiox12qWV9UzifrjgzpJsU7Q7Nzs7oEgzRVQ7pyFAWmiHR4loZeZdfkWM7r3yXUXnkwOEs31BvdV83qwv0cXmE72mmle+qRXmhWlzgyUmKx26wR+k4kkhSYItJpJcQ6SOiRzKAeRw3p1vvZvf8gOxpWLyreW81Ha/ZQV+8HwGIYdE2JaVh44cgsXVeM3i/a0SkwRUSO4rRb6dE1nh5d40NtAdOk9EBNaNGFHXur+WbHAT5dvze0j9vlbLIMYGpiNBYN6XYYCkwRkZOwGAZd3DF0cccwvF9aqN1zyBu8L7q3OvTc6Lqt5QQaxnSdjsNDunGhBekzUmJx2DWk2x4pMEVETlF8jIOBOUkMzEkKtdX7/OzafzD4VpeGHumn60pY5g0O6RoGwVm6oXeMBnuk8bEa0m3rFJgiIi3IbrOSkx5PTnrjId39lbXBt7o0vB5t084DfLbhyJBuQpyj0cpF3dPi6OKOwWLRkG5bocAUEWllFsMgLTGatMRozul7ZEi3uqa+YUj3yDKAG4rK8QeCQ7oOuyU0S/fwAgyZqXE4NaQbEQpMEZEIiYu20z/bTf/sIy/rrvcFgrN091WF7o9+vmEvH3y9CwgO6aYnxTRZBjAhzhmpb6PTUGCKiLQhdpuF7HQX2emuUJtpmpRV1jZaBrBwl4cvNu4L7RMf62hyXzQ9SUO6LUmBKSLSxhmGQUpiNCmJ0QzrkxpqP1hbz86GodzDC9O/+2XxkSFdW/C1akcvA5iZGkuUQ7/6T4V+aiIi7VRslJ2+WW76Zh0Z0vX5g0O6R5YBrGLFN/tYvmo3AAaQ5o4O3RcNTjJykRjn0DKAJ6HAFBHpQGxWS/C+ZpfGQ7rlnrpQL7R4XzXbSzys+ObIkK4rxh68J9qw6EJWWhzpyTFYLXqzy2EKTBGRDs4wDJITokhOiGJo7yNDuodqfewsrW60DODSr4rx+YNDujarhczU2FAv9PBEo2hn54yOzvldi4gIMVE2+nRPpE/3xFCbzx+gpPxQo2UAV27aT8HqPaF90hKjQ73Qw0O7bpezww/pKjBFRCQk2KsMPu95AelAcEi3oqoueE/0qOdGv/q2NHRcbJSNrKPe6pKV5iI9OaZDvaxbgSkiIidkGAZJ8VEkxUdxVq+UUHtN3eEh3eBausX7qln29S7qfcGXddusBt1SYhvdF+2e5iImqn1GT0SrLioq4oEHHmDlypU4nU6mTJnC7NmziY6OPu4x1dXV/PnPf6agoIBt27Zhs9kYOHAgs2bNYuDAgWewehGRzi3aaaN3ZiK9M48M6foDAUrKa0K90OK9Vawu3M9Ha48M6aYkRB3pjTY8N5oc3/Zf1h2xwPR4PFxzzTV069aNefPmUV5ezty5cykvL+fxxx8/7nG7d+/m1Vdf5YorruCWW27B5/Px4osvMn36dF555RWFpohIBFktFjJSYslIiSWv4dexaZocqG54s0vDfdEd+6r5elMpDe/qJsZpC00uOryebreU2DY1pBuxwHzllVfweDwsXLiQpKTgSv9Wq5XZs2eTn59P7969j3lcZmYmS5YsadQLHTFiBBdffDEvvfQSc+fOPSP1i4hIeAzDwO1y4nY5GZJ75GXdtV4fO0sPNlpPd/mqXXgbhnStlsNDuo3X042Nskfk+4hYYBYUFJCXlxcKS4CJEycyZ84cCgoKjhuYMTExTdqcTie5ubns27fvGEeIiEhbFOWw0SsjgV4ZCaG2QMBkb8WhRqsXrdtWzsfrSkL7JMc7j+qJBv+ZkhDV6vVGLDALCwu54oorGrU5HA6ysrLYunVrs8516NAhNm7cyNSpU1uyRBEROcMsFoOuybF0TY7l/AFdQu2V1Udm6R5eT3f1lv2hId1op5UbvjeYs3okHfvELSCi9zDj4+ObtMfHx1NZWdmsc/33f/83NTU1/OhHP2p2HcnJcc0+5lhSU10n30mkDdM1LG1ZaqqLXj1SGrXV1vnYXuJh224P2/d46JIU26rXcfuc23uUt956ixdeeIH/+q//Ijs7u9nHl5VVEwiYJ9/xBFJTXZSWVp3WOUQiSdewtFdJMXaSeiVzTq/kFrmOLRbjuB2piE0/io+Px+PxNGn3eDwkJCQc44imPv74Y+6++26uynavHgAAEQ5JREFUu+46fvjDH7Z0iSIiIiERC8zc3FwKCwsbtXm9Xnbs2EHPnj1PevyaNWu4+eabmTRpErfffntrlSkiIgJEMDBHjx7NZ599RkVFRahtyZIleL1exowZc8JjCwsLuf766xk2bBgPPfRQm3/YVURE2r+IBeb06dNxuVzk5+fz4YcfsnDhQh544AEmT55Mr169QvvNmTOHAQMGhD6XlZVx3XXXYbfb+elPf8r69etZtWoVq1atYsOGDZH4VkREpBOI2KSf+Ph4XnjhBR588EFmzpwZWhrv34dXA4EAfr8/9HnLli3s2RNcYmnGjBmN9s3IyOD9999v9dpFRKTzMUzTPL0pou2cZsmK6BqWjqHDzpIVERFpTxSYIiIiYVBgioiIhEGBKSIiEgYFpoiISBgUmCIiImFQYIqIiIRBgSkiIhIGBaaIiEgYFJgiIiJhUGCKiIiEQYEpIiISBgWmiIhIGBSYIiIiYVBgioiIhEGBKSIiEgYFpoiISBgUmCIiImFQYIqIiIRBgSkiIhIGBaaIiEgYFJgiIiJhUGCKiIiEQYEpIiISBgWmiIhIGBSYIiIiYVBgioiIhEGBKSIiEgYFpoiISBgUmCIi8v/bu/eYJq//D+Bv7VdwIqiAkUtFRW1F2rkZGKhDhQ1R0TlhGsWpOOZlGrtlzmHMYHgbmc4xRYPZ0hFkTiaXkWFEJ4KiTo33eGVMxCKaRemkqKDSPr8/DM/PWsCniFbl/UpI7DnnOefTpvHd59aSBAxMIiIiCRiYREREEjAwiYiIJGBgEhERScDAJCIikoCBSUREJAEDk4iISAIGJhERkQQMTCIiIgkYmERERBIwMImIiCRgYBIREUnAwCQiIpKAgUlERCQBA5OIiEgCBiYREZEEDEwiIiIJGJhEREQSMDCJiIgkYGASERFJwMAkIiKSgIFJREQkAQOTiIhIAgYmERGRBAxMIiIiCRiYREREEjAwiYiIJGBgEhERScDAJCIikoCBSUREJAEDk4iISAIGJhERkQQMTCIiIglsGpjl5eWIiYnBm2++icDAQKxYsQK1tbWSts3NzcXo0aOhVqsRHh6OHTt2PONqiYioLfufrRY2GAyYMWMGPDw8sG7dOuj1eiQmJkKv1yMpKanZbXfu3InY2FjMmTMHw4YNQ0FBAT7//HM4ODhgxIgRz+kZEBFRW2KzwMzIyIDBYEBubi6cnZ0BADKZDF988QXmz5+P/v37N7ntunXrMHr0aCxatAgAEBgYiLKyMiQnJzMwiYjombDZIdni4mIEBgaKYQkAYWFhsLOzQ3FxcZPbVVRUoKysDOHh4Wbt48aNw5kzZ6DX659ZzURE1HbZbA/z0qVLiIyMNGuzs7ODl5cXysrKmtyuoa9v375m7f369RP7Hw3hJ2nfvp3ksc9jHiJb4XuYXgVP+z5ubnubnsN0cnKyaHdyckJ1dXWT2zX0Pb5tly5dzPql6tbNwarxTXFx6dwq8xDZCt/D9Cp4lu9j3lZCREQkgc0C08nJCQaDwaLdYDCIe4uNaeh7fNuGPcvmtiUiImopmwVm3759cenSJbO2+/fvQ6fTwdvbu8ntGvoeP8/ZMFdz2xIREbWUzQJz+PDhOHz4MP777z+xbffu3bh//36zt4b07NkT3t7eFl9UsH37dqjVaqsu+CEiIpLKZoE5ZcoUODo6Yv78+di/fz9yc3OxYsUKjB07VrziFQCWLl2KgQMHmm2r0WiQn5+PpKQkHDlyBN988w0OHjyIhQsXPu+nQUREbYTNrpJ1cnJCWloaVq5ciYULF8Le3h7h4eFYvHix2TiTyQSj0WjWNmbMGNTV1WHTpk3QarXw8vLC2rVr+aUFRET0zLQTBEGwdRFEREQvOt5WQkREJAEDk4iISAKbncN82V25cgVarRanT59GaWkpvL29sX37dluXRSRZfn4+8vLycO7cOVRXV6Nnz56YOnUqpkyZgvbt+VmaXnx//vknUlNTUVZWhrt376JHjx4IDQ3F/Pnz4ejo2OrrMTBbqLS0FPv27cOgQYNgMpnAU8H0sklNTYWHhwe+/PJLuLi44MiRI1i1ahUqKioQGxtr6/KInqi6uhr+/v6YNWsWunTpgpKSEmzYsAElJSX4+eefW309XvTTQiaTSfwUvmTJEpw9e5Z7mPRS0ev1FvctJyYmYuvWrTh27Bjs7OxsVBlRy/3222+Ij49HcXExevTo0apz87hLC/GQFb3sGvuSDx8fH9y7dw+3bt2yQUVET69bt24AgAcPHrT63DwkS0Si48ePo2vXrnBxcbF1KUSSGY1G1NfXo7S0FBs3bkRISAjkcnmrr8PAJCIAwJkzZ5CTk4MFCxZAJpPZuhwiyQICAlBTUwMACAoKwtq1a5/JOgxMIsKNGzeg0WigVqsxe/ZsW5dDZJX09HTU1taitLQUKSkpmDdvHlJTU1v9gx8Dk6iNq6mpwezZs9GxY0ekpKSgQ4cOti6JyCo+Pj4AgMGDB8PX1xeRkZHYvXs3Ro8e3arrMDCJ2rB79+7hk08+QVVVFTIyMsQLJoheVj4+Pmjfvj10Ol2rz83AJGqj6uvr8emnn6KkpATp6enw9PS0dUlET+3kyZMwmUy86OdFUltbi3379gEAKisrcfv2bezcuRMAoFar+Z8PvfCWL1+OoqIiLF68GHV1dTh16pTY169fP3Tu3NmG1RE9WUxMDAIDA9G/f3/Y29vjwoUL0Gq1UCqVePfdd1t9PX5xQQtdvXoV77zzTqN9iYmJiIiIeM4VEVknJCQElZWVjfZt3rwZAQEBz7kiIuv88MMP2LNnD65evQoAkMvlGDVqFGbNmvVMPvAxMImIiCTg19UQERFJwMAkIiKSgIFJREQkAQOTiIhIAgYmERGRBAxMIiIiCRiYRK+oiooKfPzxx/Dz84NSqURBQYGtS2oxpVKJ+Ph4W5dBbRwDk0iinJwcKJVKqFQqXLt2zaJ/7ty5CAkJsUFljYuLi8O5c+eg0WiwevVqqFQqW5dE9FJjYBJZ6cGDB9i0aZOty2iWyWTCsWPHMH78eMyYMQMTJkyAm5ubrcsieqkxMIms5OPjg5ycnEb3Ml8UBoMBDx48gKOjo61LIXplMDCJrDRnzhwAkLSXaTQakZKSgtDQUKhUKowcORKrV69GXV1di9e/ePEiZs+ejcGDB+ONN97A9OnTcezYMbE/OTlZ/B7YDRs2QKlUPvFQsSAISE9Px/jx46FWqzFkyBAsXboUer3ebFxISAhiYmJw6NAhREREQK1WIywsDLm5uRZz6vV6xMXFYdiwYVCr1Rg3bhy2bdvW6NpbtmzBhAkT8PrrryMgIACzZs0ye04NCgoKMG7cOKhUKoSHh6O4uNis/86dO/j2228REhIClUqFwMBATJ8+HUePHm32+RNJwV8rIbKSh4cHIiMjkZ2djXnz5sHDw6PJsfHx8cjKysKoUaMQHR2Ns2fPQqvVorS0FD/++CPatWtn1dqXLl1CVFQUOnXqhJiYGNjb2yMzMxPR0dFITU2Fv78/QkND4erqioSEBISGhiI0NBQODg7Nzvv1118jOzsb77//PqZNm4br16/jl19+wZkzZ5CVlQV7e3tx7NWrV6HRaDBp0iRMmDABeXl5iI2NhZ2dHcaOHQvg4e9szpw5E2VlZYiKioKXlxcKCgoQFxeHW7duiR86gIfnWjMzM/H2229j4sSJAIATJ07g6NGj8PPzE8edOnUKRUVFmDp1KhwcHJCeng6NRoOioiLxdzwTEhKQn5+PadOmoV+/fjAYDDh9+jQuXrwIf39/q15rIgsCEUmSnZ0tKBQK4eTJk8K1a9cEX19fIS4uTuyfM2eOEBwcLD6+cOGCoFAohCVLlpjNs379ekGhUAiFhYVW17BgwQLB19dXuHz5sthWVVUlvPXWW8LEiRPN2hQKhbB+/fonznn8+HFBoVAIv//+u1n70aNHBYVCIWRkZIhtwcHBgkKhEPLy8sS22tpaISwsTBgxYoRgNBoFQRCEtLQ0QaFQCDk5OeK4+vp6YebMmYJKpRL0er0gCIJw+PBhQaFQCAkJCRZ1mUwm8d8KhULw9fUVysvLxbaG1zc9PV1s8/PzE5YtW/bE50zUEjwkS9QC7u7uiIyMRE5OTpM/kdXwe6nR0dFm7dHR0ZDJZNi7d69VaxqNRhw4cADBwcHo3bu32O7s7IyIiAicO3cON2/etGpOAMjPz0enTp0QFBQEvV4v/nl7e8PV1RVHjhwxG+/i4iLuSQJAx44dMWnSJFy/fh0lJSUAHj53Z2dnvPfee+I4mUyGmTNn4v79+zh06BAAYNeuXQCAhQsXWtT1+N53QEAAevXqJT4eMGAAOnfujIqKCrHN0dERp0+fxr///mv160D0JAxMohaaN28egKbPZVZWVqJdu3bo06ePWbujoyO6d+/eZNA2Ra/Xo7a21mI+APD29hbXtFZ5eTnu3r2LoUOHYsiQIWZ/N2/eRFVVldl4Ly8vtG9v/l9HQ4A3rF9ZWYlevXpBJpOZjevbty8AiL9fqNPp4OrqCmdn5yfW6e7ubtHWpUsXGAwG8fHixYvxzz//YOTIkYiIiEBSUhLKysqeODeRFDyHSdRC7u7u+OCDD5CVlSWG58vIZDKha9euSEpKarTfycnpOVfUuMdDuoHwyE/6jhkzBn5+ftizZw8OHjyI9PR0aLVaJCYmYvz48c+rVHpFcQ+T6Ck0BGVKSopFn6enJwRBwOXLl83ab9++jRs3bsDT09OqtZydnfHaa69ZzAdA3Iuydk7g4R6jwWDAoEGDMHToUIu/x7/wQKfTwWQymbWVl5ebre/p6YkrV67AaDQ2WqdcLhfXvnnzpsXVuE+je/fumDJlCpKTk7F3717I5XIkJye32vzUdjEwiZ6Cm5sbJk2ahNzcXIv7MkeMGAEASEtLM2tPS0uD0WhEcHCw2KbT6aDT6ZpdSyaTISgoCEVFRWZjb926hdzcXKhUKri6ulr9HMaOHQuTyYSNGzda9BmNRlRXV5u1VVVVYceOHeLjuro6ZGZmws3NDUqlEgAwcuRI6PV65OXlieNMJhM2b94MOzs7DBkyBAAQFhYGAI0G2qN7jlIYjUbU1NSYtTk5OUEul5sdtiVqKR6SJXpKc+fORVZWFv7++2+zPbwBAwaIh2xv376NgIAAnD9/HtnZ2QgKChIDFfj/C4MKCwubXeuzzz7DgQMHEBUVhaioKNjb22Pbtm2oqanBkiVLWlS/v78/pk2bBq1Wi5KSEgQFBaFDhw7Q6XTYtWsXNBoNIiIixPG9e/fGsmXLcP78ebi5ueGPP/7A5cuX8d1334mHTSdPnoxt27bhq6++woULF9CzZ08UFBTg0KFDWLRokXgbSEBAACIiIvDrr79Cp9Nh+PDhAB7eQqJUKq061H3nzh0MHz4co0aNEi8IOnHiBPbv348PP/ywRa8N0aMYmERPqWEvc8uWLRZ9y5cvh1wuR3Z2NgoLC+Hi4oKPPvoIGo3G6nswgYcXzWzduhVr167FTz/9BEEQoFKpsHLlyqe6zzA+Ph4DBw5ERkYGkpKSIJPJ4OHhgTFjxiAwMNBsrFwuR0JCAtasWYPS0lK4u7tbnCO0t7dHWloavv/+e+Tl5cFgMKBXr15YsWIFJk+ebDbfqlWroFQqkZmZiTVr1sDBwQG+vr5WP5+OHTsiKioKf/31FwoLC1FfXw+5XI7Y2FjMmDGjxa8NUYN2grXHPYiozQoJCUGfPn2g1WptXQrRc8dzmERERBIwMImIiCRgYBIREUnAc5hEREQScA+TiIhIAgYmERGRBAxMIiIiCRiYREREEjAwiYiIJGBgEhERSfB//sgaylX1kcMAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "6eeERmk6qhI3", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 495 + }, + "outputId": "ed51b43c-ea53-4233-9cfd-b01800472370" + }, + "source": [ + "import seaborn as sns\n", + "import matplotlib.pyplot as plt\n", + "epoch=[]\n", + "for i in range(3):\n", + " epoch.append(i+1)\n", + "sns.set(font_scale=1.4)\n", + "\n", + "plt.figure(figsize=(7,7))\n", + "dic={'Train_Acc':train_acc,'Val_Acc': val_acc}\n", + "x=pd.DataFrame(dic)\n", + "ax = sns.lineplot(data=x)\n", + "ax.set(ylim=(0,1),xlabel='No. of epochs',ylabel='Acc',title='XL-NET')\n", + "ax.set_xticks(range(3)) # <--- s\n", + "ax.set_xticklabels(['1','2','3'])" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[Text(0, 0, '1'), Text(0, 0, '2'), Text(0, 0, '3')]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 68 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcwAAAHNCAYAAACTh0sjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeVxU5eIG8OfMyjpsYooIigqpkGkqaO6K5JZF5s8y9yyjtDKX8ma5Fd3KvKaldVOvqTczM1Jzg1xQy657uaQGEu4LKMMyMMyc8/tjYGQcloMsg/B8Px8+eN73LO8MRx7e97znjCBJkgQiIiIqlcLRDSAiIrofMDCJiIhkYGASERHJwMAkIiKSgYFJREQkAwOTiIhIBgYmERGRDAxMohrozTffRGhoKJKSkuzq1q1bh5CQEPzwww8AgF69emHcuHH3dJwRI0YgJCSk2O3T09MREhKCRYsWWct+++03hISElPj1zTffYMOGDaWuU/SL6H6icnQDiMjetGnTsGvXLsycORNr1qyBIAgAgBs3buCjjz5CREQEnnzyyUo73r59+3Ds2DE8/PDDstYfPnw42rRpY1fepk0bKJVKfPjhhzbl06ZNQ0REBKKjoyulvUSOwMAkqoG8vb0xffp0vPXWW/juu+8wdOhQAMC8efOQm5uL2bNnV9qxGjRogLy8PCxatAjLli2Ttc0jjzyCAQMGlFjfuHFjm+Vp06YhMDAQgwcPrlBbiRyJQ7JENVR0dDTCw8Px8ccf4+bNm9i1axe2bduGCRMmoEmTJpV2HGdnZ4wdOxb79u3D0aNHK22/RLUNA5OoBps9ezYMBgNmzZqF2bNno1mzZhg/fnylH2f48OHw8vLC4sWLZa2fnZ2N9PR0uy+z2VzpbSOqKRiYRDVY06ZN8eKLLyI+Ph5Xr17FnDlzoNFoKv04rq6u5eplzpw5E506dbL7SklJqfS2EdUUvIZJVMN5eXlZv7du3brKjvPcc89h+fLlWLRoEZYvX17quhMmTEB4eLhduZ+fX1U1j8jhGJhENdi1a9fwySefoEWLFvjrr7+waNEiTJs2rVz7yMzMRG5urnVZrVbD09PTbj0XFxeMGzcOH3/8MY4cOVLqddLg4GB07ty5XO0gut9xSJaoBnvvvfdgMpnw+eefIzo6GitXrsSZM2fKvY8uXbpYvyZOnFjiusOHD4e3t7fsa5lEdQl7mEQ11M6dO7F9+3ZMnjwZAQEBmDp1Knbu3Il3330X33zzjfXezLI8//zzePzxx63LOp2uxHVdXFwwduxYay+TiO5gD5OoBsrOzsacOXMQHByMsWPHArBcw5w2bRqOHj2KdevWyd5X8+bN0blzZ+tXaGhoqesX9jI/++yzCr0GotqGPUyiGmjhwoW4evUqPvnkE6jVamt5dHQ0NmzYgPnz56NPnz7w8fEBAFy8eBGff/653X5atGiByMjIch278FrmRx99VOI6hw8fhslksitv3rx5lU5MInIkBiZRDXPixAmsXr0aQ4cORbt27ezqZ8+ejcGDByM2NhYff/wxACAlJQULFy60W7d///7lDkzA0stcvnw50tLSiq1fs2YN1qxZY1c+duxYBibVWoIkSZKjDv73339j2bJlOH78OM6dO4egoCBs3rxZ1rZxcXFYunQpLl26hICAALz88svo379/FbeYiIjqKof2MM+dO4c9e/agTZs2EEURcrN727ZtmD59Ol544QU8+uijSEhIwOTJk+Hq6oru3btXcauJiKgucmgPUxRFKBSWeUdvvvkmTpw4IauH2a9fPwQHB9sMQY0dOxZ6vR7r16+vsvYSEVHd5dBZsoVhWR4XLlxAcnKy3SclDBw4EH/88QfS09Mrq3lERERW991tJcnJyQCAZs2a2ZQ3b97cpp6IiKgy3XeBmZGRAcD+5msPDw+beiIiosp03wUmERGRI9x392EW9iT1ej18fX2t5YU9y8J6uW7dyoYoVmzek4+PG9LSsiq0DyJH4jlMtUFlnMcKhQAvL9di6+67wAwKCgJguVZZ9DpmUlKSTb1coihVODAL90N0P+M5TLVBVZ7H992QbOPGjREUFIQtW7bYlG/evBlhYWHw9vZ2UMuIiKg2c2gP02AwYM+ePQCAS5cuISsrC9u2bQMAhIWFoVGjRpgxYwbi4uJw6tQp63aTJk3C66+/joCAAHTu3Bk///wz9u/fjy+++MIhr4OIiGo/hwZmWloaXn31VZuywuXY2FhER0dDFEWYzWabdfr164fc3FwsXboUy5YtQ0BAAObPn8+n/BARUZVx6JN+aoK0tKwKj3n7+rrjxo3MSmoRUfXjOUy1QWWcxwqFAB8ft+LrKrRnIiKiOoKBSUREJAMDk4iISAYGJhERkQwMTCIiIhkYmERERDIwMImIiGRgYBIREcnAwCQiIpKBgUlERCQDA5OIiEgGBiYREZEMDEwiIiIZGJhEREQyMDCJiIhkYGASERHJwMAkIiKSQeXoBhAREZWXJEkwmUUY8swwGE3IM5rh6eVapcdkYBIRUbURRQm5RhMMeWbLd6MZuXmW74Y8E3Kty3fWyS2ou3vZLEo2+36yR3MMigiosrYzMImIqFSSJMFoEpFbEGjWMCsIttwiYVdcsBmMBct5ZuTlm2UdU6tWwkmjhJNWBWeNEs5aFXw91XDSqOCstSw7aZTWZSeNCl0faYzszNwqex8YmEREtZTJLFp7bDZBZ9NruyvwCoLN8r2wzgxRkso8nlIhwElTGGaWIHN30aC+151gc9ao7gRhQRg6FYSfc8E6Wo0SSkX5p9i4OKkZmEREdYUkScjLN5c5HFkYbDbDm4VBVxCSRpMo65hajdLai3MqCDSdq4s1zAp7cNblgnWctSo4ae8EnUqpgCAIVfwOOQ4Dk4ioElgmoBQzPGkNsLuXba/fFQ0+GZ05KBWCpYdWJMw83DRooC0IOs2dMLP/fmeYU6tWQqGovSFXmRiYRFRniZKEPKPZrtdmOzxZGGgF4VYk6IoOc5rMZffmBABOBQFX2ENz1ijh6a69a6jyTsAVhqGlN3dn6FKt4l2B1Y2BSUT3FevtBNYAu2tYssgsyzvX4oq/fpdrlDcBRa1SWIOqsJfmrXOymZRi/V44CaVg6NK5SEBqNUooavGQZW3HwCSiamG5naAw3O7qpeXdFXR3DWve/f3u2wmKIwiwnWiiVcLVSQUfndY+zIpch7NsU+QancZybY6IgUlEJZIkCfkm0SbMrL00a6+tyDW6ostFJ6GU43YCjUph12vz0TnZ9NqKhpndTMuCdTTq2j0BhaofA5OoFjOZReTkmpCTZ0J2bj4MuSZkFyzn5OZb68wScDszt8h1ujuBJ+d2AoUgFOmdWb67OqtRz8PZZkjSWWvfe7MOcxYs38vtBETVgYFJVIOJkoTcPBNybIKuIOzyLGWGXBNy8vKt9ZZQtNQb80ufiKJUCHB1UsHdVQO1UgFnrQruns7WXppl0smdXpv9pBRLmUbF3hzVfgxMoipU+ISUnLt7dUWWiwvCwnUMeSaU1r8TADhrVXBxKvjSqtDA2wXOTiq4Fiy7OKmtdZb11HDRWurVBUHn6+uOGzcyq+ttIbovMTCJylB4f13RICtriNPS87OEn8lc+pCmRq2Aa0GIOTup4OWmRaN6rnDRqu8EoZPKulw0CJ20nHVJVF0YmFTrSZJldmZOwVClwW6IM79Ij892uDMnz/IpCKUpvIHc1elOD85H52TT63O16eXZ9vg4A5Po/sDApPtCvsls03srbvjSeh2vmCAsa96Ks1Zp04Pz9XRGoJPKpufnWqSXVzQIORuTqG5gYFK1EEXJEmLFXse7M0nlzhCn7Tr5ZTwTU61S2PTaPFw1aOjtYjec6WLtCaqtIeisUfHRYET3OdGgB+BepcdgYJIshQ+ELm/QFfYCDXmlD2sqBMEaaIVB5uWmtRm+dHWy1Llo1TbDny5aJdQqZTW9E0RUUZIkAvl5gCBAUDtBMubAfP08pHwDYDRAys+FZDRAcHKHplVPy2WVHZ9CMhog5RsgGXOBgu9uY5ZAUCiR98saIGBalbabgVmHyL0nz+46XsFszbKeruKkUdpcp/PROaFxfbcSruPdma3p4mS5NYHDmkQ1lyRJgDkfUn6uNdQU3o0hKBQwXToFMePqnSArCDxNaCSU9QKRf3YfjMe2FISdwRKWkKB+qB+cIv4P4u2rMGz5yO6YivpB0LTqCUEQIOZmQlAoIbh4QuHhDEHjBKidAUkEoIS6Zc8qfw8YmPeRqr4nT6UUbG45cHVWw9fT+U7QlTBpxdVJDWctbzgnqokkUbQJMRT00gQXTyh9AiAa9Mg/tbMgyCy9Nik/F4KzB5x7jAMAZH0zFVJWOiDZjhS5jVwMOLkh/+TPMKUcthQKAqB2hqBxhtQs3FKkdYXCuxEEtZO1TlA7QVE/CACg8PKD86C3Csot9VA7QVDeiSjXwW+X+jpVfg9W1ltW8jGq/AhkVR335Lk4qQpmbFpCzcPmOt6dsCucwFL0fj01bz4nqhEkSbL+XxQzrkHKy7YEXr4BMFqCTx3SFYJaC+Ofe2C+/CeQn2sNRSk/F04R/wdVYFvk/7kbefu+tjuG+sEeUHYbDeTlwHg4DlBpLYGmKQg0reuddQuCD2onCBonCGpnQOMMqDQAAG2XEdB2GWEpV2nsfo+oAttCFdi2xNcrqJ2gahhSwXet6jEwK+jGbQPOXNbjyvXMSrknT6tW2vTeLPfkuZVxHc+yzHvyiBxLMudbrr2ptBDUWojZtyDe/PtO765gSFLh1Qjq5hEQc24jd9e/LcOc1t6d5dqd27APAQA5P30IKSvN7liqgDYQ1L4Q0y/CfO0va5AJzjooPB4ANC4AAGWDFtB2eqYg5JysPTjB1QsAIHjUh9vzyyAoSp4HoO04pNTXrXDxvNe37L7CwKygtT/sxdC89XCDAmZJATOUEAUlbgme2Oo0EC5OKjwpbYfGyQxBqYJCpYZCqYJSrYa+1ZNwdnWH7voRaHKuQ63RQKFSQVCoAIUKqoCHoNDVh5hxFea0C5bhCaUaUCghKFQQXL2gcPeCZDZByr4FmFUQlYXbKwGlCoLAYVKi0kiiCJgKembGXEDMh7JeEwBAfvJBSNm37lx7Kwg0bcQwKFy9kHc4Dvln91t6d0YDIJoAANquo6Fp2QPmK38id+cXdx1RgKpFZ6ibRwAKpSUg1QUBprZcmxNcPKxrO3UZYfmH2jKMaR2u1LpZ6jsPBzoPL/H1Kb0bQ+nduMR6QVBYhqeoTAzMChr9eFsIp6/DbDRCBTMUMANmExq5eqF95/YAgJytP0PKzQJEA2DOgpSfD+SZ4R/gCUHrCsOpUzCdPwSTZHuNUXCdCIWuPkypvyPv1//aHVvdug+cHn0O4u0ryPl+pl29wssfrk/PA2C5BgFTHqBQWYK04LvLoDchaFyQdzgO5uvJlr8ylZbAhkIFTWhvKOs1genqOZhTj1vqlHdCXVEvEKoGLSDlZcN0+bR1v4WhDo0LlN6NAABi5s07Qc5QpwqSTEZIxhxriBX24pSeflB4NoD51mWYzv1inVlZGGrK+kHQdnwaUm4Wsv77huX/RVFaV7iP+gwAYDzyI8T0i5ZypcYy0UTjDOTnAgAEVy8oH2huvSaHgu/KB5pbNvEPhcsT79j07CxDlpZzXuHkDtcn3in1daoCHq7Ed40qgoFZQe4+vvB98qVSn8Pp0u+NUvfh3CcGQMFfuqIJEM2QzPmW/4AA1C06Q+nX0lJnNkEq+C64egMAFK5ecOo+DpJoCWuIJkhmk801CFXT9pYhH7MZEPML1jNbQgsATEZIuZkF+7YcH6IZUrOOAADxxnkYj28pmJF2hzosCqoGLSBm3kBu/GK716bwCYDrU3MAANnrZwL5Brt13EZ9ZvnDYecXMF86eVeoK6Ht9CxUfi2Rn3wQ+WcSi4SyCoJSBaV/GNTNOkLMvoX8UzsLtldaQ11w8YA6qIPlZaYctfw1XaSnDqUaCp8ACAqF5V4u0cxQryKSJEIQFJAkEWL6pTu3ERRcd4M5H5rQSABA3uEfId66eKeuIBido16F0icAeQe/R/4f2+2OoQ0fCo1nf0iZ1y3nbOE1uYIeGgp/lmonqFv1tPTaig5Xal2s+3LuPwWCUm3p0RUzZKl5sDvwYPcSX6/CyR1wqtp7A6n6MDBrEEGhABSWi+gCnO+UO7lB6eRW8nZOblCHdC11304R/1dqvTZ8KLSl1GvC+kIT1rdIqBeEcsEsNoWnH1yemmsX6oWTAgDL0JJkMhaEdX5BeJsAleXIygYtIKi1d8oLQlsoqIc5H1Jull2oC271AABSzm0Yj/1kF+oK36bWwDTsXGrfowDgNuYLQKFF7p5llp70XZyjXoMq8GEYTybAeGSjXU9dHdwFmrC+EG9fRe6v/7XrqSs86kPbdhAAIO9wnGWnRXrqUKqgDukCQaGC+dpflhGJoj11pQoKjwaWmYeFAVJNoV54z5zlvjg3CCoNxNtXYU6/YDvRxGiAyr81VP6hMN9IQd6Bb6zX5ApvQ1A2CIbLgKmAKBY7KgJBgLp1H8ttBGmpEG9fvjOr0tnDEmoF55SqaXsoPB4oMkxZsJ6b5Q9JZeOHLNfmSriuLyhVcIoYVuprryvX5kgeBiaVy51Q19hc9hBUGih9Sr5OAlh6yqXRtOpV5val7UPp2xTu45fbhXpRLk+8bQnsoqEumiw9TgCa0EiIgW1teuoQTVB4NgAAKDwaQNWknV1PXdBY/sCRRFPxPXVDhrUNxuNbAJOx+PdHYQlU88UTdvXO/SZD1fghGE/thPF/39m/f236Qxs+FOYbKTBs+8Sup67wCYRzj+cBAIbtCwveNEtgX3fWIDczC06Rr0AQBEtv//LpgskouXfaMHC6pbef9BuMh3+46weghqB1gco/1HJrASxDlgpNQ+twpMKzoaVcqYJT5MSCsLO91cB6rL4Ti/kp36Fq0AJo0KLEeo4KUGUTJEnGp8PWYmlpWRDLuCG/LPxoJCovu1AXzRBcPC09K/11SHnZdqGuqN8MCmcdzGmpMF9PtuupKx9oDpV/KES9ZSjSrqeuqw+nTs8AALLj5gJmS29fMpugVACiQguXJ9+BoNIg79hPkDKuWa/JFYaaKqANFK5eELNvQcrNsp2EouTf3+RYlfG7WKEQ4ONT/IgeA5OBScRzmGqFqg5MjlkQERHJwMAkIiKSgYFJREQkAwOTiIhIBgYmERGRDAxMIiIiGRiYREREMjAwiYiIZGBgEhERycDAJCIikoGBSUREJAMDk4iISAYGJhERkQwMTCIiIhkYmERERDIwMImIiGRgYBIREcng0MBMSUnBuHHj0LZtW0RERGDu3LkwGAxlbpeTk4OPP/4Yffr0QZs2bdC3b18sXrwYRqOxGlpNRER1kcpRB9br9Rg5ciT8/PywcOFCpKenIzY2Funp6ViwYEGp286aNQsJCQl4/fXX0aJFC/z+++/49NNPodfrMWPGjGp6BUREVJc4LDDXrl0LvV6PuLg4eHt7AwCUSiWmTJmCmJgYtGjRotjtTCYTtm3bhueffx4jRowAAERERODy5cvYvHkzA5OIiKqEw4ZkExMTERERYQ1LAIiKioJGo0FiYmKJ20mSBLPZDHd3d5tynU4HSZKqrL1ERFS3OSwwk5KS0Lx5c5syjUaDgIAAJCcnl7idWq3G4MGDsWrVKhw/fhzZ2dk4cOAA1q1bh+HDh1d1s4mIqI5y6DVMnU5nV67T6ZCRkVHqtnPmzMG7776LoUOHWstGjx6NV155pdLbSUREBDgwMCti/vz52LNnD+bNm4cmTZrg2LFj+Oyzz1CvXj2MHz++XPvy8XGrlDb5+rqXvRJRDcZzmGqDqjyPHRaYOp0Oer3erlyv1yMoKKjE7c6ePYvly5fj888/R+/evQEAHTp0gMlkwqeffopnnnkGbm7yQzAtLQuiWLFrn76+7rhxI7NC+yByJJ7DVBtUxnmsUAgldqQcdg2zWbNmSEpKsikzGo1ITU0tNTD/+usvAEDLli1tylu1agWj0Yhr165VfmOJiKjOc1hgduvWDQcOHMCtW7esZfHx8TAajejevXuJ2zVq1AgAcPLkSZvyEydOQBAE+Pn5VU2DiYioTnPYkOywYcOwevVqxMTEICYmBmlpafjggw/Qv39/m9mzM2bMQFxcHE6dOgUACA0NxUMPPYR3330XaWlpCAwMxO+//44vv/wSTz31FJydnR31koiIqBZz6DXMlStXYt68eZg4cSK0Wi0GDBiAqVOn2qwniiLMZrN1WalUYunSpVi4cCG+/PJL3Lx5Ew0bNsTYsWPx4osvVvfLICKiOkKQ6vjd/pz0Q8RzmGqHWjvph4iI6H7CwCQiIpKBgUlERCQDA5OIiEgGBiYREZEMDEwiIiIZGJhEREQyMDCJiIhkYGASERHJwMAkIiKSgYFJREQkAwOTiIhIBgYmERGRDAxMIiIiGRiYREREMjAwiYiIZGBgEhERycDAJCIikoGBSUREJAMDk4iISAYGJhERkQwMTCIiIhkYmERERDIwMImIiGRgYBIREcnAwCQiIpKBgUlERCQDA5OIiEgGBiYREZEMDEwiIiIZGJhEREQyMDCJiIhkYGASERHJwMAkIiKSgYFJREQkAwOTiIhIBgYmERGRDAxMIiIiGRiYREREMjAwiYiIZGBgEhERycDAJCIikoGBSUREJAMDk4iISAYGJhERkQwMTCIiIhkYmERERDIwMImIiGRgYBIREcnAwCQiIpKBgUlERCQDA5OIiEgGBiYREZEMDEwiIiIZGJhEREQyMDCJiIhkYGASERHJwMAkIiKSgYFJREQkAwOTiIhIBgYmERGRDAxMIiIiGRiYREREMjg0MFNSUjBu3Di0bdsWERERmDt3LgwGg6xtMzMz8d5776Fbt24IDQ1Fr169sHDhwipuMRER1VUqRx1Yr9dj5MiR8PPzw8KFC5Geno7Y2Fikp6djwYIFpW6bk5OD5557DoIgYOrUqahfvz4uXLiAq1evVlPriYiornFYYK5duxZ6vR5xcXHw9vYGACiVSkyZMgUxMTFo0aJFidt++eWXyMzMxKZNm+Dq6goACA8Pr5Z2ExFR3eSwIdnExERERERYwxIAoqKioNFokJiYWOq269evx5AhQ6xhSUREVNUcFphJSUlo3ry5TZlGo0FAQACSk5NL3O7ixYu4ceMGvLy8MGHCBISFhaF9+/aYNm0aMjIyqrrZRERURzksMPV6PXQ6nV25TqcrNfhu3rwJAPjwww/h6uqKL774AtOnT0diYiImT55cZe0lIqK6zWHXMO+VKIoAgMDAQHz88ccQBAEA4O7ujldffRW///47HnroIdn78/Fxq5R2+fq6V8p+iByF5zDVBlV5HjssMHU6HfR6vV25Xq9HUFBQidt5eHgAADp16mQNy8JlADh37ly5AjMtLQuiKMlevzi+vu64cSOzQvsgciSew1QbVMZ5rFAIJXakHDYk26xZMyQlJdmUGY1GpKamlhqYjRs3hkajKbE+Ly+v0tpIRERUyGGB2a1bNxw4cAC3bt2ylsXHx8NoNKJ79+4lbqfRaPDoo4/il19+gSTd6Rnu378fABAaGlp1jSYiojrLYYE5bNgwuLu7IyYmBnv37kVcXBzmzp2L/v3728yenTFjBlq1amWz7SuvvIKkpCRMnjwZe/fuxbfffovZs2ejS5cu5RqOJSIiksuh1zBXrlyJefPmYeLEidBqtRgwYACmTp1qs54oijCbzTZloaGh+OqrrzB//nzExMTAzc0N/fv3x5QpU6rzJRARUR0iSEXHNesgTvoh4jlMtUOtnfRDRER0P2FgEhERycDAJCIikoGBSUREJAMDk4iISAYGJhERkQwMTCIiIhkYmERERDIwMImIiGRgYBIREcnAwCQiIpKBgUlERCQDA5OIiEgGBiYREZEMDEwiIiIZGJhEREQyMDCJiIhkYGASERHJwMAkIiKSQXZgHj58GF988UWJ9V9++SWOHj1aKY0iIiKqaWQH5meffYbTp0+XWP/nn39i8eLFldIoIiKimkZ2YJ46dQpt27Ytsf7hhx/GqVOnKqVRRERENY3swDQYDBAEodR1srOzK9wgIiKimkh2YDZt2hT79u0rsT4xMRGBgYGV0igiIqKaRnZgPv3000hMTMS8efNw+/Zta/mtW7cwd+5c7N+/H0OGDKmSRhIRETmaSu6Kw4cPx+nTp7F69WqsWbMGPj4+AIC0tDRIkoQnn3wSo0aNqrKGEhEROZLswASAefPmYdCgQdi+fTsuXLgAAAgICEBUVBQ6duxYJQ0kIiKqCcoVmAAQHh6O8PDwqmgLERFRjSX7GmZSUhJ+/PHHEus3btyIpKSkSmkUERFRTSM7MOfPn4+ffvqpxPotW7ZgwYIFldIoIiKimkZ2YB4/frzUodjw8HAcO3asUhpFRERU08gOTL1eD2dn5xLrNRoNMjIyKqVRRERENY3swPT398ehQ4dKrD906BD8/PwqpVFEREQ1jezAHDRoELZu3YoVK1bAZDJZy00mE5YvX45t27Zh4MCBVdJIIiIiRxMkSZLkrJifn48JEyZg//798PDwQNOmTQEA58+fR0ZGBjp16oQvvvgCGo2mShtc2dLSsiCKst6CEvn6uuPGjcxKahFR9eM5TLVBZZzHCoUAHx+3Yutk34epVqvx1Vdf4YcffsCOHTuQmpoKAGjbti2ioqLwxBNPIDU1lc+TJSKiWkl2D7Mk6enp2LJlCzZu3Ig//vij1M/MrInYwyTiOUy1Q43pYRaVm5uLhIQEbNy4Eb/88gtMJhMCAwMxZsyYCjWUiIioppIdmJIkYf/+/di4cSMSEhKQk5MDQRAwZMgQjBkzBkFBQVXZTiIiIocqMzBPnDiBjRs3YsuWLbh586a1JxkWFoYJEyaga9euDEsiIqr1Sg3Mfv36ISUlBQ888AAGDRqEgQMHonXr1gBgnfRDRERUF5QamOfPn4e/vz/eeOMN9O7d+767ZYSIiKiylPrggnnz5qFRo0Z444030KlTJ0ybNg179uyB2WyurvYRERHVCKX2MIcMGYIhQ4bg2rVr2LhxIzZt2lk8t/UAACAASURBVISNGzfC09MTHTt2hCAIEAShutpKRETkMOW+D/PPP/+0TgK6evUqvL290b17d/Tu3RudO3eGi4tLVbW1SvA+TCKew1Q7VPV9mPf84AJJkvDbb7/hxx9/RHx8PLKysqDVanH8+PEKNba6MTCJeA5T7VAjH1wAAIIgICIiAhEREZg9ezYSEhKwadOme24kERFRTVbhR+Pd79jDJOI5TLVDVfcwZX+8FxERUV3GwCQiIpKBgUlERCQDA5OIiEgGBiYREZEMDEwiIiIZGJhEREQyMDCJiIhkYGASERHJwMAkIiKSgYFJREQkAwOTiIhIBgYmERGRDAxMIiIiGRiYREREMjAwiYiIZHBoYKakpGDcuHFo27YtIiIiMHfuXBgMhnLtIz4+HiEhIRg4cGAVtZKIiAhQOerAer0eI0eOhJ+fHxYuXIj09HTExsYiPT0dCxYskLUPg8GA999/H/Xq1avi1hIRUV3nsMBcu3Yt9Ho94uLi4O3tDQBQKpWYMmUKYmJi0KJFizL38fnnn8Pf3x+NGjXCiRMnqrrJRERUhzlsSDYxMRERERHWsASAqKgoaDQaJCYmlrl9UlISVq1ahZkzZ1ZlM4mIiAA4MDCTkpLQvHlzmzKNRoOAgAAkJyeXuf2cOXMwZMgQBAcHV1UTiYiIrBx6DVOn09mV63Q6ZGRklLrtTz/9hLNnz2LRokUVboePj1uF9wEAvr7ulbIfIkfhOUy1QVWexw4LzHuVlZWFDz74AJMnTy42cMsrLS0LoihVaB++vu64cSOzwm0hchSew1QbVMZ5rFAIJXakHDYkq9PpoNfr7cr1ej08PDxK3G7p0qXw9PREZGQk9Ho99Ho98vPzIYoi9Ho9jEZjVTabiIjqKIf1MJs1a4akpCSbMqPRiNTUVERHR5e4XXJyMs6ePYvw8HC7ug4dOuCtt97C6NGjK7u5RERUxzksMLt164YlS5bg1q1b8PLyAmB5CIHRaET37t1L3O61117DqFGjbMq+/PJLnD9/HrGxsQgMDKzSdhMRUd3ksMAcNmwYVq9ejZiYGMTExCAtLQ0ffPAB+vfvbzN7dsaMGYiLi8OpU6cAoNhZsT/88AOuXbtWbK+TiIioMjgsMHU6HVauXIl58+Zh4sSJ0Gq1GDBgAKZOnWqzniiKMJvNDmolERGRhSBJUsWmiN7nOEuWiOcw1Q61dpYsERHR/YSBSUREJAMDk4iISAYGJhERkQwMTCIiIhkYmERERDIwMImIiGRgYBIREcnAwCQiIpKBgUlERCQDA5OIiEgGBiYREZEMDEwiIiIZGJhEREQyMDCJiIhkYGASERHJwMAkIiKSgYFJREQkAwOTiIhIBgYmERGRDAxMIiIiGRiYREREMjAwiYiIZGBgEhERycDAJCIikoGBSUREJAMDk4iISAYGJhERkQwMTCIiIhkYmERERDIwMImIiGRgYBIREcnAwCQiIpKBgUlERCQDA5OIiEgGBiYREZEMDEwiIiIZGJhEREQyMDCJiIhkYGASERHJwMAkIiKSgYFJREQkAwOTiIhIBgYmERGRDAxMIiIiGRiYREREMjAwiYiIZGBgEhERycDAJCIikoGBSUREJAMDk4iISAYGJhERkQwMTCIiIhkYmERERDIwMImIiGRgYBIREcnAwCQiIpKBgUlERCQDA5OIiEgGBiYREZEMDEwiIiIZGJhEREQyMDCJiIhkUDny4CkpKZg7dy6OHDkCrVaLAQMGYMqUKXB2di5xm6ysLKxYsQKJiYk4f/48VCoVWrdujcmTJ6N169bV2HoiIqpLHNbD1Ov1GDlyJLKzs7Fw4UK8+eab2Lx5M2bMmFHqdpcvX8a3336Lzp07Y8GCBYiNjYUoihg2bBhOnjxZTa0nIqK6xmE9zLVr10Kv1yMuLg7e3t4AAKVSiSlTpiAmJgYtWrQodjt/f3/Ex8fb9EI7d+6M3r17Y/Xq1YiNja2W9hMRUd3isB5mYmIiIiIirGEJAFFRUdBoNEhMTCxxOxcXF7shW61Wi2bNmuH69etV1l4iIqrbHBaYSUlJaN68uU2ZRqNBQEAAkpOTy7WvnJwcnD59GkFBQZXZRCIiIiuHDcnq9XrodDq7cp1Oh4yMjHLt61//+hcMBgOee+65crfDx8et3NsUx9fXvVL2Q+QoPIepNqjK89ihs2Qrw6ZNm7By5Uq88847CAwMLPf2aWlZEEWpQm3w9XXHjRuZFdoHkSPxHKbaoDLOY4VCKLEj5bAhWZ1OB71eb1eu1+vh4eEhax/79+/HW2+9hXHjxmH48OGV3UQiIiIrhwVms2bNkJSUZFNmNBqRmpoq61rk77//jldeeQX9+vXD1KlTq6qZREREABwYmN26dcOBAwdw69Yta1l8fDyMRiO6d+9e6rZJSUkYP3482rVrh/fffx+CIFR1c4mIqI5zWGAOGzYM7u7uiImJwd69exEXF4e5c+eif//+NrNnZ8yYgVatWlmX09LSMG7cOKjVajz//PM4efIkjh07hmPHjuHUqVOOeClERFQHOGzSj06nw8qVKzFv3jxMnDjR+mi8u4dXRVGE2Wy2Lv/111+4cuUKAGD06NE26zZq1Ag7d+6s8rYTEVHdI0iSVLEpovc5zpIl4jlMtUOtnSVLRER0P2FgEhERycDAJCIikoGBSUREJAMDk4iISAYGJhERkQwMTCIiIhkYmERERDLc9x/vVdUMhmxkZd2G2WwqcZ3r1xUQRbEaW0VVRalUwc3NE87Oro5uChHVMAzMUhgM2cjMvAVPT1+o1ZoSH/KuUilgMjEw73eSJCE/34jbt28AAEOTiGxwSLYUWVm34enpC41Gy09EqQMEQYBGo4Wnpy+ysm47ujlEVMMwMEthNpugVmsc3QyqZmq1ptQheCKqmxiYZWDPsu7hz5yIisPAJCIikoGBSUREJANnydZyXbq0L3OdMWPGY9y4F+9p/8uWfYG1a1cjPn7vPW1fFlEUMWTIIFy/fg0rV65Fs2bNq+Q4RERlYWDWckuXrrBZnjBhDIYM+T/06fOYtax+/fr3vP9Bg55A585d7nn7shw7dgTXr18DAOzYsRUvvTSxyo5FRFQaBmYtFxoaZldWv36DYssL5eXlQqt1krX/+vUfQP36D9xz+8qyY8dWODu7oFmz5khI2I4JE17hpBwicghew6zjtmzZhC5d2uPkyRN4441J6NOnCz755EMAwLp1/8Xzz49EVFR3DBzYB5MnT0RycpLN9suWfYHIyK7W5SNHDqFLl/Y4ePA3zJkzE5GR3RAdPQBffbW03E9DMhqN2L37Z3Tr1h0DBw7GtWtXcfz4Ubv1UlLOY8aMqejXrxd6934Uo0Y9g/j4bdZ6URSxdu1qDB8+BD17dsLjj0fh7benIysrq1ztIaK6jT1MAgDMmjUDAwY8jmefHQGNxnLv6bVr1xAd/TQaNGgIg8GATZt+wEsvjcWaNd+jXr16pe7vo4/eR58+UXj//Y9w8OAB/Oc/X6Fx4wBERfWX3aZfftmLrKws9OnzGMLC2uCTTz7Ejh1b8fDD7azrXLiQigkTxsDXtz5ee20KvL19cP58Eq5du2pdZ8GCj7Bx4wYMHfos2rfvCIMhB7/8sg8GQw7c3NzK+U4RUV3FwCyn/X9cwb7fr9iUCQIgSdVz/C4PNcSjYQ0rfb+DBj2BkSPH2pRNnPi69d9msxkdO0bgySf7IyFhG4YNe67U/XXr1hMvvBADAOjQIRz/+99v2L3753IF5o4d2+Dp6YUOHcKhUqnQuXMX7Nr1M15/fRrUajUAYPnyL6FSqbF06XK4urpZj1coNfVvxMWtxwsvxGDEiDHW8h49estuBxERwMCkAl26dLMrO3HiDyxbthRnz/6JjIwMa/mFC6ll7i88PMJmuUmTprK2K5SZmYkDB/ZjwIDBUKksp2nfvv2we/fPOHBgP7p27QEAOHz4IHr06G0Ny7sdOXIQkiRh4MDBso9NRFQcBmY5PRpm38OrDQ9f9/LysVm+evUqJk9+BcHBIXjjjbdQr149aDQavP32dBiNxjL35+6us1lWq9UwGvNkt2fXrgQYjUZ07vwoMjMzAQBhYQ/B1dUVO3ZsswZmRsbtUoeHMzIyoFQq4eXlLfvYRETFYWASAPvHwf322y8wGHLw/vsfQ6e7E356vb5a2rNjx1YAwNSpr9nV7d+/F9nZWXB1dYOHhydu3rxZ4n48PDxgNptx61Y6Q5OIKoSBScXKy8uDIAjW4VAA2LdvD3Jysqv82IWzYQcNehKRkVE2dZcuXcQ//zkPe/bsQv/+g9C+fUfs3v0zYmImwsXF/uO42rXrAEEQ8NNPG/Hcc6OrvO1EVHsxMKlYjzzSAQDw/vuzMHjwU7hwIRWrV/8Hnp5eVX7s+PhtkCQJzzzzHAICAm3q2rZ9BKtWrcCOHVvRv/8gjBkzHr/8shcvvTQOw4ePgo9PPaSkJCM3NxfDh49CQEAgBg9+Cv/+9xLo9Xq0b98Rubm5+PXXfRg79gX4+t77QxuIqG7hfZhUrGbNmuMf/5iFc+fOYvr0ydi6dRPeffc9eHlVT2C2bh1mF5aAZej4sccG4MiRQ7h58yYaNw7AkiXL0bChH+bP/wDTp7+OzZt/RIMGd64zT548DS+++DISE3dj2rTXMH9+LHJysuHi4lLlr4WIag9BkqrrhoiaKS0tC6JY/Ftw9erfaNDA/pf23WrDpB+yJfdnX1v4+rrjxo1MRzeDqEIq4zxWKAT4+BQ/6549TCIiIhl4DZOqnclkKrW+6EQjIqKagr+ZqNr16BFRav2+fYeqqSVERPIxMKnaffXV145uAhFRuTEwqdo9+GArRzeBiKjcOOmHiIhIBgYmERGRDAxMIiIiGRiYREREMjAwiYiIZGBgEhERycDArOWmT38dTz/9eIn127b9hC5d2uPUqRNl7mvIkEH45JN/lrsN2dlZ6NXrUfToEYGMjNvl3p6IqCZgYNZyffv2w5Url/HHH8eLrd+xYxv8/RujVavQKmvD7t07YTTmwWQyYdeuhCo7DhFRVWJg1nJdunSDi4sr4uO32dXdupWOw4f/h8jIx6q0DTt2bIOfXyP4+TXCjh327SAiuh/wST+1nFbrhO7de2LXrp8xadIbNg8237kzHmazGRERjyI2dg6OHDmMtLSb8PX1RdeuPTB+/ARotU4VOv7Nmzdw9OghjBgxBoIgYOXKZbh69YrN51UCwIkTv2PZsi9w8uQJSJKEJk2a4oUXXkKHDpbnzhqNRvznP18hPn47bt68Dk9PL7Rv3xH/+MesCrWPiEguBmYd0LfvY9i6dTMOHfofIiI6W8vj47ejZctWcHFxgZubOyZOfB3u7u64dOkCVqz4ClevXsG8eeW/ZllUfPx2iKKIyMjHoFAIBaG3DSNGjLGu8/vvx/Dqqy+hVatQTJ/+Ntzd3fHnn6dw7dpV6zpvvz0Nhw8fxIgRY9C6dRhu376FPXt2VahtRETlwcC8BzmbYm2WBUGAJElwGfQWACD3lzUQ01LtttN2ehbKeoHIP7MX+Wf32dWrg7tAHdIV5pt/I+/X/9rVK3wC4NR5eLnb265dB/j41ENCwnZrYF6+fAknTvyOSZPeQNOmQZg48XXr+mFhbeDh4YkZM6YiI+M2PDw8y33MQvHxW9GiRTCaNGkKwPIc2R07ttoE5pIln6JRo8b49NOlUCqVAICOHe98osnBgwfwyy/78O6782yGj6t6KJmIqCgGZh2gVCrRp09fbNr0I/Ly8qDVahEfv81aLkkSvvvuG2zc+AMuX74MozHPuu2FCxfuOTBTUs7j7NkziImZZC3r2/cxfPrpJzh37ixatAhGbm4uTp48gRdffNkalnc7dOggnJyc0KdP1D21g4ioMjAw70FhT7KQSqWAySRal8vqBapDukId0rXEemW9QLtjVFRkZD98++1/sX//XvTq1QcJCdvxyCMd4O3tg2+/XYPFi/+FZ54ZgUce6QB3d3ekpJzH++/PtgnP8tq+fQsAICKiMzIzM63/Xrz4X9aeZ2amHqIool493xL3o9dnwMenHgRBuOe2EBFVFAOzjnjwwZYIDGyC+PhtaNw4AOfPJ+PZZ0cCAHbt+hldunSz6Qlev36tQseTJAkJCdsBACNHDrOrT0jYgZdemgQ3N3coFArcvHmjxH3pdB5IS7sJSZIYmkTkMAzMOiQy8jF8/fVyeHl5QavVonv3ngCAvLxcqNUam3UrevvH778fx5UrlzF69PNo1679XXXH8NVXS3H06GG0a9cerVuHYdu2nzBs2HPFDsu2b98Ra9asxM6d8ejdu2+F2kVEdK8YmHVIZORj+Oqrpdi0KQ69evWBi4srAKBDh3CsW/cNvvtuLQIDm2DXrgScO3e2QseKj98KrVaLZ555Dq6ubjZ1LVu2xpo1XyM+fhvatWuPCRMm4tVXJ+DVV19CdPRQuLu74+zZP+Hh4YmBAwejQ4dwdOpkufXl0qWLaNUqFHq9Hrt3/4w5c2JLaAERUeXigwvqkEaN/BEa+hAkSUJkZD9r+ejR4xEV1R8rVvwb77zzFvLz8zFt2ox7Pk7hE326dOluF5YA4OzsjJ49e2PXrp9hNBrRps3DWLToCygUCrz//iz84x/TkJi42+ZezXnzPsSQIcPw448bMGXKJCxevADOzs733EYiovISJEmSHN0IR0pLy4IoFv8WXL36Nxo0CCxzH3dP+qH7n9yffW3h6+uOGzcyHd0MogqpjPNYoRDg42P/hz7AHiYREZEsvIZJ98RkMpVYJwhCifdUEhHdrxiYVG5Xrlwu9SPDGjRoiPXrN1Vji4iIqh4Dk8qtXj1ffPXV1yXW332LChFRbcDApHJTq9V48MFWjm4GEVG14qQfIiIiGRiYZajjd93USfyZE1FxGJilUCpVyM83OroZVM3y841QKnm1gohsMTBL4ebmidu3b8BozGOvow6QJAlGYx5u374BN7d7/wxQIqqd+Gd0KZydLc9azci4CbO55PsOFQoFRJFP+qkNlEoV3N29rD97IqJCDMwyODu7lvnLk48VIyKq/Rw6JJuSkoJx48ahbdu2iIiIwNy5c2EwGGRtGxcXh8ceewxhYWEYMGAAtmzZUsWtJSKiusxhPUy9Xo+RI0fCz88PCxcuRHp6OmJjY5Geno4FCxaUuu22bdswffp0vPDCC3j00UeRkJCAyZMnw9XVFd27d6+mV0BERHWJwwJz7dq10Ov1iIuLg7e3NwBAqVRiypQpiImJQYsWLUrcduHChXjsscfwxhtvAAAiIiKQnJyMRYsWMTCJiKhKOGxINjExEREREdawBICoqChoNBokJiaWuN2FCxeQnJyMAQMG2JQPHDgQf/zxB9LT06uszUREVHc5rIeZlJSEp556yqZMo9EgICAAycnJJW5XWNesWTOb8ubNm1vri4ZwWRQKQfa61bEfIkfhOUy1QUXP49K2d+g1TJ1OZ1eu0+mQkZFR4naFdXdv6+HhYVMvl5dX5dw+UNIHjhLdL3gOU21QlecxH1xAREQkg8MCU6fTQa/X25Xr9Xprb7E4hXV3b1vYsyxtWyIionvlsMBs1qwZkpKSbMqMRiNSU1MRFBRU4naFdXdf5yzcV2nbEhER3SuHBWa3bt1w4MAB3Lp1y1oWHx8Po9FY6q0hjRs3RlBQkN2DCjZv3oywsLByTfghIiKSy2GBOWzYMLi7uyMmJgZ79+5FXFwc5s6di/79+1tnvALAjBkz0KqV7YcVT5o0CVu3bsWCBQvw22+/4f3338f+/fsxceLE6n4ZRERURzhslqxOp8PKlSsxb948TJw4EVqtFgMGDMDUqVNt1hNFEWaz2aasX79+yM3NxdKlS7Fs2TIEBARg/vz5fGgBERFVGUHi51YRERGVibeVEBERycDAJCIikoGfh3mP/v77byxbtgzHjx/HuXPnEBQUhM2bNzu6WUSybd26FZs2bcLJkyeRkZGBxo0b45lnnsGwYcOgUPBvaar5duzYgRUrViA5ORk5OTl44IEHEBkZiZiYGLi7u1f68RiY9+jcuXPYs2cP2rRpA1EUwUvBdL9ZsWIF/Pz8MG3aNPj4+OC3337De++9hwsXLmD69OmObh5RmTIyMtChQweMGTMGHh4eOHPmDBYvXowzZ85g+fLllX48Tvq5R6IoWv8Kf/PNN3HixAn2MOm+kp6ebnffcmxsLL755hscOnQIGo3GQS0junfffvst3nnnHSQmJuKBBx6o1H1z3OUecciK7nfFPeSjZcuWyMvLw+3btx3QIqKK8/LyAgDk5+dX+r45JEtEVocPH4anpyd8fHwc3RQi2cxmM0wmE86dO4fPPvsMvXr1gr+/f6Ufh4FJRACAP/74Axs2bMDLL78MpVLp6OYQyRYeHo7MzEwAQNeuXTF//vwqOQ4Dk4hw48YNTJo0CWFhYRg/fryjm0NULqtWrYLBYMC5c+ewZMkSTJgwAStWrKj0P/wYmER1XGZmJsaPHw8nJycsWbIEarXa0U0iKpeWLVsCANq1a4fWrVvjqaeeQnx8PB577LFKPQ4Dk6gOy8vLw0svvYS0tDSsXbvWOmGC6H7VsmVLKBQKpKamVvq+GZhEdZTJZMKrr76KM2fOYNWqVWjUqJGjm0RUYUePHoUoipz0U5MYDAbs2bMHAHDp0iVkZWVh27ZtAICwsDD+8qEab86cOdi1axemTp2K3NxcHDt2zFrXvHlzuLm5ObB1RGUbN24cIiIi0KJFC2i1Wpw+fRrLli1DSEgI+vTpU+nH44ML7tHFixfRu3fvYutiY2MRHR1dzS0iKp9evXrh0qVLxdZ9/fXXCA8Pr+YWEZXPv/71L/z888+4ePEiAMDf3x99+/bFmDFjquQPPgYmERGRDHxcDRERkQwMTCIiIhkYmERERDIwMImIiGRgYBIREcnAwCQiIpKBgUlUS124cAHPP/882rdvj5CQECQkJDi6SfcsJCQE77zzjqObQXUcA5NIpg0bNiAkJAShoaG4fPmyXf2LL76IXr16OaBlxZs5cyZOnjyJSZMm4cMPP0RoaKijm0R0X2NgEpVTfn4+li5d6uhmlEoURRw6dAiDBg3CyJEjMXjwYDRo0MDRzSK6rzEwicqpZcuW2LBhQ7G9zJpCr9cjPz8f7u7ujm4KUa3BwCQqpxdeeAEAZPUyzWYzlixZgsjISISGhqJHjx748MMPkZube8/H//PPPzF+/Hi0a9cODz/8MEaMGIFDhw5Z6xctWmR9DuzixYsREhJS5lCxJElYtWoVBg0ahLCwMHTq1AkzZsxAenq6zXq9evXCuHHj8OuvvyI6OhphYWGIiopCXFyc3T7T09Mxc+ZMPProowgLC8PAgQOxbt26Yo+9Zs0aDB48GA899BDCw8MxZswYm9dUKCEhAQMHDkRoaCgGDBiAxMREm/rs7Gz885//RK9evRAaGoqIiAiMGDECBw8eLPX1E8nBTyshKic/Pz889dRT+P777zFhwgT4+fmVuO4777yD9evXo2/fvhg9ejROnDiBZcuW4dy5c/jyyy8hCEK5jp2UlIRnn30WLi4uGDduHLRaLb777juMHj0aK1asQIcOHRAZGYl69eph1qxZiIyMRGRkJFxdXUvd77vvvovvv/8eTzzxBIYPH44rV65g9erV+OOPP7B+/XpotVrruhcvXsSkSZPw9NNPY/Dgwdi0aROmT58OjUaD/v37A7B8zuaoUaOQnJyMZ599FgEBAUhISMDMmTNx+/Zt6x8dgOVa63fffYcuXbrgySefBAAcOXIEBw8eRPv27a3rHTt2DLt27cIzzzwDV1dXrFq1CpMmTcKuXbusn+M5a9YsbN26FcOHD0fz5s2h1+tx/Phx/Pnnn+jQoUO53msiOxIRyfL9999LwcHB0tGjR6XLly9LrVu3lmbOnGmtf+GFF6SePXtal0+fPi0FBwdLb775ps1+Pv30Uyk4OFjauXNnudvw8ssvS61bt5bOnz9vLUtLS5M6duwoPfnkkzZlwcHB0qefflrmPg8fPiwFBwdLP/zwg035wYMHpeDgYGnt2rXWsp49e0rBwcHSpk2brGUGg0GKioqSunfvLpnNZkmSJGnlypVScHCwtGHDBut6JpNJGjVqlBQaGiqlp6dLkiRJBw4ckIKDg6VZs2bZtUsUReu/g4ODpdatW0spKSnWssL3d9WqVday9u3bS7Nnzy7zNRPdCw7JEt2Dhg0b4qmnnsKGDRtK/Iisws9LHT16tE356NGjoVQqsXv37nId02w2Y9++fejZsyeaNGliLff29kZ0dDROnjyJmzdvlmufALB161a4uLiga9euSE9Pt34FBQWhXr16+O2332zW9/HxsfYkAcDJyQlPP/00rly5gjNnzgCwvHZvb288/vjj1vWUSiVGjRoFo9GIX3/9FQCwfft2AMDEiRPt2nV37zs8PByBgYHW5QcffBBubm64cOGCtczd3R3Hjx/HtWvXyv0+EJWFgUl0jyZMmACg5GuZly5dgiAIaNq0qU25u7s7fH19SwzakqSnp8NgMNjtDwCCgoKsxyyvlJQU5OTkoHPnzujUqZPN182bN5GWlmazfkBAABQK218dhQFeePxLly4hMDAQSqXSZr1mzZoBgPXzC1NTU1GvXj14e3uX2c6GDRvalXl4eECv11uXp06dir/++gs9evRAdHQ0FixYgOTk5DL3TSQHr2ES3aOGDRtiyJAhWL9+vTU870eiKMLT0xMLFiwotl6n01Vzi4p3d0gXkop8pG+/fv3Qvn17/Pzzz9i/fz9WrVqFZcuWITY2FoMGDaquplItxR4mUQUUBuWSJUvs6ho1agRJknD+/Hmb8qysLNy4cQONGjUq17G8vb3h7Oxstz8A1l5UefcJWHqMer0ebdq0QefOne2+7n7gQWpqKkRRtClLSUmxOX6jRo3w999/w2w2F9tOf39/67Fv3rxpNxu3Inx9tvQrEgAAAv9JREFUfTFs2DAsWrQIu3fvhr+/PxYtWlRp+6e6i4FJVAENGjTA008/jbi4OLv7Mrt37w4AWLlypU35ypUrYTab0bNnT2tZamoqUlNTSz2WUqlE165dsWvXLpt1b9++jbi4OISGhqJevXrlfg39+/eHKIr47LPP7OrMZjMyMjJsytLS0rBlyxbrcm5uLr777js0aNAAISEhAIAePXogPT0dmzZtsq4niiK+/vpraDQadOrUCQAQFRUFAMUGWtGeoxxmsxmZmZk2ZTqdDv7+/jbDtkT3ikOyRBX04osvYv369Th79qxND+/BBx+0DtlmZWUhPDwcp06dwvfff4+uXbtaAxW4MzFo586dpR7rtddew759+/Dss8/i2WefhVarxbp165CZmYk333zzntrfoUMHDB8+HMuWLcOZM2fQtWtXqNVqpKamYvv27Zg0aRKio6Ot6zdp0gSzZ8/GqVOn0KBBA2zcuBHnz5/Hxx9/bB02HTp0KNatW4e3334bp0+fRuPGjZGQkIBff/0Vb7zxhvU2kPDwcERHR+O///0vUlNT0a1bNwCWW0hCQkLKNdSdnZ2Nbt26oW/fvtYJQUeOHMHevXvx3HPP3dN7Q1QUA5Ooggp7mWvWrLGrmzNnDvz9/fH9999j586d8PHxwdixYzFp0qRy34MJWCbNfPPNN5g/fz7+/e9/Q5IkhIaGYt68eRW6z/Cdd95Bq1atsHbtWixYsABKpRJ+fn7o168fIiIibNb19/fHrFmz8NFHH+HcuXNo2LCh3TVCrVaLlStX4pNPPsGmTZug1+sRGBiIuXPnYujQoTb7e++99xDy/+3ZsQ2DMBBA0UuHxAo0VBTegjFcsAZDMAMTsINLpkJKqvRHiqTIexPY13zZN01xHEds2xZ930cp5fZ9uq6LWmuc5xmttbiuK4ZhiHVdY1mWj2cDb4/n3X8P4G/N8xzjOMa+778+CnydHSYAJAgmACQIJgAk2GECQIIXJgAkCCYAJAgmACQIJgAkCCYAJAgmACS8AA5pXYmOBZzjAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Xb2Pyat2qwYD", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 495 + }, + "outputId": "93ed2037-e7b9-4a39-8a5e-4e89b6f41151" + }, + "source": [ + "import seaborn as sns\n", + "import matplotlib.pyplot as plt\n", + "epoch=[]\n", + "for i in range(3):\n", + " epoch.append(i+1)\n", + "sns.set(font_scale=1.4)\n", + "\n", + "plt.figure(figsize=(7,7))\n", + "dic={'Train_F1_Score':train_f1,'Val_F1_Score': val_f1}\n", + "x=pd.DataFrame(dic)\n", + "ax = sns.lineplot(data=x)\n", + "ax.set(ylim=(0,1),xlabel='No. of epochs',ylabel='F1 Score',title='XLNET')\n", + "ax.set_xticks(range(3)) # <--- s\n", + "ax.set_xticklabels(['1','2','3'])" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[Text(0, 0, '1'), Text(0, 0, '2'), Text(0, 0, '3')]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 69 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcwAAAHNCAYAAACTh0sjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeVxU5eI/8M+ZlWWYYUdAQAHXQNM0cUnTNBfsdlMzWzTTrC6Fpakl/e4107Rbml/S1ErtWlZeteJezSzNbqZWt1VT65qggiCLgAzIMtv5/TEwMg7LAWYcls/79eJFPOecZ56Zjnx4nvOc5wiiKIogIiKiBsnc3QAiIqK2gIFJREQkAQOTiIhIAgYmERGRBAxMIiIiCRiYREREEjAwiYiIJGBgErVyzz77LOLi4pCenu6wbceOHejRowc+/vhjAMCoUaMwe/bsRuvr0aMHEhMTYbFYHLb369cPzz77rO3nCxcuoEePHvV+rVq1Ct99912D+9T+unDhQgs/ESL3ULi7AUTUsEWLFuHLL7/EX//6V7z33nsQBAEAUFBQgFdeeQUJCQm46667mlzvmTNn8OmnnyIxMVHS/hMmTMCtt97qUN69e3cEBQXh5ZdftitfuXIlgoODHQLc39+/yW0lag0YmEStnL+/P5555hksXrwYO3fuxNSpUwEAy5cvR2VlJZYuXdrkOpVKJaKiorB+/XqMHz8eMlnjg029evXCnXfeWe/2a7elpqYiKCiowWOI2hIOyRK1AZMmTcKgQYOwatUqXLp0CV9++SX27duHxx57DF26dGlyfYIgICkpCWfOnMHevXud32CidoiBSdRGLF26FBUVFXj++eexdOlSxMTEYM6cOc2ub/z48YiNjcX69evrvJZ5rYqKChQVFTl8GY3GZreBqC1hYBK1EV27dsWjjz6K/fv3Izc3Fy+88AJUKlWz65PJZHj88ceRnp4uqZe5fv16DB482OHrm2++aXYbiNoSXsMkakP8/Pxs32+44YYW1zdu3DisX78er7/+OiZMmNDgtcwpU6bUOUGoV69eLW4HUVvAHiZRG5GXl4dXX30V3bp1Q3FxMdauXdviOmUyGZKSkpCRkYFPPvmkwX2joqIwZMgQh6+aECdq7xiYRG3Eiy++CJPJhPXr12PSpEnYunUr/ve//7W43vHjx6Nbt26Sr2USdVQMTKI24ODBg/jss8+QlJSEyMhILFy4ED4+PliyZAla+gz4mhmzGRkZ2LNnj5NaTNT+MDCJWrkrV67ghRdeQPfu3TFr1iwA1muYixYtws8//4wdO3a0+DXGjx+P7t27s5dJ1ABO+iFq5VJTU5Gbm4tXX30VSqXSVj5p0iR89NFHWL16NUaPHo2AgAAA1qXs1q9f71BPt27dMGbMmDpfo6aX+dRTT9Xbjt9++w3/+te/HMrDw8MxYMCApr4tojaHgUnUip04cQLbtm3D1KlT0b9/f4ftS5cuxZ133omVK1di1apVAIBz584hNTXVYd8JEybUG5iAdcZs9+7dcfr06Tq37927t87bT8aOHcvApA5BEFt6AaQFzp8/j82bN+PYsWP4448/EB0dLfkaSlpaGjZu3Ijs7GxERkbi8ccfx4QJE1zcYiIi6qjc2sP8448/8NVXX6Fv376wWCySJy/s27cPzzzzDB555BEMHToUBw4cwPz58+Ht7Y0RI0a4uNVERNQRubWHabFYbDdKP/vsszhx4oSkHmbNBIXaw06zZs2CXq/Hrl27XNZeIiLquNw6S1bKExKulZWVhYyMDIcVRyZOnIhff/0VRUVFzmoeERGRTZu7rSQjIwMAEBMTY1ceGxtrt52IiMiZ2lxglpSUAAC0Wq1duU6ns9tORETkTG0uMImIiNyhzd2HWdOT1Ov1CAoKspXX9CxrtktVXHwFFkvL5j0FBGhQWFjWojqI3InnMLUHzjiPZTIBfn7edW5rc4EZHR0NwHqtsvZ1zPT0dLvtUlksYosDs6YeoraM5zC1B648j9vckGxERASio6MdVhzZs2cP4uPj4e/v76aWERFRe+bWHmZFRQW++uorAEB2djbKysqwb98+AEB8fDzCw8ORkpKCtLQ0nDp1ynbc3LlzMW/ePERGRmLIkCH44osvcOTIEbzxxhtueR9ERNT+uTUwCwsL8eSTT9qV1fy8cuVKTJo0CRaLBWaz2W6f8ePHo7KyEhs3bsTmzZsRGRmJ1atXc5UfIiJyGbeu9NMaFBaWtXjMOyjIBwUFpU5qEdH1x3OY2gNnnMcymYCAAE2d29rcpB8iat0qKq6grKwEZrPR3U2hDiY/X9bg81zlciU0Gh08PeueBdsYBiYROY3RaEBpaTF8fQOhVKohCIK7m0QdiEIhg8lUd2CKogijsQqXL1+CQqGEUqlqcv1tbpYsEbVepaWXodHooFJ5MCypVREEASqVB7y9dSgru9ysOhiYROQ0JpMBarWnu5tBVC8PD08YjYZmHcvAJCKnsVjMkMnk7m4GUb1kMjksFnPjO9Z1rJPbQkQdHIdiqTVryfnJwCQiIpKAs2SJiK4xbNiARvd56KE5mD370WbVv3nzG9i+fRv27/+6WcfX58UXn8enn+5xKI+P74sNGzYDAL744nMcPLgfp06dREFBPpKSnsR9901v8mt9880RvPfeVpw9m46qqioEBATihhviMXPmw4iMjGrxe2mNGJhERNfYuPFtu58fe+whTJlyD0aPHmcrCw4Obnb9d9zxZwwZMqzZxzckLCwcf/vbcrsyb++r9x1++eUXyMnJxpAhw/Cvf33UrNf44ovPsWRJCsaNS8S9906HSqXE+fPncPDgAZw/f5aBSUTUUcTFxTuUBQd3qrO8RlVVJdRqD0n1BweHIDg4pNnta4harW6wnS+8sBIymfVqXHMDc9euf6J//wH4f/9vqa1s4MAETJkyrcGFA5zFZDJBEATI5dd3ghmvYRIRNdHevbsxbNgAnDx5Ak8/PRejRw/Dq6++DADYseN9PPzwDIwdOwITJ47G/PnJyMhItzt+8+Y3MGbMLbaff/rpBwwbNgDff/8dXnjhrxgzZjgmTUrEpk0bnR5ANWHZEqWlpfD3D5BU/9Gjh/GXv8zCbbcNxbhxI/HEE4/g9Onfbdtzc3Px178+i3HjbsVttw1FcvKj+O23k3Z1TJlyB1599e/Yvn0b7r77Txg1agguXSoAAHz22V7MnHkfRo0aggkTxmDt2ldhMDTvtpHGsIdJRNRMzz+fgsTEP+G++6ZDpbKuHJOXl4dJk+5Gp06hqKiowO7dH+Mvf5mF9977EIGBgQ3W98orKzB69FisWPEKvv/+W/zjH5sQERGJsWMnNKldJpPJ7me5XO7U2cs9evTEwYP78f7772LkyNsQGhpW535ffPE5nn/+OQwbNgJ/+9tyqFRKHD9+DAUFBejevSfKy68gOfkRiKKIefMWwcvLCx98sA3JyY9i8+ZtiIrqYqvrq6++RGhoGJ544ikolSpoNBrs3Lkd69atwZQp9yApaS5yc3OwYcM6VFRUYNGi55z2fmswMInIpY78ehGHj1902+sP6xOKofGhLqn7jjv+jBkzZtmVJSfPs/232WzGzTcn4K67JuDAgX2YNu2BBusbPnwkHnkkCQAwcOAg/Pe/3+E///miSYF59mwGbr01wa5s1arXkJAwRHIdjXnssSdw7txZrF+fivXrUxEQEIghQ4Zh8uR7EBvbDYB1KbrXX0/FwIEJWLlyle3YwYOvXrv95JPdyM29iK1bP0B0dCwAYMCAQbj77juwbds/8Nxzz9v2NRqNePXVdfDy8gIAlJdfwVtvbcA999yPpKS5AKxL43l5eWPZsr9h+vSH6g3y5mJgEhE107Bhwx3KTpz4FZs3b8Tp07+jpKTEVp6VldlofYMG2Qddly5dJR1XW3h4ZyxdusKuLCIiskl1NCYoKBhvvbUVv/zyE7777hscO/YzPvnk39i37xOsWLEKgwcPRWbmeeTn5+Hxx5+qt55jx35G167RtrAEAE9PTwwZcgt++eUnu33797/JFpaA9XMuL7+C2267vVaPWoabbroZZrMZp0//zsAkorZlaLzrenju5udnfx0vNzcX8+c/ge7de+DppxcjMDAQKpUK/+//PSPpupqPj9buZ6VSCYOhqkltUqlU6Nmzd5OOaQ6ZTIb+/Qegf3/rLTinT/+OJ554FG+9tR6DBw9FSYl1vdbAwKB666jvWqifnz/0ev01Zfb7Xb5cDACYPbvuXnteXq70NyMRA5OIqJmuvS743XdHUVFRjhUrVkGrvRp+1/7yb4+6d++JgQNvxjffHAUA6HS+AGCbnFMXrVaLzMxzDuXFxUV2nx8AXHsJtuaPixdffBkhIZ0AAHK5DGazdZKUK2Yhc5YsEZGTVFVVQRAEKBRX+yKHD3+F8vIrbmyV8xUVFTqUWSwWXLhwAQEB1p5gZGQUgoNDsHfv7nrr6dPnRmRkpOPs2QxbWWVlJY4ePYy+ffs12Ib4+L7w9PREXl4eevbsjZ49e6NXr962/65vFm9LsIdJROQkN900EACwYsXzuPPOycjKysS2bf+Ar6+fm1t21dmzGTh37mpAZWScwZdfHgAAjBw5WlIdTz+djIiIKAwdegs6dQrF5cuX8ckn/0Z6+h+YO/dpANbe9+OPP4nnn38Ozz23EOPGJUKpVOHkyV/Rs2dvDB16CxIT78COHe9j0aJ5mDPnL/Dy8sQHH2xDZWUFHnhgZoNt0Gg0mDPnL9i4cS0KCvLRv/9NUKvVuHDhAo4e/Rrz5z/j9F4mA5OIyEliYmLx3HPPY8uWN/HMM/MRExODJUtexKpVKxo/+Do5eHA/3n77LdvP+/Z9gn37PgEAHD78g6Q67r//QRw8eABvvbUBRUWF8PbWICqqC1588WWMGDHKtt9tt90OtdoD77yzBUuWPAeVSoUePXpi+PBbAQBeXt5Yu/ZNrFu3BqtXr4TJZEKvXjfgtdc22t1SUp+pU+9DUFAwtm9/Dx99tANyuQKhoaEYNGiIw/VgZxBEURSdXmsbUlhYBoulZR9BUJAPCgpKndQiouvPWedwbu55dOrUPpdFo9ZPoZDBZGp8oYeGzlOZTEBAgKbubS1qHRERUQfBIVkiojbg2tV7rlV7olFLmM1mNDTw6OxVg9oSBiYRURtw7eo915J6/bExTz75F4dFA2p77bWNtnsvOxoGJhFRG7Bp0zvX5XUWLUpBeXl5vdvb66O7pGBgEhG1Addj9R4AiIzscl1epy3ipB8iIiIJGJhEREQSMDCJiIgkYGASERFJwMAkIiKSgIFJREQkAW8rISK6xjPPzENGRjp27vx3ndv37fsEy5cvwZtv/gO9e8c1WNeUKXdgyJBhmD//GUmv/cQTj9S5cMDtt4/H3/62DADw0Uc78e23R3Dq1AlcvnwZy5a9JPlJI7V9/vmn2LnzA2RmnocoAkFBQYiP74tHH30cfn7+Ta6vvWNgEhFd4/bbx2PJkhT8+usxxMf3ddj++ef70LlzRKNh2Vzx8X3x+ONP2ZX5+V19RFjN00USEoba/rup3ntvKzZuXIepU+/F7NmPAQAyMtKxf/+nuHSpgIFZBwYmEdE1hg0bDi8vb+zfv88hMIuLi/Djj//FjBmzXPb6Pj4+iIuLr3f7xo1bIJPJcPFiTrMDc9euf2L8+IlITp5vK0tIGIL77psOi6XxJ360VFVVJdRqD5e/jjMxMImIrqFWe2DEiJH48ssvMHfu03YLmx88uB9msxkJCUOxcuUL+OmnH1FYeAlBQUG45ZZbMWfOYy4PApms5dNPSkv1CAgIlFT/p5/uwY4d7+P8+XPw9PREr143YMGCxejUKRSA9SHUr7/+Go4f/xmCIMONN/ZHcvI8RERE2uoYNmwAHn30CZSXX8Hevbtx+XIxvvrqO4iiiJ07P0Ba2oe4eDEHfn7+uPPOSZgxY1arW+SdgUlEVIfbbx+HTz/dgx9++C8SEobYyvfv/wy9evWGl5cXNBofJCfPg4+PD7Kzs/D225uQm3sRy5f/vUWvLYqiw9NJnPU0kho9evTCxx/vQqdOoRg2bHi94fn+++9g/frXkJj4J8yZkwSz2YQff/wBly8Xo1OnUOTl5SIpaQ46dQpFSsoSiCKwZcsbePzxOdi6dbvdUPKuXdvRo0cvLFr0HEwmIwBg3bo1+PjjDzF9+kzExfXB6dO/Y/PmNyGTyTB9+kNOfc8txcAkIpcr372yznKvOxYDACqPvgdLYabDdvXg+yAPjILxf1/DePqww3Zl92FQ9rgF5kvnUfXN+w7bZQGR8Bhyf7Pa3L//QAQEBOLAgc9sgZmTk40TJ45j7tyn0bVrNJKT59n2j4/vC53OFykpC1FSchk6nW+zXhcAvvnmiMPTSd57bxeioro0u85rPf30M0hJWYhXXlmBV15ZgdDQcAwdegvuuec+hIaGAQDKysqwZcub+NOf7sKiRc/Zjr3llltt/71jx/swmYxYs+Z1WzjGxcVj6tQ78dFHOzB79qO2fTUaDV56abWtB5udfQE7d27HvHmLcNddUwAAAwcOgsUiYtu2tzFlyjR4eno67T23FAOTiKgOcrkco0ffjt27/4Wqqiqo1Wrs37/PVl4zlPjvf3+MnJwcGAxVtmOzsrJaFJh9+tyIuXPn25XVDH86S3R0LN59dwd++OE7/Pe/3+GXX37Erl3bsXfvbrz++pvo1q0HTpw4jsrKSkyceGe99Rw79gv69x9o15MMDg5BfHxfHD/+i92+gwcPsxvu/eGH/0IURYwcOdquRz1w4M144411yMw8jx49ejrxXbcMA5OIXK6mJ1mfxnqByh63QNnjlnq3ywOjGn2N5hgzZjz++c/3ceTI1xg1ajQOHPgMN900EP7+AfjnP9/DunX/h3vvnY6bbhoIHx8fnDt3FitWLLULz+bQaDTX5ekkSqUSgwcPw+DBwwAA3333DRYtegpvv70JK1a8Ar2+BAAQGBhUbx2lpXp069bdodzf3x+Zmecdymq7fLkYoihi4sS6b4nJy8tlYBIRtQU9e/ZCVFQX7N+/DxERkTh7NgP33TcDAPDll19g2LDhSEqaa9s/Pz/PXU11ikGDBiMmphvOnz8LANBqdQCAS5cKEBwcUucxWq0WRUWFDuVFRUW246+yn8Sj1eogCALWr98EpVLpUEdYWOdmvAvXYWASETVgzJhxeOedLfDz84NarcaIESMBWG+LUCpVdvt+/vk+dzSxWYqKCuHvH2BXVlVVifz8PHTtGg0AiIvrAw8PD+zdu7vee0779LkR//73x3bXbQsK8nHixHE88MDMBttw000DAVh7mrWvi7ZWDEwiogaMGTMOmzZtxO7daRg1ajS8vLwBWCen7NjxAXbu3I6oqC748ssD+OOP09elTb//fgoXL+bg8uXLAICTJ08AADw8PDF48FBJdcyYMQ1Dh96Cm28ejMDAQBQU5OPDD3egpOQy7r77XgDWoeGHHpqDDRvWwmy2YPjwEbBYRPz00w8YM2YsevbsjalT78Mnn+zGvHlP4MEHZ8FisWDLljfh46PFpElTG2xDZGQUpkyZhuXLl2DatAcQFxcPs9mC7OwLOHToP0hNXd+CT8n5GJhERA0ID++MuLg+OHHiOMaMGW8rnzlzDkpKSvD222/BYrFg2LDhWLQoBfPmPe7yNn344Q58+uke28/bt28DYJ0YtGvXbkl1zJr1CI4c+Rrr1q3B5cvF0Ol8ERPTDampG9C//wDbfvff/yB8ff2wY8f72LdvD7y8vHDDDX3g62u9HhkS0gmvv/4WXn89FcuXLwEgoF+//njxxVfsJgLVZ+7c+YiKikJa2kd49923oVKp0blzBIYOrf+atbsIoiiK7m6EOxUWlsFiadlHEBTkg4KCUie1iOj6c9Y5nJt7Hp06RTmhRURNp1DIYDI1vkpRQ+epTCYgIEBT97YWtY6IiKiD4JAsEdF1cu3qPbUJggC5XO6U1zGbzWho8NDZqwZ1FPzUiIiug4sXc3D33X+qd3tTrj825p57/ozc3Iv1bt+589+21XxIOgYmEdF1EBgYhE2b3ql3+7W3qLTE3/++BkajocG2UNMxMImIrgOlUnldVu8BgJiY2OvyOh0NJ/0QERFJwMAkIqfq4HeqUSvXkvOTgUlETiOXKxq8dkbkbkajAXJ5865GMjCJyGk0Gl9cvlwAg6GKPU1qVURRhMFQhcuXC6DRNO/Ra5z0Q0RO4+lpXWe1pOQSzOb67zkkcgWZTAaLpf6VfuRyBXx8/GznaVMxMInIqTw9vZv9C4moJVy9TCmHZImIiCRgYBIREUnAwCQiIpKAgUlERCQBA5OIiEgCBiYREZEEDEwiIiIJGJhEREQSuDUwz507h9mzZ6Nfv35ISEjAsmXLUFFR0ehx5eXlWLVqFUaPHo2+ffvi9ttvx7p162AwcA1LIiJyDbet9KPX6zFjxgyEhYUhNTUVRUVFWLlyJYqKirBmzZoGj33++edx4MABzJs3D926dcPx48fx2muvQa/XIyUl5Tq9AyIi6kjcFpjbt2+HXq9HWloa/P39AQByuRwLFixAUlISunXrVudxJpMJ+/btw8MPP4zp06cDABISEpCTk4M9e/YwMImIyCXcNiR76NAhJCQk2MISAMaOHQuVSoVDhw7Ve5woijCbzfDx8bEr12q1fDoCERG5jNsCMz09HbGxsXZlKpUKkZGRyMjIqPc4pVKJO++8E++++y6OHTuGK1eu4Ntvv8WOHTtw//33u7rZRETUQbn1GqZWq3Uo12q1KCkpafDYF154AUuWLMHUqVNtZTNnzsQTTzzh9HYSEREBbfTxXqtXr8ZXX32F5cuXo0uXLvjll1/w+uuvIzAwEHPmzGlSXQEBGqe0KSjIp/GdiFoxnsPUHrjyPHZbYGq1Wuj1eodyvV6P6Ojoeo87ffo0tmzZgvXr1+O2224DAAwcOBAmkwmvvfYa7r33Xmg00kOwsLAMFkvLrn26+hlsRK7Gc5jaA2ecxzKZUG9Hym3XMGNiYpCenm5XZjAYkJmZ2WBgnjlzBgDQq1cvu/LevXvDYDAgLy/P+Y0lIqIOz22BOXz4cHz77bcoLi62le3fvx8GgwEjRoyo97jw8HAAwMmTJ+3KT5w4AUEQEBYW5poGExFRh+a2Idlp06Zh27ZtSEpKQlJSEgoLC/HSSy9hwoQJdrNnU1JSkJaWhlOnTgEA4uLi0KdPHyxZsgSFhYWIiorC8ePH8eabb2Ly5Mnw9PR011siIqJ2zK3XMLdu3Yrly5cjOTkZarUaiYmJWLhwod1+FosFZrPZ9rNcLsfGjRuRmpqKN998E5cuXUJoaChmzZqFRx999Hq/DSIi6iAEsYPf7c9JP0Q8h6l9aLeTfoiIiNoSBiYREZEEDEwiIiIJGJhEREQSMDCJiIgkYGASERFJwMAkIiKSgIFJREQkAQOTiIhIAgYmERGRBAxMIiIiCRiYREREEjAwiYiIJGBgEhERScDAJCIikoCBSUREJAEDk4iISAIGJhERkQQMTCIiIgkYmERERBIwMImIiCRgYBIREUnAwCQiIpKAgUlERCQBA5OIiEgCBiYREZEEDEwiIiIJGJhEREQSKNzdACIioqawWETkFpUjM68UmfllyMwrxYWCK7j7tm4Y2jvEZa/LwCQiolarymhGdsEV+3DML4PBZAEAKOQCwoM06BsTgL7dglzaFgYmERG1CmUVRpzPK0VWXpktIC8WXoEoWrd7qRWIDNFgxI3hiAzRICrEB50CvKCQW68uBgX5oKCg1GXtY2ASEdF1JYoiCksqcT6vDFn5pcjMK0NmfimK9FW2ffy1akQG+2BAjyBEBPsgKkSDAJ0HBEFwW7sZmERE5DImswUXC6uvN9YKyPIqEwBAEIDQAG907+yLyBAfRIZoEBGsgY+Xys0td8TAJCIip6g0mJCVX2btMVYPqWYXXIHJbL3eqFLI0DlYg5t7hyAyWIPIEB+EB3lDrZS7ueXSMDCJiKjJSsqqbJNwagIyv7gC1ZcbofFUIjJEg9EDOiMyRIPIYB908veCTOa+IdWWYmASEVG9LKKIguIKh3AsuWKw7ROo80BUiA8Gx3WyDqsGa+Dno3br9UZXYGASEREAwGiyIOfSFdtM1fP5pcjKL0OVwQwAkMsEhAZ4I66rPyJCrBNxIoI18PJQurnl1wcDk4ioAyqvNCIrv8w6UzWvFOfzrLdwmC3WQVW1So7IYA2GxYVah1RDfBAW6A2louMuEMfAJCJqx0RRRHFple3WjZoh1UsllbZ9dN4qRIb4oG9sgG1INcjPE7J2NqTaUgxMIqJ2oq4l4zLzylBWYQQACACC/b3QNVSLETeG2cJRp1G7t+FtBAOTiKgNqjKacaGgzG5VnLqWjOvXLdB2f2PnIA081fy131z85IiIWrlrl4w7n1eK3KJyhyXjbu0XjohgxyXjyDkYmERErUTtJeMy80qrJ+WUorjUccm4gT2DW82ScR0FA5OIyA2kLhnXI9IXkcGte8m4joKBSUTkYg5LxuWVIftSGUxm65hqXUvGdQ7yhqqNLBnXUTAwiYicSPqScRHtZsm4joKBSUTUDDVLxp2vda0xK6+sQy4Z11EwMImIGtGUJeNqP6KqoywZ11EwMImIaimvNFavinN1WJVLxhHAwCSiDqpZS8aFaBDkyyXjOioGJhG1exaLiItF5ciqmYiTzyXjqOkYmETUrkhdMq5/98DqG/99EB7kzSXjqFE8Q4iozSotN9iuNWbllXHJOHIpBiYRtXqiKOJSSaXtOmNjS8bVDKlyyThyJgYmEbUq1y4ZVzOsWsEl48jNGJhE5DYVVSZcKGh8ybhBXDKOWgEGJhFdFyVlVThfvcj4+bwyZNWxZFwUl4yjVoyBSUROVdeScZl5ZdA3sGRcVIgPfDUqXm+kVo2BSUTNZjRZkH2pzO5a47VLxoUFeiOeS8ZRO8DAJCJJuGQcdXQMTCKyY7dkXHWv0WHJOI0KkcFcMo46FgYmUQcmiiLyL1fgZFYJTv5R0OiScVEhPogI8YHOm7dwUMfDwCTqQKoMZpzL1eNMdgnSs/VIzylBabk1HOtaMq5zsDc8VPw1QQS4OTDPnTuHZcuW4aeffqCFn+kAACAASURBVIJarUZiYiIWLFgAT0/PRo8tLS3Fa6+9hs8++wxFRUUIDg7GnXfeiSeffPI6tJyo9RNFEQUllUjPLqn+0iMrvwyW6nXjQvy90Cc6ADHhOgyIC4WHDFwyjqgBbgtMvV6PGTNmICwsDKmpqSgqKsLKlStRVFSENWvWNHhseXk5HnjgAQiCgIULFyI4OBhZWVnIzc29Tq0nan0MRjPO5ZYiPbvE2oPM0dtu5VCr5IgO1WLC4EjEhOkQE66DxvPqTNWgIB8UFJS6q+lEbYLbAnP79u3Q6/VIS0uDv78/AEAul2PBggVISkpCt27d6j32zTffRGlpKXbv3g1vb28AwKBBg65Lu4laA1EUUVhSiTM51UOr2SXIyi+zzVgN9vPEDV38ERuuRUy4Dp2DNFwAgKiF3BaYhw4dQkJCgi0sAWDs2LFISUnBoUOHGgzMXbt24b777rOFJVF7ZzTV9B6t4XgmpwQlZdbeo0opQ3SoFuMGWXuP0eFaaLmuKpHTuS0w09PTMXnyZLsylUqFyMhIZGRk1HvchQsXUFBQAD8/Pzz22GM4cuQI1Go1Ro0aheeeew46nc7VTSdyuSJ9Jc7UDK1m65GZV2rrPQb5eqBXlB9iwnSIDdehc7A35DJeeyRyNbdew9RqtQ7lWq0WJSUl9R536dIlAMDLL7+MUaNG4Y033kB2djZWr16NwsJCbN682WVtJnIFo8mC83mlVyfn5Ohtj61SKWTo0skHt98cgdgwHaLDdbylg8hN2tx8cYvF+tT0qKgorFq1yrb2pI+PD5588kkcP34cffr0kVxfQIDGKe0KCvJxSj3U/hWWVOD3c8X47VwRfj9fhPQLJTCZred1sL8X4mMD0TPKHz27+KFrmO66zVzlOUztgSvPY7cFplarhV6vdyjX6/WIjo6u97iaIdfBgwfbLdQ8ePBgAMAff/zRpMAsLCyDxSI2vmMDOMOQ6mMy1/Qe9dW9xxIU6a29R4Vchi6hPhh9U2fEhOsQE66Fr0Ztd3xx0ZXr0k6ew9QeOOM8lsmEejtSbgvMmJgYpKen25UZDAZkZmZi0qRJ9R4XEREBlar+Iamqqqp6txG52uWyKts9j2dySnDuYqmt9xigVSM2XIeYgdbbOiJDNLzvkagNcVtgDh8+HBs2bEBxcTH8/PwAAPv374fBYMCIESPqPU6lUmHo0KE4evQoRFG09TKPHDkCAIiLi3N944lg7T1m5ZdVT8yxhmSh3rreqkIuIKqTD0b1D7eGZLgOfj7qRmokotZMEEWxSeORZWVlOHbsGAoLCzFkyBAEBgY264X1ej0mTpyI8PBwJCUlobCwEC+99BIGDx5st3BBSkoK0tLScOrUKVvZiRMnMG3aNIwZMwaTJk1CTk4OXn31VcTFxTV50g+HZEmqkiuGWqvmlOBcbikMJmvv0c9HjZhwHWLDtNW9R5829ZQOnsPUHrSqIdmNGzfijTfeQEVFBQRBwJYtWxAYGIiioiKMHDkSzz77LO69915JdWm1WmzduhXLly9HcnKybWm8hQsX2u1nsVhgNpvtyuLi4rBp0yasXr0aSUlJ0Gg0mDBhAhYsWNCUt0NUL7PFggv5V2y9xzPZJbandchl1t7jiBvDEROuRWy4Dv5aDze3mIhcTXIP84MPPsDSpUtx9913Y8iQIZg3bx7efvtt22Sb+fPno7CwEFu3bnVpg52NPUwCAH25wTasmp5dgrO5ehiM1t6jTqOyDqtW3/cY1UkDpULu5hY7F89hag9aTQ/z3Xffxbhx47Bs2TIUFxc7bO/Vq1ebC0vqmMwWC7ILrlxdczVbj/zLFQCsvcfIEA2G9wmzzVwN0HrYzcgmoo5JcmBmZWXhwQcfrHe7TqdrcMEBIncpLTcgPUdvu/Z49mIpqozWYX6ttwoxYdZnPcaE69Clkw9UyvbVeyQi55AcmDqdDoWFhfVu/+OPPxAUFOSURhE1l8UiIvtS7d5jCfKKrb1HmSAgIkSDYfGhiKlelDxQx94jEUkjOTBHjBiBHTt24L777nPY9vvvv2Pnzp24++67ndo4osaUVRiRkVOCMzXXHi/qUWmw9h59vJSICdNhWJ9QxIbr0CVUCzV7j0TUTJIn/RQUFGDq1KkwGo249dZb8eGHHyIxMRFmsxkHDhxAp06dsGPHDts9lW0FJ/20HRZRRE517zE9W4/0nBJcLCwHAAgCEBGsqb61w3rtMcjXk71HiXgOU3vg6kk/TboPs+bhzp9//rnteqW3tzfGjh2LBQsW2D2qq61gYLZe5ZVGZOTobUOrGRf1qKiy9h41nkrEVN/zGBOuQ9dQH3io2tzSyK0Gz2FqD1pFYBoMBhw7dgxBQUHo0qULAGt4WiwW+Pv7Q9aGHy3EwGwdLKKIi4Xldk/syLlkXUdVEIDwQA1iO+sQE2a97zHYj71HZ+I5TO1Bq7itRKFQ4KGHHsLixYttgdkWe5PUelRUmex7jzl6lFeZAADeHgrEhOswqFdwde9RC081e49E5F6SfgvJZDKEh4fjypXr8+QEal9EUURuUbntnsf0nBLkFFyBCEAAEBbkjYG9ghFTfe2xk78Xe49E1OpI/rN9xowZ2LJlCyZPnoyAgABXtonauIoqE85e1NuGVtOzS3Cl0tp79FIrEB2uxcAeV3uPXh7sPRJR6yf5N1V5eTk8PT0xZswYjBkzBhEREVCr7Z++IAgCHn74Yac3klovURSRV1xhu/Z4JluP7EtlqLkyHhbojf7dg6yzV8N16BTgBRl7j0TUBkmeJduzZ8/GKxME/Pbbby1u1PXEST9NU2kw4ezFUrvJOWUVRgCAp1qO6LCrE3Oiw7Tw8lC6ucUkRUc6h6n9ahWTfgDgiy++aFEjqO0RRRH5lyvsFiXPKrjaewwN8MKNsYG2VXPCAr3ZeySidktyYIaHh7uyHdQKVBnNOHdRbzc5p7Tc2ntUq+SICdNi4uAuiKnuPWo82Xskoo6jybMtSkpKcPToUWRnZwOwBumQIUOg0+mc3jhyHVEUcamk0u6JHVn5ZbBUdx9D/L3QJzrAtjBAeKA3ZDL2Homo42pSYL711ltYt24dDAYDal/6VKvVSE5O5oSfVsxgNONcbunVgMzRQ3/FAABQK+XoGuqD8QmR1uc+huvYeyQiuobkwNy5cydWr16NwYMH48EHH0RMTAwAID09He+88w5Wr14NX19fTJkyxWWNJWlEUUShvhLp2VcXBsjKL4O5enJTsJ8nbujij9jqa4/hQd6Qt+HVmoiIrgfJs2TvuOMOBAYGYsuWLQ43lYuiiIceegiFhYXYvXu3SxrqKu1hlqzRVNN7tE7MOZNTgpIya+9RpZShayet7WHIMWE6aL1VbmsrtU7uPoeJnKHVzJI9f/48pk2bVucKLIIgYMyYMfj73//e/FaSZEX6SruJOedzS229xyBfD/SK8kNMmPW+x87B7D0SETmD5MDUarXIysqqd3tWVha0Wq1TGkVXGU0WnM+zv++xuLQKAKBUyNC1kw9uHxhhm5yjY++RiMglJAfmyJEjsW3bNvTq1Qt/+tOfbD1NURSxe/duvPfee7jrrrtc1tCOori0qtbEHGvv0WS29h4DtB7oHuFre6xVRLAGCjl7j0RE14Pka5jFxcWYMWMGzpw5A39/f0RGRgIAMjMzUVRUhNjYWLz77rvw9fV1aYOdzZ3XME1mCzLzymwTc9JzSlCkt/YeFXIZuoT62B6GHBOug69G3UiNRM3Da5jUHrSaa5h+fn748MMPsX37dnz11VfIyckBYF0yb+TIkZg6dSpUKg4HNuRyWZVt1Zwz1b1Ho8kCAPDXqhETpsPYgdah1cgQ9h6JiFoTyT3M9spVPUyT2YKs/DK7hQEK9ZUAAIVcQFQnH9vEnJhwHfx82Hsk92EPk9qDVtPDzM/PR2ZmJgYMGFDn9h9++AFRUVEICgpqXivbuJIrhqsTc7JLcC63FIbq3qOfjxoxYVqMHtAZseE6RIb4QKlg75GIqC2RHJh///vfcfHiRbz//vt1bk9NTUVoaChefvllpzWuLfjkm3M4/Gsu8orKAQBymbX3OOLGcMSEW5/a4a/1cG8jiYioxSQH5vfff4/777+/3u233HJLvWHanlVUmREdrsOIvmGICdeiSycfKBVydzeLiIicTHJgFhcXNzgDVqvVorCw0CmNakum3BrD6z9ERB2A5AtpISEhOHnyZL3bT548icDAQKc0ioiIqLWRHJhjxozBRx99hM8//9xh22effYaPP/4YY8aMcWrjiIiIWgvJQ7KPP/44jh49iieffBKxsbHo3r07AOD06dM4c+YMYmNjkZyc7LKGEhERuZPkHqZGo8H27duRlJQEADhw4AAOHDgAwBqmO3bsgI+Pj2taSURE5GZcuKAdPN6LqKV4DlN74OqFC1p093xubi6OHz+OkpKSllRDRETU6jUYmMeOHcO6detQVFRkV15QUIDp06dj5MiRuOeeezB06FC88sorLm0oERGROzUYmB988AF2794Nf39/u/LFixfj+++/x4ABAzBz5kzExsZiy5YtSEtLc2ljiYiI3KXBWbK//PILhg8fbleWmZmJw4cPY9iwYdi0aRMAwGg0YsqUKdi1axf+/Oc/u661RERE1URRtD2b2aLPB4JcO/G0wcAsKChAly5d7Mr+85//QBAE3HvvvbYypVKJxMREW4ASERE1RBQtgGiBIFNArCyDuTgbMFRANJRDNFRANFRApguGMvpmWCpLUfmfTUBVzTbrd0HtBc19qwEAFQffAGJcu5Z5g4FZk9y1/fzzzwCAfv362ZUHBgaioqLCiU0jIqLWSLRYAFMlBJUXAMCcnwGxosQuzGCogDL+dsi8fGE4cQDG9G+rA9G6D4yVUA+dDtUNt8GUcwqVB9Y7vI6i6wAoo2+GIMggll+GoPKCTBcMqDwhqLwgeOps+6oHTnb5+24wMCMjI/HTTz/ZFl03mUz47rvvEBUV5XBds6ioyKGMiIhaF9FiBgwVgEwOQeUJS4Uelrz0Wj0763eZNgiq3qMgGqtQ/snLgLHC1vODsRJQesDnoY0AgMrDW2G5dN7+hWQKKGJuBrx8AYgQ5EoIOl112FkDTx7UFQAg79QDnhMWVpd72gIRciUAQFB7w3vS0gbflyK8t9M/K4fXaGjj5MmTsWLFCkRHR+Omm27Cnj17UFxcjOnTpzvs+8MPP6Br164ua2hrZSm/jIK976PSJECQKwGFCoJCDcFLB2VsAgDAlH0KEAQICnX1dpX1u6cWgsDnYhKRNDVhVxNoAGDK+R1iWaFd4MFQAVXfCZDpQqy9u1NfXA07UxUAQDVwMtT97oClMBMVn6fav5BcCUVEH6D3KECuhKBUQ/DyBVRetlAT1F623T2GPwSIojXslNXbFSrbdlXcGKji6l86Vealg8xLV+/21qLBwLznnntw+PBhrF27FoIgQBRFDBo0CLNmzbLbLycnB4cPH8a8efNc2tjWSKy6gvLT/4XZUGk9EavXgZD5d7YFZsX+dYCh3OFYzYx1gIcGFQfWw5T169Ugrf6uHjQVirBeMGUdh/GPbxwCVx4SC0V4b4hVV2DKPlXreDWgUFqHLzQB1nZaTBBkkldCJCInE80mW5hBqYbMyxeWCj3MWcevGcosh+DtD/VNf4YoiijfmQKxqhyisQIwGWz1aR7eAkEmg+Gnf8Gc89vVF5KrIKg8oeg+DDJdCAQPDWR+4dU9Ny/rUKbKE/JO3ay7B0fD667n7Xp2gvzq7wpBJoNX4qIG35s8sItTP6vWqsHfoAqFAhs2bMCvv/6KrKwshIeHo2/fvg77GY1GrF69GgMHDnRZQ1sruV84op7ajIKCUoiiCFjMgKnKekG7mteEBRBNVdZykwEwGa0/K60PllZE9rH+9WYyVO9ngGg2WHusAMQrl2HOO1PreAMgWqDsMw6K8N6w6AtQeeB1h7bJAqLgPdk6jFG29QnrcbbAVUNQqOD1579CUHqg6sd/wVKYWStwrfspuw+FzDcU5qIsWC5l2veQFWoI3n6QeftZ//I1VVnLGczUzohmoy3MREMFZIFREAQZTFm/wnL5Yq2enXW76saJkAd1geHUQRh+TLMeazba6lP1nQD1oKkQSy9ZJ7PUUKisQ5UhsQCs80hkgV0gKJS1enfW74D1j3OP4Q9VV1rds7vm358yNsH2x3tdrEOjXZzyObV3XBqvjS6NJ1pM1iEQuRKiyQCLPq86cA2274JCDUVkHwBA1c97AGNlre3WYPYY/RcIMgUqj7wLc87vdsfDVAXPcU9BEdEHVT/vgeH7XQ7tqPmHb85PR3naMmuhILcFqzwkBp63zwUAlO9dBQgyu6Hrmp60IFfAeO5HiOUljj3pgEgIam/rX+Amw9XAZjA7TXteGk80GWyTUERDOQS1N2S6EFgqS2H646i191YzXGkoh+CphcctDwIAyrY/A/FKIWA22dWpefB1CGpvVHz+GkznfrIWKtS2QFMPfQCK8N4wZf0K09kfrvbcqr/L/DtDHhhlbVv1ZBaoPHhOt5Crl8bj/502qvY/LEGhgtw/osH91f0mNrjdY6jjdWnAep8TAKhuuA3KmJsdAlfwsT4DVfD2hzrh3lo9ZCNgqoLg7X+1HosZotEaerb9TAaoE6YBAIwnDtgPLVXzTFwERXhvGE4cgOGHj2p/CIBCBdWNiVD3uwPmS+dR+fU/rgaq3Ppd5t8Z6hsTAVj/cBDkCvuha4UK8og+EAQZLKUF1mF1udLWE4dMXueMcXItURQBs9E2jFnTi5OH9oAgV8J0/meYC87ZzcoUDeVQ3TgRis43wPj7IVQefgew2Iedsuet8Bg+E6gsQ9U3H1QXetiuzclkV+cVKLveVGv71cCrmYyiHv4QPIbPsgaiTO7wHhQR8VBExNf7HgWFCoI2uGUfFF03DExqUE1Q2Gav1UPm7QdVn7EN1uM18ZkGX8tz7JMQjZUOPWV5gPWPAUVkXwgePtcMXRshD4iseZHq7VUQK8uqg9lgGwoTLZY6e8mAAM2cLQCAyv9sgvni/67ZLIPnxGegCO0Bw6+fw/j7f2xD2jXBrIhNgDJ6ICwleTD+7+vqQFba9hM0AVCE9YIoirDkp9sfXzNELm8//xytYWewm4RivW9OA3lQF4hVV2A4vg+isdJ+u9IDnmOfBABc2b4IYmmBQ93e016BoA2C8eyPMJ0+bBdmUHkCohmAdR6BKv726qHMWoGnDQEACNoQaB583TpJRVb35Dv1oKkNvk+ZB5/Q1JG0n3+h1OYJSg8I1dd16yIPjII8MKr+7QGR8Bo/v/76ZTJoHt7kEMgwGWx/GKj63wnxSrFDT7pm8pTgqYXMN+zq9sorEE3FECutw0CW0gIYjn9qvZZdu22d46AI6wVYTCj/13LHxsnk8Hl4MwCgfN8aWIqzrUEqrzUJbPC9kPuFw3juR5hzfncIXHlILOQBkbBUlsJSmFXrWnV1cFf3oqQQTVW2oUrUupFcEREPQeUJ49kfYc45BdFQabdddWMilDE3w5T+LSoPvuFQryL6ZniOToJoNsHw855aPbvqwPNU2/ZVxY2GaDZat1WHIlSeEKpnU3oMmwEMn1Vv2MmDoyEPjq73PQoyGaD2lvR5EAEMTOpgBJkCUClsN1xfq7F7uRqbQKHoHAefhzdbrzHXTO4yGYCa4TpBBs9x8+2GpGE2AJark8TkIbEQ1Bq7SV5iVXnNHA9YLmXCePqwtd5awawedI81MPPTUbHv/xzaJo/oA6/x1tcue2eu3eStKpUSJjNsk8TKP14GS/EFhzq8Ji2FPDAK5tzTMP5xtDroqntu3r4QlNZbCWSBUVDdfLd9GKq8IPP2tX4Mnlpo5mxu8LYqVXz9IxYA7G5bILoeOOmnjU76IQJQK5itk6EElad1mbGiC9Yh0Vo9aZmXLxRRN0I0VaHq+4/sJnepFAIMghqetz4MADD+cRSiscox8LRBDCpqtVw96YeBycAk4jlM7UKrfoB0bV9++SUWL17srOqIiIhaFacF5u+//87nYRIRUbvFhUyJiIgkaHCWbF3L4NXHbDY3vhMREVEb1WBgmkwmdO7cGf3792+0ov/973/47TfHVVqIiIjagwYDMzY2Fp6enli5cmWjFW3YsIGBSURE7VaD1zDj4uLw22+/wWQyNbQbERFRu9dgD3PChAmwWCwoLi5GUFBQgxWNGjUKnTp1cmrjiIiIWgsuXMCFC4h4DlO70GYWLiAiImrPGgzM5cuX48SJE3ZlFRUV6OCdUiIi6oAaDMxt27bh7Nmztp+Li4vRv39/fPvtty5vGBERUWvS5CFZ9i6JiKgj4jVMIiIiCdwamOfOncPs2bPRr18/JCQkYNmyZaioqGhSHfv370ePHj0wceJEF7WSiIiokfswASArKwvHjx8HAJSWWqfrZmRkwNvbu879+/TpI+mF9Xo9ZsyYgbCwMKSmpqKoqAgrV65EUVER1qxZI6mOiooKrFixAoGBgZL2JyIiaq5GA3Pt2rVYu3atXdny5csd9hNFEYIgSF4eb/v27dDr9UhLS4O/vz8AQC6XY8GCBUhKSkK3bt0arWP9+vXo3LkzwsPDHWbzEhEROVODgSllDdnmOnToEBISEmxhCQBjx45FSkoKDh061Ghgpqen491338WOHTuwZcsWl7WTiIgIaCQw77rrLpe9cHp6OiZPnmxXplKpEBkZiYyMjEaPf+GFFzBlyhR0797dVU0kIiKyaXRI1lX0ej20Wq1DuVarRUlJSYPHfvLJJzh9+rTDUHFz1LcEUlMFBfk4pR4id+E5TO2BK89jtwVmc5WVleGll17C/Pnz6wzcpuJaskQ8h6l9aLdryWq1Wuj1eodyvV4PnU5X73EbN26Er68vxowZA71eD71eD6PRCIvFAr1eD4PB4MpmExFRB+W2HmZMTAzS09PtygwGAzIzMzFp0qR6j8vIyMDp06cxaNAgh20DBw7E4sWLMXPmTGc3l4iIOji3Bebw4cOxYcMGFBcXw8/PD4B1EQKDwYARI0bUe9xTTz2FBx980K7szTffxNmzZ7Fy5UpERUW5tN1ERNQxuS0wp02bhm3btiEpKQlJSUkoLCzESy+9hAkTJiA2Nta2X0pKCtLS0nDq1CkAqHNW7Mcff4y8vLw6e51ERETO4LbA1Gq12Lp1K5YvX47k5GSo1WokJiZi4cKFdvtZLBaYzWY3tZKIiMhKEDv440c4S5aI5zC1D+12liwREVFbwsAkIiKSgIFJREQkAQOTiIhIAgYmERGRBAxMIiIiCRiYREREEjAwiYiIJGBgEhERScDAJCIikoCBSUREJAEDk4iISAIGJhERkQQMTCIiIgkYmERERBIwMImIiCRgYBIREUnAwCQiIpKAgUlERCQBA5OIiEgCBiYREZEEDEwiIiIJGJhEREQSMDCJiIgkYGASERFJwMAkIiKSgIFJREQkAQOTiIhIAgYmERGRBAxMIiIiCRiYREREEjAwiYiIJGBgEhERScDAJCIikoCBSUREJAEDk4iISAIGJhERkQQMTCIiIgkYmERERBIwMImIiCRgYBIREUnAwCQiIpKAgUlERCQBA5OIiEgCBiYREZEEDEwiIiIJGJhEREQSMDCJiIgkYGASERFJwMAkIiKSgIFJREQkAQOTiIhIAgYmERGRBAxMIiIiCRiYREREEjAwiYiIJGBgEhERScDAJCIikoCBSUREJAEDk4iISAIGJhERkQQMTCIiIgkYmERERBIo3Pni586dw7Jly/DTTz9BrVYjMTERCxYsgKenZ73HlJWV4e2338ahQ4dw9uxZKBQK3HDDDZg/fz5uuOGG69h6IiLqSNzWw9Tr9ZgxYwauXLmC1NRUPPvss9izZw9SUlIaPC4nJwf//Oc/MWTIEKxZswYrV66ExWLBtGnTcPLkyevUeiIi6mjc1sPcvn079Ho90tLS4O/vDwCQy+VYsGABkpKS0K1btzqP69y5M/bv32/XCx0yZAhuu+02bNu2DStXrrwu7Scioo7FbT3MQ4cOISEhwRaWADB27FioVCocOnSo3uO8vLwchmzVajViYmKQn5/vsvYSEVHH5rbATE9PR2xsrF2ZSqVCZGQkMjIymlRXeXk5fvvtN0RHRzuziURERDZuG5LV6/XQarUO5VqtFiUlJU2q6//+7/9QUVGBBx54oMntCAjQNPmYugQF+TilHiJ34TlM7YErz2O3zpJ1ht27d2Pr1q3429/+hqioqCYfX1hYBotFbFEbgoJ8UFBQ2qI6iNyJ5zC1B844j2Uyod6OlNuGZLVaLfR6vUO5Xq+HTqeTVMeRI0ewePFizJ49G/fff7+zm0hERGTjtsCMiYlBenq6XZnBYEBmZqaka5HHjx/HE088gfHjx2PhwoWuaiYREREANwbm8OHD8e2336K4uNhWtn//fhgMBowYMaLBY9PT0zFnzhz0798fK1asgCAIrm4uERF1cG4LzGnTpsHHxwdJSUn4+uuvkZaWhmXLlmHChAl2s2dTUlLQu3dv28+FhYWYPXs2lEolHn74YZw8eRK//PILfvnlF5w6dcodb4WIiDoAt0360Wq12Lp1K5YvX47k5GTb0njXDq9aLBaYzWbbz2fOnMHFixcBADNnzrTbNzw8HAcPHnR524mIqOMRRFFs2RTRNo6zZIl4DlP70G5nyRIREbUlDEwiIiIJGJhEREQSMDCJiIgkYGASERFJwMAkIiKSgIFJREQkAQOTiIhIAgYmERGRBAxMIiIiCRiYREREEjAwiYiIJGBgEhERScDAJCIikoCBSUREJAEDk4iISAIGJhERkQQMTCIiIgkYmERERBIwMImIiCRgYBIREUnAwCQiIpKAgUlERCQBA5OIiEgCBiYREZEEDEwiIiIJGJhEREQSMDCJiIgkYGASERFJwMAkIiKSgIFJREQkAQOTiIhIAgYmERGRBAxMIiIiCRiY4QQq9wAADx1JREFUREREEjAwiYiIJGBgEhERScDAJCIikoCBSUREJAEDk4iISAIGJhERkQQMTCIiIgkYmERERBIwMImIiCRgYBIREUnAwCQiIpKAgUlERCQBA5OIiEgCBiYREZEEDEwiIiIJGJhEREQSMDCJiIgkYGASERFJwMAkIiKSgIFJREQkAQOTiIhIAgYmERGRBAxMIiIiCRiYREREEjAwiYiIJGBgEhERScDAJCIikoCBSUREJIFbA/PcuXOYPXs2+vXrh4SEBCxbtgwVFRWSjk1LS8O4ceMQHx+PxMRE7N2718WtJSKijkzhrhfW6/WYMWMGwsLCkJqaiqKiIqxcuRJFRUVYs2ZNg8fu27cPzzzzDB555BEMHToUBw4cwPz58+Ht7Y0RI0Zcp3dAREQdidsCc/v27dDr9UhLS4O/vz8AQC6XY8GCBUhKSkK3bt3qPTY1NRXjxo3D008/DQBISEhARkYG1q5dy8AkIiKXcNuQ7KFDh5CQkGALSwAYO3YsVCoVDh06VO9xWVlZyMjIQGJiol35xIkT8euvv6KoqMhlbSYioo7LbT3M9PR0TJ482a5MpVIhMjISGRkZ9R5Xsy0mJsauPDY21ra9dgg3RiYTJO97Peohcheew9QetPQ8buh4t17D1Gq1DuVarRYlJSX1Hlez7dpjdTqd3Xap/Py8m7R/fQICNE6ph8hdeA5Te+DK85i3lRAREUngtsDUarXQ6/UO5Xq93tZbrEvNtmuPrelZNnQsERFRc7ktMGNiYpCenm5XZjAYkJmZiejo6HqPq9l27XXOmroaOpaIiKi53BaYw4cPx7fffovi4mJb2f79+2EwGBq8NSQiIgLR0dEOCxXs2bMH8fHxTZrwQ0REJJXbAnPatGnw8fFBUlISvv76a6SlpWHZsmWYMGGCbcYrAKSkpKB37952x86dOxeffvop1qxZg++++w4rVqzAkSNHkJycfL3fBhERdRBumyWr1WqxdetWLF++HMnJyVCr1UhMTMTChQvt9rNYLDCbzXZl48ePR2VlJTZu3IjNmzcjMjISq1ev5qIFRETkMoIoiqK7G0FERNTa8bYSIiIiCf5/e/cfU3X1/wH8qXcCiagBTn5cRcnulbhkcxBgoXALSJPISziFFIhQynVrawZrQWg5lmY00dFqN4Y3k/gVixZZCGoWOfAHU0S6BXgBXRPu+KVgcO/7+4fj/e0G6r3Ix5v5fGx3433O2Tmv9/sPXu9z3r+YMImIiCxgs2uY97qLFy9Co9GgoaEBOp0O3t7e+Pbbb20dFpHFKisrUVFRgcbGRvT29mLevHlYv3491q1bh6lTeS5N/34//PAD8vPz0dLSgmvXrmHu3LkIDw/Hq6++Cicnp0kfjwlzgnQ6HY4ePYolS5bAZDKBl4LpXpOfnw8PDw+89dZbcHFxwYkTJ7Bjxw60t7cjLS3N1uER3VZvby8CAgKQlJSEWbNmobm5GXv37kVzczM+//zzSR+PN/1MkMlkEs/C09PTce7cOc4w6Z5iMBjGPLecnZ2NgwcPor6+HnZ2djaKjGjivvrqK2RmZuLYsWOYO3fupPbNdZcJ4pIV3evGe8mHj48Prl+/jp6eHhtERHTnHnzwQQDA8PDwpPfNJVkiEp08eRKzZ8+Gi4uLrUMhspjRaMTIyAh0Oh327dsHpVIJqVQ66eMwYRIRAODs2bMoKyvDli1bIJFIbB0OkcUCAwPR398PAAgJCcHu3bv/J+MwYRIRrly5ArVaDT8/P6SkpNg6HCKraLVaDA4OQqfTIS8vD6mpqcjPz5/0Ez8mTKL7XH9/P1JSUuDg4IC8vDxMmzbN1iERWcXHxwcAsHTpUvj6+iImJgY//vgjnnnmmUkdhwmT6D52/fp1vPLKK+ju7kZhYaF4wwTRvcrHxwdTp06FXq+f9L6ZMInuUyMjI3j99dfR3NwMrVYLT09PW4dEdMdOnz4Nk8nEm37+TQYHB3H06FEAQGdnJwYGBvD9998DAPz8/PjPh/71tm/fjpqaGmzduhVDQ0M4c+aMWLdo0SLMmDHDhtER3V5ycjKCgoLw8MMPw97eHk1NTdBoNJDL5Xj66acnfTy+uGCCOjo68NRTT41bl52dDZVKdZcjIrKOUqlEZ2fnuHX79+9HYGDgXY6IyDoff/wxDh8+jI6ODgCAVCpFREQEkpKS/icnfEyYREREFuDraoiIiCzAhElERGQBJkwiIiILMGESERFZgAmTiIjIAkyYREREFmDCJPqPam9vx8svvwx/f3/I5XJUVVXZOqQJk8vlyMzMtHUYdJ9jwiSyUFlZGeRyORQKBS5dujSmfvPmzVAqlTaIbHwZGRlobGyEWq3Gzp07oVAobB0S0T2NCZPISsPDw/jkk09sHcYtmUwm1NfXIyoqChs3bkR0dDTc3NxsHRbRPY0Jk8hKPj4+KCsrG3eW+W/R19eH4eFhODk52ToUov8MJkwiK23atAkALJplGo1G5OXlITw8HAqFAqGhodi5cyeGhoYmPP6FCxeQkpKCpUuX4rHHHsOGDRtQX18v1ufm5orvgd27dy/kcvltl4oFQYBWq0VUVBT8/PwQHByMt99+GwaDwaydUqlEcnIyamtroVKp4Ofnh8jISJSXl4/p02AwICMjA0888QT8/PywevVqFBUVjTv2gQMHEB0djUcffRSBgYFISkoy26dRVVVVWL16NRQKBZ599lkcO3bMrP7q1av44IMPoFQqoVAoEBQUhA0bNqCuru6W+09kCX6thMhKHh4eiImJQWlpKVJTU+Hh4XHTtpmZmSgpKUFERAQSExNx7tw5aDQa6HQ6fPrpp5gyZYpVY//xxx+Ii4vD9OnTkZycDHt7exQXFyMxMRH5+fkICAhAeHg4XF1dkZWVhfDwcISHh8PR0fGW/b777rsoLS3F888/j/j4eFy+fBlffPEFzp49i5KSEtjb24ttOzo6oFarERsbi+joaFRUVCAtLQ12dnZYtWoVgBvf2UxISEBLSwvi4uIwf/58VFVVISMjAz09PeJJB3DjWmtxcTGefPJJrFmzBgBw6tQp1NXVwd/fX2x35swZ1NTUYP369XB0dIRWq4VarUZNTY34Hc+srCxUVlYiPj4eixYtQl9fHxoaGnDhwgUEBARYdayJxhCIyCKlpaWCTCYTTp8+LVy6dEnw9fUVMjIyxPpNmzYJYWFh4nZTU5Mgk8mE9PR0s3727NkjyGQyobq62uoYtmzZIvj6+gqtra1iWXd3t/D4448La9asMSuTyWTCnj17btvnyZMnBZlMJnz99ddm5XV1dYJMJhMKCwvFsrCwMEEmkwkVFRVi2eDgoBAZGSmsWLFCMBqNgiAIQkFBgSCTyYSysjKx3cjIiJCQkCAoFArBYDAIgiAIv/76qyCTyYSsrKwxcZlMJvFvmUwm+Pr6Cm1tbWLZ6PHVarVimb+/v7Bt27bb7jPRRHBJlmgC3N3dERMTg7Kyspt+Imv0e6mJiYlm5YmJiZBIJDhy5IhVYxqNRhw/fhxhYWFYsGCBWO7s7AyVSoXGxkZ0dXVZ1ScAVFZWYvr06QgJCYHBYBB/3t7ecHV1xYkTJ8zau7i4iDNJAHBwcEBsbCwuX76M5uZmADf23dnZGc8995zYTiKRICEhAX/99Rdqa2sBAIcOHQIAvPbaa2Pi+ufsOzAwEF5eXuL24sWLMWPGDLS3t4tlTk5OaGhowJ9//mn1cSC6HSZMoglKTU0FcPNrmZ2dnZgyZQoWLlxoVu7k5IQ5c+bcNNHejMFgwODg4Jj+AMDb21sc01ptbW24du0ali1bhuDgYLNfV1cXuru7zdrPnz8fU6ea/+sYTeCj43d2dsLLywsSicSs3UMPPQQA4vcL9Xo9XF1d4ezsfNs43d3dx5TNmjULfX194vbWrVvx+++/IzQ0FCqVCjk5OWhpablt30SW4DVMoglyd3fHCy+8gJKSEjF53otMJhNmz56NnJyccetnzpx5lyMa3z+T9Cjhb5/0XblyJfz9/XH48GH8/PPP0Gq10Gg0yM7ORlRU1N0Klf6jOMMkugOjiTIvL29MnaenJwRBQGtrq1n5wMAArly5Ak9PT6vGcnZ2xgMPPDCmPwDiLMraPoEbM8a+vj4sWbIEy5YtG/P75wsP9Ho9TCaTWVlbW5vZ+J6enrh48SKMRuO4cUqlUnHsrq6uMXfj3ok5c+Zg3bp1yM3NxZEjRyCVSpGbmztp/dP9iwmT6A64ubkhNjYW5eXlY57LXLFiBQCgoKDArLygoABGoxFhYWFimV6vh16vv+VYEokEISEhqKmpMWvb09OD8vJyKBQKuLq6Wr0Pq1atgslkwr59+8bUGY1G9Pb2mpV1d3fju+++E7eHhoZQXFwMNzc3yOVyAEBoaCgMBgMqKirEdiaTCfv374ednR2Cg4MBAJGRkQAwbkL7+8zREkajEf39/WZlM2fOhFQqNVu2JZooLskS3aHNmzejpKQEv/32m9kMb/HixeKS7cDAAAIDA3H+/HmUlpYiJCRETKjA/98YVF1dfcux3njjDRw/fhxxcXGIi4uDvb09ioqK0N/fj/T09AnFHxAQgPj4eGg0GjQ3NyMkJATTpk2DXq/HoUOHoFaroVKpxPYLFizAtm3bcP78ebi5ueGbb75Ba2srPvzwQ3HZdO3atSgqKsI777yDpqYmzJs3D1VVVaitrcWbb74pPgYSGBgIlUqFL7/8Enq9HsuXLwdw4xESuVxu1VL31atXsXz5ckRERIg3BJ06dQo//fQTXnzxxQkdG6K/Y8IkukOjs8wDBw6Mqdu+fTukUilKS0tRXV0NFxcXvPTSS1Cr1VY/gwncuGnm4MGD2L17Nz777DMIggCFQoH333//jp4zzMzMxCOPPILCwkLk5ORAIpHAw8MDK1euRFBQkFlbqVSKrKws7Nq1CzqdDu7u7mOuEdrb26OgoAAfffQRKioq0NfXBy8vL7z33ntYu3atWX87duyAXC5HcXExdu3aBUdHR/j6+lq9Pw4ODoiLi8Mvv/yC6upqjIyMQCqVIi0tDRs3bpzwsSEaNUWwdt2DiO5bSqUSCxcuhEajsXUoRHcdr2ESERFZgAmTiIjIAkyYREREFuA1TCIiIgtwhklERGQBJkwiIiILMGESERFZgAmTiIjIAkyYREREFmDCJCIissD/AUYloLStcTiTAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "DpH5mIv6q16L" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file