bert4news.py

# -*- coding: utf-8 -*-
"""bert4news.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1mhVTtMpJH6jJYpxNfErGycqTg9WxKvWS
"""

import os
path = '/content/drive/MyDrive/VLSP_ReINTEL'
#you should change this path to your project folder path
os.chdir(path)

#download bert4news
# you can create shortcut to your drive from this link
#https://drive.google.com/file/d/11aFSTpYIurn-oI2XpAmcCTccB_AonMOu/view

# you must run this cell first time you run this code
#from zipfile import ZipFile
#zip = ZipFile(path+'/bert4news.pytorch.zip')
#zip.extractall()

! pip install transformers

import torch
from torch import nn
from transformers import *
from torch.nn import CrossEntropyLoss, MSELoss
import torch.nn.functional as F
import pandas as pd
from transformers import BertTokenizer
import pandas as pd
import torch
import re
from sklearn.model_selection import StratifiedKFold
from transformers import get_linear_schedule_with_warmup,get_constant_schedule
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from transformers import get_linear_schedule_with_warmup
import argparse
class BertClassification(BertPreTrainedModel):
    def __init__(self, config):
       super(BertClassification, self).__init__(config)
       self.num_labels = config.num_labels
       self.bert = BertModel(config)
       self.dropout = nn.Dropout(config.hidden_dropout_prob)
       self.classifier = nn.Linear(4*config.hidden_size, self.config.num_labels)
       self.init_weights()

    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None,
                position_ids=None, head_mask=None, inputs_embeds=None, labels=None):

        outputs = self.bert(input_ids,
                            attention_mask=attention_mask,
                            token_type_ids=token_type_ids,
                            position_ids=position_ids,
                            head_mask=head_mask,
                            inputs_embeds=inputs_embeds)

        #pooled_output = outputs[1]

        pooled_output = torch.cat((outputs[2][-1][:,0, ...],outputs[2][-2][:,0, ...], outputs[2][-3][:,0, ...], outputs[2][-4][:,0, ...]),-1)
        pooled_output = self.dropout(pooled_output)
        logits = self.classifier(pooled_output)

        outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here

        if labels is not None:
            if self.num_labels == 1:
                loss_fct = MSELoss()
                loss = loss_fct(logits.view(-1), labels.view(-1))
            else:
                loss_fct = CrossEntropyLoss()
                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
            outputs = (loss,) + outputs

        return outputs  # (loss), logits, (hidden_states), (attentions)

from transformers import BertTokenizer
import pandas as pd
import torch

if torch.cuda.is_available():       
    device = torch.device("cuda")
    print('There are %d GPU(s) available.' % torch.cuda.device_count())
    print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
    print('No GPU available, using the CPU instead.')
    device = torch.device("cpu")

MODEL_PATH = path+'/bert4news.pytorch'
MAX_LEN = 256
batch_size = 32
epochs = 5
lr = 3e-5

#load data
import pandas as pd
train = pd.read_csv(path+'/VSLP_data/public_train (1).csv')
train['post_message'] = train['post_message'].fillna('none')
public_test = pd.read_csv(path+'/VSLP_data/public_test.csv')
test = pd.read_csv(path+'/VSLP_data/final_private_test_dropped_no_label - final_private_test_dropped_no_label.csv')

sentences = train.post_message.values
labels = train.label.values

print('Loading BERT tokenizer...')
tokenizer = BertTokenizer.from_pretrained(MODEL_PATH, do_lower_case=False)
input_ids = []

for sent in sentences:
    encoded_sent = tokenizer.encode(
                        sent,                      # Sentence to encode.
                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'
                        #return_tensors = 'pt',     # Return pytorch tensors.
                   )
    input_ids.append(encoded_sent)
from keras.preprocessing.sequence import pad_sequences
input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype="long", 
                          value=0, truncating="post", padding="post")
attention_masks = []
for sent in input_ids:
    att_mask = [int(token_id > 0) for token_id in sent]
    attention_masks.append(att_mask)
from sklearn.model_selection import train_test_split

# Use 80% for training and 20% for validation.
train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels, 
                                                            random_state=42, test_size=0.2)
# Do the same for the masks.
train_masks, validation_masks, _, _ = train_test_split(attention_masks, labels,
                                             random_state=42, test_size=0.2)

train_inputs = torch.tensor(train_inputs,dtype=torch.long)
validation_inputs = torch.tensor(validation_inputs,dtype=torch.long)

train_labels = torch.tensor(train_labels,dtype=torch.long)
validation_labels = torch.tensor(validation_labels,dtype=torch.long)

train_masks = torch.tensor(train_masks,dtype=torch.long)
validation_masks = torch.tensor(validation_masks,dtype=torch.long)

from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler

# Create the DataLoader for our training set.
train_data = TensorDataset(train_inputs, train_masks, train_labels)
train_sampler = RandomSampler(train_data)
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)

# Create the DataLoader for our validation set.
validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)
validation_sampler = SequentialSampler(validation_data)
validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)

from transformers import BertForSequenceClassification, AdamW, BertConfig

model = BertForSequenceClassification.from_pretrained(
    MODEL_PATH,
    num_labels = 2,
    output_attentions = False,
    output_hidden_states = True,
)

# Tell pytorch to run this model on the GPU.
if torch.cuda.is_available():
    model.cuda()

# Get all of the model's parameters as a list of tuples.
params = list(model.named_parameters())

print('The BERT model has {:} different named parameters.\n'.format(len(params)))

print('==== Embedding Layer ====\n')

for p in params[0:5]:
    print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))

print('\n==== First Transformer ====\n')

for p in params[5:21]:
    print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))

print('\n==== Output Layer ====\n')

for p in params[-4:]:
    print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))

optimizer = AdamW(model.parameters(),
                  lr = lr,
                  eps = 1e-8
                )

from transformers import get_linear_schedule_with_warmup

# Total number of training steps is number of batches * number of epochs.
total_steps = len(train_dataloader) * epochs

# Create the learning rate scheduler.
scheduler = get_linear_schedule_with_warmup(optimizer, 
                                            num_warmup_steps = 0, # Default value in run_glue.py
                                            num_training_steps = total_steps)

import numpy as np
from sklearn.metrics import f1_score, roc_auc_score
# Function to calculate the accuracy of our predictions vs labels
def flat_auc(preds, labels):
    preds = np.array(preds)
    labels = np.array(labels)
    return roc_auc_score(labels, preds)
def flat_f1(preds, labels):
    pred_flat = np.argmax(preds, axis=1).flatten()
    labels_flat = labels.flatten()
    return f1_score(pred_flat, labels_flat)
import time
import datetime

def format_time(elapsed):
    '''
    Takes a time in seconds and returns a string hh:mm:ss
    '''
    # Round to the nearest second.
    elapsed_rounded = int(round((elapsed)))
    
    # Format as hh:mm:ss
    return str(datetime.timedelta(seconds=elapsed_rounded))

import random

# Set the seed value all over the place to make this reproducible.
seed_val = 42

random.seed(seed_val)
np.random.seed(seed_val)
torch.manual_seed(seed_val)
torch.cuda.manual_seed_all(seed_val)

# Store the average loss after each epoch so we can plot them.
loss_values = []

# For each epoch...
for epoch_i in range(0, epochs):

    print("")
    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
    print('Training...')

    t0 = time.time()

    # Reset the total loss for this epoch.
    total_loss = 0
    model.train()

    # For each batch of training data...
    for step, batch in enumerate(train_dataloader):

        # Progress update every 40 batches.
        if step % 40 == 0 and not step == 0:
            # Calculate elapsed time in minutes.
            elapsed = format_time(time.time() - t0)
            
            # Report progress.
            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))

        b_input_ids = batch[0].to(device)
        b_input_mask = batch[1].to(device)
        b_labels = batch[2].to(device)

        model.zero_grad() 

        outputs = model(input_ids=b_input_ids, 
                    token_type_ids=None, 
                    attention_mask=b_input_mask, 
                    inputs_embeds=None,
                    labels=b_labels)
    
        loss = outputs[0]

        total_loss += loss.item()
        loss.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
        optimizer.step()
        scheduler.step()

    # Calculate the average loss over the training data.
    avg_train_loss = total_loss / len(train_dataloader)            
    
    # Store the loss value for plotting the learning curve.
    loss_values.append(avg_train_loss)

    print("")
    print("  Average training loss: {0:.2f}".format(avg_train_loss))
    print("  Training epcoh took: {:}".format(format_time(time.time() - t0)))
        
    # ========================================
    #               Validation
    # ========================================
    # After the completion of each training epoch, measure our performance on
    # our validation set.

    print("")
    print("Running Validation...")

    t0 = time.time()

    model.eval()

    # Tracking variables 
    eval_loss, eval_accuracy = 0, 0
    nb_eval_steps, nb_eval_examples = 0, 0

    # Evaluate data for one epoch
    predictions = []
    labels_gold = []
    for batch in validation_dataloader:
        
        # Add batch to GPU
        batch = tuple(t.to(device) for t in batch)
        
        # Unpack the inputs from our dataloader
        b_input_ids, b_input_mask, b_labels = batch
        
        # Telling the model not to compute or store gradients, saving memory and
        # speeding up validation
        with torch.no_grad():        
            outputs = model(b_input_ids, 
                            token_type_ids=None, 
                            attention_mask=b_input_mask)
        
        logits = outputs[0]

        # Move logits and labels to CPU
        logits = F.softmax(logits,dim=1)
        logits = logits.detach().cpu().numpy()
        predictions.append(logits)
        label_ids = b_labels.to('cpu').numpy()
        for l in label_ids:
            labels_gold.append(l)
           # Calculate the accuracy for this batch of test sentences.
        tmp_eval_accuracy = flat_f1(logits, label_ids)
        
        # Accumulate the total accuracy.
        eval_accuracy += tmp_eval_accuracy

        # Track the number of batches
        nb_eval_steps += 1

    flat_predictions = [item[1] for sublist in predictions for item in sublist]
    roc_score = flat_auc(flat_predictions, labels_gold)    
     
    # Report the final accuracy for this validation run.
    print("  F1 score: {0:.4f}".format(eval_accuracy/nb_eval_steps))
    print("  ROC score: {0:.4f}".format(roc_score))  
    print("  Validation took: {:}".format(format_time(time.time() - t0)))
    # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()


    output_dir = os.path.join(path+'/bert4news_model', 'checkpoint-{}-{}-{}'.format(lr,eval_accuracy/nb_eval_steps,roc_score))

    # Create output directory if needed
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    print("Saving model to %s" % output_dir)

    # Save a trained model, configuration and tokenizer using `save_pretrained()`.
    # They can then be reloaded using `from_pretrained()`
    model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training
    model_to_save.save_pretrained(output_dir)
    tokenizer.save_pretrained(output_dir)

print("")
print("Training complete!")

id_test = test.id.values.tolist()
sentences = test.post_message.values
print('Loading BERT tokenizer...')
tokenizer = BertTokenizer.from_pretrained(MODEL_PATH, do_lower_case=False)

MODEL_PATH = path+'/bert4news_model'

# Tokenize all of the sentences and map the tokens to thier word IDs.
input_ids = []
# For every sentence...
for sent in sentences:
    try:
        if(len(sent)==0):
            sent=''
            print(sent)
    except:
        sent= ''
        print(sent)
    encoded_sent = tokenizer.encode(
                        sent,                      # Sentence to encode.
                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'
                   )
    
    input_ids.append(encoded_sent)

from keras.preprocessing.sequence import pad_sequences
# Pad our input tokens
input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, 
                          dtype="long", truncating="post", padding="post")

# Create attention masks
attention_masks = []

# Create a mask of 1s for each token followed by 0s for padding
for seq in input_ids:
  seq_mask = [float(i>0) for i in seq]
  attention_masks.append(seq_mask) 

# Convert to tensors.
prediction_inputs = torch.tensor(input_ids,dtype=torch.long)
prediction_masks = torch.tensor(attention_masks,dtype=torch.long)


# Create the DataLoader.
prediction_data = TensorDataset(prediction_inputs, prediction_masks)
prediction_sampler = SequentialSampler(prediction_data)
prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)

# Prediction on test set
print('Predicting labels for {:,} test sentences...'.format(len(prediction_inputs)))

# load_moel
from transformers import BertForSequenceClassification, AdamW, BertConfig
import torch.nn.functional as F
import os

list_model = os.listdir(MODEL_PATH)
list_predictions = []
for model_name in list_model:
    model = BertForSequenceClassification.from_pretrained(
        os.path.join(MODEL_PATH,model_name),
        num_labels = 2,
        output_attentions = False,
        output_hidden_states = False
    )
    if torch.cuda.is_available():
        model.cuda()
    model.eval()

    # Tracking variables 
    predictions= []
    # Predict 
    for batch in prediction_dataloader:
        batch = tuple(t.to(device) for t in batch)
        b_input_ids, b_input_mask = batch
        with torch.no_grad():
            outputs = model(b_input_ids, token_type_ids=None, 
                            attention_mask=b_input_mask)

        logits = outputs[0]
        logits = F.softmax(logits,dim=1)
        logits = logits.detach().cpu().numpy()  
        predictions.append(logits)
    flat_predictions = [item for sublist in predictions for item in sublist]
    list_predictions.append(flat_predictions)

list_predictions = np.asarray(list_predictions)

list_predictions = np.mean(list_predictions,axis=0)

re = pd.DataFrame(list_predictions)
result = pd.DataFrame({'id':test['id'],'pro':re[1]})
result.to_csv('result.csv',index=False,header=False)