Skip to content

Commit 7cd20d2

Browse files
antonis19antonis19
authored
Add files via upload
0 parents  commit 7cd20d2

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+20409
-0
lines changed

‎IMDBModel.py

+215
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,215 @@
1+
import numpy as np
2+
3+
from keras.preprocessing import sequence
4+
from keras.models import Sequential
5+
from keras.layers import Dense, Embedding, LSTM
6+
from keras.models import Sequential, Model, Input, load_model
7+
from keras.datasets import imdb
8+
from glove_utils import load_embedding
9+
import pickle
10+
import time
11+
import tensorflow as tf
12+
from data_utils import IMDBDataset
13+
from embedding import Embedding
14+
import utils
15+
from itertools import dropwhile
16+
17+
18+
class IMDBModel:
19+
'''
20+
Class representing a model trained on the IMDB sentiment analysis dataset.
21+
'''
22+
def __init__(self,model_filename : str, embedding: Embedding, maxlen = 200):
23+
'''
24+
Create an IMDBModel class.
25+
26+
Parameters
27+
-------------
28+
model_filename: str
29+
the path containing the underlying Keras model.
30+
31+
embedding: Embedding
32+
the embedding space used by the model.
33+
34+
maxlen: int
35+
the maximum number of words to pad text to.
36+
37+
'''
38+
self.model = load_model(model_filename)
39+
self.embedding_model = self.create_embedding_model(self.model)
40+
self.embedding = embedding
41+
self.word2index = embedding.word2index
42+
self.index2word = embedding.index2word
43+
self.index2embedding = embedding.index2embedding
44+
self.maxlen = maxlen
45+
46+
47+
def sequence_to_embedding(self,seq):
48+
'''
49+
Convert sequence of word indexes (rows in embedding matrix) to matrix of embeddings.
50+
'''
51+
return np.array([self.index2embedding[index] for index in seq])
52+
53+
54+
def words_to_sequence(self, words) :
55+
'''
56+
Convert list of words to sequence of word indexes.
57+
'''
58+
return np.array([self.word2index[word] for word in words])
59+
60+
def sequence_to_words(self,seq):
61+
'''
62+
Convert sequence of word indexes to words.
63+
'''
64+
return [self.index2word[idx] for idx in self.unpad_sequence(seq)]
65+
66+
def seq2text(self,seq):
67+
'''
68+
Convert sequence of word indexes to text.
69+
'''
70+
return " ".join(self.sequence_to_words(seq))
71+
72+
def text2seq(self,text, clean_text = True):
73+
'''
74+
Convert text to sequence of word indexes.
75+
'''
76+
text = IMDBDataset.clean_text(text)
77+
sample_indexes = IMDBDataset.text_to_index(text, self.word2index)
78+
sample_indexes = sequence.pad_sequences([sample_indexes], maxlen=self.maxlen, padding = 'pre', truncating = 'pre').squeeze()
79+
return sample_indexes
80+
81+
def model_predict(self,x) :
82+
'''
83+
Predict probability of positive sentiment from list of word indexes.
84+
85+
Parameters
86+
------------
87+
x: list
88+
list of word indexes (rows in embedding matrix) representing the text.
89+
90+
Returns
91+
-------------
92+
out: float
93+
probability of positive sentiment.
94+
'''
95+
out = self.model.predict(np.expand_dims(x, axis=0))[0][0]
96+
return out
97+
98+
def model_predict_class(self,x) :
99+
'''
100+
Predict the class 1(positive)/ 0(negative) from list of word indexes.
101+
'''
102+
out = self.model.predict_classes(np.expand_dims(x, axis=0))[0][0]
103+
return out
104+
105+
def predict(self,text):
106+
'''
107+
Predict probability of positive sentiment from text.
108+
109+
Parameters
110+
------------
111+
text: str
112+
The text whose sentiment we want to predict.
113+
114+
Returns
115+
-------------
116+
out: float
117+
probability of positive sentiment.
118+
'''
119+
indexes = self.text2seq(text)
120+
return self.model_predict(indexes)
121+
122+
def predict_class(self, text):
123+
'''
124+
Predict the class of a given text.
125+
Parameters
126+
------------
127+
text: str
128+
The text whose sentiment we want to predict.
129+
130+
Returns
131+
-------------
132+
0 (negative) or 1 (positive)
133+
'''
134+
prediction = self.predict(text)
135+
if prediction < 0.5 :
136+
return 0
137+
else :
138+
return 1
139+
140+
def preprocess_text(self, text):
141+
'''
142+
Preprocess text by cleaning it and padding to maximum length.
143+
'''
144+
text = IMDBDataset.clean_text(text)
145+
tokens = tf.keras.preprocessing.text.text_to_word_sequence(text, lower=False, split=' ', filters='\t\n')
146+
return ' '.join(tokens[-self.maxlen:])
147+
148+
def unpad_sequence(self, seq, pad_char = 0):
149+
'''
150+
Remove pad characters from sequence.
151+
'''
152+
return np.array(list(dropwhile(lambda index: index == pad_char, seq)))
153+
154+
def pad_sequence(self, seq, pad_char = 0):
155+
'''
156+
Pad sequence to maximum length.
157+
'''
158+
return sequence.pad_sequences([seq], maxlen = self.maxlen, padding = 'pre', truncating= 'pre').squeeze()
159+
160+
def create_embedding_model(self,model):
161+
'''
162+
Get submodel that takes word embeddings as input (instead of word indexes)
163+
164+
Parameters
165+
-----------
166+
model
167+
The underlying Keras model.
168+
169+
Returns
170+
-------------
171+
embedding_model
172+
The Keras model that takes a sequence of word vectors (instead of word indexes) as input and produces
173+
the probability of positive sentiment.
174+
'''
175+
# Create submodel that takes word embeddings as input, instead of discrete word indexes
176+
embedding_input = Input(shape = [None, None])
177+
prediction_layer = embedding_input
178+
for layer in model.layers[1:]:
179+
prediction_layer = layer(prediction_layer) # append layer
180+
embedding_model= Model(inputs= embedding_input, outputs = prediction_layer) # create the submodel
181+
return embedding_model
182+
183+
def embedding_model_predict(self, word_embeddings):
184+
'''
185+
Get the output of the model given a sequence of word embeddings as input.
186+
187+
Parameters
188+
-------------------
189+
word_embeddings:
190+
a sequence of word vectors corresponding to the word embeddings of the sequence of words in the text
191+
192+
Returns
193+
------------------
194+
Probability of positive sentiment (float)
195+
'''
196+
return self.embedding_model.predict(np.expand_dims(word_embeddings, axis=0))[0][0]
197+
198+
199+
200+
if __name__ == '__main__' :
201+
from glove_utils import load_embedding
202+
start_time = time.time()
203+
GLOVE_FILENAME = 'data/glove.6B.100d.txt'
204+
word2index, index2word, index2embedding = load_embedding(GLOVE_FILENAME)
205+
print('Loaded %s word vectors in %f seconds' % (len(word2index), time.time() - start_time))
206+
embedding = Embedding(word2index, index2word, index2embedding)
207+
imdb_model = IMDBModel('models/lstm_model.h5', embedding)
208+
text = "Really good movie, highly recommended."
209+
prediction = imdb_model.predict(text)
210+
print(prediction)
211+
seq = imdb_model.text2seq(text)
212+
word_embeddings = imdb_model.sequence_to_embedding(seq)
213+
pred = imdb_model.embedding_model_predict(word_embeddings)
214+
print(pred)
215+
assert prediction == pred

‎README.md

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Robustness Evaluation for NLP
2+
3+
### Download Data
4+
Run `sh data/download_data.sh` to download the data needed for the experiments.
5+
This will download the preprocessed IMDB dataset, the word embeddings for the neural network and for the adversarial attack, and dictionaries of cached nearest neighbors.
6+
7+
### Download StanfordCoreNLP
8+
Run `sh download_stanfordcorenlp.sh` to download the Standford CoreNLP parser.
9+
10+
### Train LSTM model
11+
Run `python train_lstm_model.py` to train an LSTM model for sentiment classification on the IMDB dataset.
12+
13+
14+
### Perform verification using DeepGo
15+
Run `python verification.py` to perform robustness analysis using [DeepGo](https://arxiv.org/abs/1805.02242) on the LSTM.
16+
17+
### Obtain explanations for sentiment analysis.
18+
Run the code in `sbe_examples.ipynb` to generate explanations for sentiment analysis, using the adaptation of [Spectrum-Based Explanations](https://arxiv.org/abs/1908.02374v1) to text classification.
19+
20+
### Generate adversarial examples
21+
Run the code in `attack.ipynb` to generate adversarial examples for sentiment analysis.
22+
To visualize the generated adversarial examples run the code in `visualize_attack.ipynb`.
23+
Some examples of adversarial examples are already shown in `visualize_attack.ipynb`.
24+
25+
### Fix the classification
26+
Run the code in `fix_classification.ipynb` to generate suggestions that change the classification
27+
of a text to the correct class. To visualize the suggestions run the code in `visualize_fixing.ipynb`.
28+

0 commit comments

Comments
 (0)