Updated slides

Author: hunkim

08_2_dataset_loade_logistic.py

@@ -12,7 +12,8 @@ class DiabetesDataset(Dataset):
 
     # Initialize your data, download, etc.
     def __init__(self):
-        xy = np.loadtxt('./data/diabetes.csv.gz', delimiter=',', dtype=np.float32)
+        xy = np.loadtxt('./data/diabetes.csv.gz',
+                        delimiter=',', dtype=np.float32)
         self.len = xy.shape[0]
         self.x_data = torch.from_numpy(xy[:, 0:-1])
         self.y_data = torch.from_numpy(xy[:, [-1]])

12_4_hello_rnn_emb.py

@@ -30,7 +30,8 @@ class Model(nn.Module):
     def __init__(self):
         super(Model, self).__init__()
         self.embedding = nn.Embedding(input_size, embedding_size)
-        self.rnn = nn.RNN(input_size=embedding_size, hidden_size=5, batch_first=True)
+        self.rnn = nn.RNN(input_size=embedding_size,
+                          hidden_size=5, batch_first=True)
         self.fc = nn.Linear(hidden_size, num_classes)
 
     def forward(self, x):

13_1_rnn_classification_basics.py

@@ -0,0 +1,95 @@
+# Original code is from https://github.com/spro/practical-pytorch
+import time
+import math
+import torch
+import torch.nn as nn
+from torch.autograd import Variable
+from torch.utils.data import DataLoader
+
+from name_dataset import NameDataset
+from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
+
+# Parameters and DataLoaders
+HIDDEN_SIZE = 100
+N_CHARS = 128  # ASCII
+N_CLASSES = 18
+
+
+class RNNClassifier(nn.Module):
+
+    def __init__(self, input_size, hidden_size, output_size, n_layers=1):
+        super(RNNClassifier, self).__init__()
+        self.hidden_size = hidden_size
+        self.n_layers = n_layers
+
+        self.embedding = nn.Embedding(input_size, hidden_size)
+        self.gru = nn.GRU(hidden_size, hidden_size, n_layers)
+        self.fc = nn.Linear(hidden_size, output_size)
+
+    def forward(self, input):
+        # Note: we run this all at once (over the whole input sequence)
+
+        # input = B x S . size(0) = B
+        batch_size = input.size(0)
+
+        # input:  B x S  -- (transpose) --> S x B
+        input = input.t()
+
+        # Embedding S x B -> S x B x I (embedding size)
+        print("  input", input.size())
+        embedded = self.embedding(input)
+        print("  embedding", embedded.size())
+
+        # Make a hidden
+        hidden = self._init_hidden(batch_size)
+
+        output, hidden = self.gru(embedded, hidden)
+        print("  gru hidden output", hidden.size())
+        # Use the last layer output as FC's input
+        # No need to unpack, since we are going to use hidden
+        fc_output = self.fc(hidden)
+        print("  fc output", fc_output.size())
+        return fc_output
+
+    def _init_hidden(self, batch_size):
+        hidden = torch.zeros(self.n_layers, batch_size, self.hidden_size)
+        return Variable(hidden)
+
+# Help functions
+
+
+def str2ascii_arr(msg):
+    arr = [ord(c) for c in msg]
+    return arr, len(arr)
+
+# pad sequences and sort the tensor
+def pad_sequences(vectorized_seqs, seq_lengths):
+    seq_tensor = torch.zeros((len(vectorized_seqs), seq_lengths.max())).long()
+    for idx, (seq, seq_len) in enumerate(zip(vectorized_seqs, seq_lengths)):
+        seq_tensor[idx, :seq_len] = torch.LongTensor(seq)
+    return seq_tensor
+
+# Create necessary variables, lengths, and target
+def make_variables(names):
+    sequence_and_length = [str2ascii_arr(name) for name in names]
+    vectorized_seqs = [sl[0] for sl in sequence_and_length]
+    seq_lengths = torch.LongTensor([sl[1] for sl in sequence_and_length])
+    return pad_sequences(vectorized_seqs, seq_lengths)
+
+
+if __name__ == '__main__':
+    names = ['adylov', 'solan', 'hard', 'san']
+    classifier = RNNClassifier(N_CHARS, HIDDEN_SIZE, N_CLASSES)
+
+    for name in names:
+        arr, _ = str2ascii_arr(name)
+        inp = Variable(torch.LongTensor([arr]))
+        out = classifier(inp)
+        print("in", inp.size(), "out", out.size())
+
+
+    inputs = make_variables(names)
+    out = classifier(inputs)
+    print("batch in", inputs.size(), "batch out", out.size())
+
+

13_1_rnn_classification.py 13_2_rnn_classification.py

@@ -25,6 +25,7 @@
                           batch_size=BATCH_SIZE, shuffle=True)
 
 N_COUNTRIES = len(train_dataset.get_countries())
+print(N_COUNTRIES, "countries")
 N_CHARS = 128  # ASCII
 
 
@@ -90,9 +91,7 @@ class RNNClassifier(nn.Module):
 
     def __init__(self, input_size, hidden_size, output_size, n_layers=1, bidirectional=True):
         super(RNNClassifier, self).__init__()
-        self.input_size = input_size
         self.hidden_size = hidden_size
-        self.output_size = output_size
         self.n_layers = n_layers
         self.n_directions = int(bidirectional) + 1
 

13_2_char_rnn.py 13_3_char_rnn.py

@@ -103,6 +103,7 @@ def train_teacher_forching(line):
 
     return loss.data[0] / len(input)
 
+
 def train(line):
     input = str2tensor(line[:-1])
     target = str2tensor(line[1:])
@@ -144,5 +145,3 @@ def train(line):
                 print('[(%d %d%%) loss: %.4f]' %
                       (epoch, epoch / n_epochs * 100, loss))
                 print(generate(decoder, 'Wh', 100), '\n')
-
-

14_3_pack_pad.py 13_4_pack_pad.py

@@ -120,7 +120,6 @@ def translate(enc_input='thisissungkim.iloveyou.', predict_len=100, temperature=
     optimizer = torch.optim.Adam(params, lr=0.001)
     criterion = nn.CrossEntropyLoss()
 
-
     train_loader = DataLoader(dataset=TextDataset(),
                               batch_size=BATCH_SIZE,
                               shuffle=True,

14_2_seq2seq_att.py

@@ -1,7 +1,7 @@
 # PyTorchZeroToAll
 Quick 3~4 day lecture materials for HKUST students.
 
-## Video Lectures: RNN TBA]
+## Video Lectures: (RNN TBA)
 * [Youtube](http://bit.ly/PyTorchVideo)
 * [Bilibili](https://www.bilibili.com/video/av15823922/)