Using tf.keras + default Initializers in Python files. (#204)

Involves the "sentiment" and "translation" examples.

Author: hereismari

sentiment/python/imdb.py

@@ -32,7 +32,7 @@
 import json
 import os
 
-import keras
+import tensorflow as tf
 import tensorflowjs as tfjs
 
 
@@ -50,7 +50,7 @@ def get_word_index(reverse=False):
   Returns:
     The word index as a `dict`.
   """
-  word_index = keras.datasets.imdb.get_word_index()
+  word_index = tf.keras.datasets.imdb.get_word_index()
   if reverse:
     word_index = dict((word_index[key], key) for key in word_index)
   return word_index
@@ -85,10 +85,10 @@ def get_imdb_data(vocabulary_size, max_len):
     y_test: Same as `y_train`, but for test.
   """
   print("Getting IMDB data with vocabulary_size %d" % vocabulary_size)
-  (x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(
+  (x_train, y_train), (x_test, y_test) = tf.keras.datasets.imdb.load_data(
       num_words=vocabulary_size)
-  x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_len)
-  x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_len)
+  x_train = tf.keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_len)
+  x_test = tf.keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_len)
   return x_train, y_train, x_test, y_test
 
 
@@ -122,38 +122,30 @@ def train_model(model_type,
     ValueError: on invalid model type.
   """
 
-  model = keras.Sequential()
-  model.add(keras.layers.Embedding(vocabulary_size, embedding_size))
+  model = tf.keras.Sequential()
+  model.add(tf.keras.layers.Embedding(vocabulary_size, embedding_size))
   if model_type == 'bidirectional_lstm':
     # TODO(cais): Uncomment the following once bug b/74429960 is fixed.
-    # model.add(keras.layers.Embedding(
+    # model.add(tf.keras.layers.Embedding(
     #     vocabulary_size, 128, input_length=maxlen))
-    # model.add(keras.layers.Bidirectional(
-    #     keras.layers.LSTM(64,
-    #                       kernel_initializer='glorot_normal',
-    #                       recurrent_initializer ='glorot_normal')))
-    # model.add(keras.layers.Dropout(0.5))
+    # model.add(tf.keras.layers.Bidirectional(
+    #     tf.keras.layers.LSTM(64))
+    # model.add(tf.keras.layers.Dropout(0.5))
     raise NotImplementedError()
   elif model_type == 'cnn':
-    model.add(keras.layers.Dropout(0.2))
-    model.add(keras.layers.Conv1D(250,
+    model.add(tf.keras.layers.Dropout(0.2))
+    model.add(tf.keras.layers.Conv1D(250,
                                   3,
                                   padding='valid',
                                   activation='relu',
                                   strides=1))
-    model.add(keras.layers.GlobalMaxPooling1D())
-    model.add(keras.layers.Dense(250, activation='relu'))
+    model.add(tf.keras.layers.GlobalMaxPooling1D())
+    model.add(tf.keras.layers.Dense(250, activation='relu'))
   elif model_type == 'lstm':
-    model.add(
-        keras.layers.LSTM(
-            128,
-            kernel_initializer='glorot_normal',
-            recurrent_initializer='glorot_normal'))
-    # TODO(cais): Remove glorot_normal and use the default orthogonal once
-    #   SVD is available.
+    model.add(tf.keras.layers.LSTM(128))
   else:
     raise ValueError("Invalid model type: '%s'" % model_type)
-  model.add(keras.layers.Dense(1, activation='sigmoid'))
+  model.add(tf.keras.layers.Dense(1, activation='sigmoid'))
 
   model.compile('adam', 'binary_crossentropy', metrics=['accuracy'])
   model.fit(x_train, y_train,
@@ -210,7 +202,7 @@ def main():
   tfjs.converters.save_keras_model(model, FLAGS.artifacts_dir)
   print('\nSaved model artifacts in directory: %s' % FLAGS.artifacts_dir)
 
-
+ 
 if __name__ == '__main__':
   parser = argparse.ArgumentParser('IMDB sentiment classification model')
   parser.add_argument(

translation/python/translation.py

@@ -31,8 +31,8 @@
 import json
 import os
 
-from keras.models import Model
-from keras.layers import Input, LSTM, Dense
+from tensorflow.keras.models import Model
+from tensorflow.keras.layers import Input, LSTM, Dense
 import numpy as np
 import tensorflowjs as tfjs
 
@@ -143,8 +143,7 @@ def seq2seq_model(num_encoder_tokens, num_decoder_tokens, latent_dim):
   # Define an input sequence and process it.
   encoder_inputs = Input(shape=(None, num_encoder_tokens))
   encoder = LSTM(latent_dim,
-                 return_state=True,
-                 recurrent_initializer=FLAGS.recurrent_initializer)
+                 return_state=True)
   _, state_h, state_c = encoder(encoder_inputs)
   # We discard `encoder_outputs` and only keep the states.
   encoder_states = [state_h, state_c]
@@ -156,8 +155,7 @@ def seq2seq_model(num_encoder_tokens, num_decoder_tokens, latent_dim):
   # return states in the training model, but we will use them in inference.
   decoder_lstm = LSTM(FLAGS.latent_dim,
                       return_sequences=True,
-                      return_state=True,
-                      recurrent_initializer=FLAGS.recurrent_initializer)
+                      return_state=True)
   decoder_outputs, _, _ = decoder_lstm(decoder_inputs,
                                        initial_state=encoder_states)
   decoder_dense = Dense(num_decoder_tokens, activation='softmax')
@@ -285,11 +283,20 @@ def main():
     # Take one sequence (part of the training set)
     # for trying out decoding.
     input_seq = encoder_input_data[seq_index: seq_index + 1]
+    # Get expected output
+    target_seq = decoder_target_data[seq_index]
+    # One-hot to index
+    target_seq = [
+      reverse_target_char_index[np.argmax(c)] for c in target_seq
+    ]
+    # Array to string
+    target_seq = ''.join(target_seq).replace('\n', '')
     decoded_sentence = decode_sequence(
         input_seq, encoder_model, decoder_model, num_decoder_tokens,
         target_begin_index, reverse_target_char_index, max_decoder_seq_length)
     print('-')
     print('Input sentence:', input_texts[seq_index])
+    print('Target sentence:', target_seq)
     print('Decoded sentence:', decoded_sentence)
 
 
@@ -308,7 +315,7 @@ def main():
   parser.add_argument(
       '--epochs',
       type=int,
-      default=100,
+      default=20,
       help='Number of training epochs.')
   parser.add_argument(
       '--latent_dim',
@@ -325,15 +332,6 @@ def main():
       type=int,
       default=100,
       help='Number of example sentences to test at the end of the training.')
-  # TODO(cais): This is a workaround for the limitation in TF.js Layers that the
-  # default recurrent initializer "Orthogonal" is currently not supported.
-  # Remove this once "Orthogonal" becomes available.
-  parser.add_argument(
-      '--recurrent_initializer',
-      type=str,
-      default='orthogonal',
-      help='Custom initializer for recurrent kernels of LSTMs (e.g., '
-      'glorot_uniform)')
   parser.add_argument(
       '--artifacts_dir',
       type=str,