Initial commit

Author: blublinsky

Diff for: .gitignore

@@ -0,0 +1,5 @@
+target/
+# IntelliJ
+.idea/
+.idea_modules/
+*.iml

Diff for: CustomModelLearn.py

@@ -0,0 +1,34 @@
+import numpy as np
+import tensorflow as tf
+# Declare list of features, we only have one real-valued feature
+def model(features, labels, mode):
+    # Build a linear model and predict values
+    W = tf.get_variable("W", [1], dtype=tf.float64)
+    b = tf.get_variable("b", [1], dtype=tf.float64)
+    y = W*features['x'] + b
+    # Loss sub-graph
+    loss = tf.reduce_sum(tf.square(y - labels))
+    # Training sub-graph
+    global_step = tf.train.get_global_step()
+    optimizer = tf.train.GradientDescentOptimizer(0.01)
+    train = tf.group(optimizer.minimize(loss),
+                     tf.assign_add(global_step, 1))
+    # ModelFnOps connects subgraphs we built to the
+    # appropriate functionality.
+    return tf.contrib.learn.ModelFnOps(
+        mode=mode, predictions=y,
+        loss=loss,
+        train_op=train)
+
+estimator = tf.contrib.learn.Estimator(model_fn=model)
+
+# define our data set
+x = np.array([1., 2., 3., 4.])
+y = np.array([0., -1., -2., -3.])
+input_fn = tf.contrib.learn.io.numpy_input_fn({"x": x}, y, 4, num_epochs=1000)
+
+# train
+estimator.fit(input_fn=input_fn, steps=1000)
+# evaluate our model
+print(estimator.evaluate(input_fn=input_fn, steps=10))
+

Diff for: LinearRegression.py

@@ -0,0 +1,28 @@
+import tensorflow as tf
+import numpy as np
+
+# Model parameters
+W = tf.Variable([.3], tf.float32)
+b = tf.Variable([-.3], tf.float32)
+# Model input and output
+x = tf.placeholder(tf.float32)
+linear_model = W * x + b
+y = tf.placeholder(tf.float32)
+# loss
+loss = tf.reduce_sum(tf.square(linear_model - y)) # sum of the squares
+# optimizer
+optimizer = tf.train.GradientDescentOptimizer(0.01)
+train = optimizer.minimize(loss)
+# training data
+x_train = [1,2,3,4]
+y_train = [0,-1,-2,-3]
+# training loop
+init = tf.global_variables_initializer()
+sess = tf.Session()
+sess.run(init) # reset values to wrong
+for i in range(1000):
+    sess.run(train, {x:x_train, y:y_train})
+
+# evaluate training accuracy
+curr_W, curr_b, curr_loss  = sess.run([W, b, loss], {x:x_train, y:y_train})
+print("W: %s b: %s loss: %s"%(curr_W, curr_b, curr_loss))

Diff for: LinearRegressionLearn.py

@@ -0,0 +1,138 @@
+import tensorflow as tf
+import numpy
+import matplotlib.pyplot as plt
+
+from tensorflow.python.tools import freeze_graph
+
+# Example from https://blog.altoros.com/using-linear-regression-in-tensorflow.html
+# As a training set for the tutorial, we use house prices in Portland, Oregon,
+# where X (the predictor variable) is the house size and Y (the criterion variable) is the house price.
+# The data set contains 47 examples.
+
+# exporting is based on https://medium.com/@hamedmp/exporting-trained-tensorflow-models-to-c-the-right-way-cf24b609d183#.rhjmkrdln
+# and https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/tools/freeze_graph_test.py
+
+# Train a data set
+
+size_data = numpy.asarray([ 2104,  1600,  2400,  1416,  3000,  1985,  1534,  1427,
+                            1380,  1494,  1940,  2000,  1890,  4478,  1268,  2300,
+                            1320,  1236,  2609,  3031,  1767,  1888,  1604,  1962,
+                            3890,  1100,  1458,  2526,  2200,  2637,  1839,  1000,
+                            2040,  3137,  1811,  1437,  1239,  2132,  4215,  2162,
+                            1664,  2238,  2567,  1200,   852,  1852,  1203 ])
+price_data = numpy.asarray([ 399900,  329900,  369000,  232000,  539900,  299900,  314900,  198999,
+                             212000,  242500,  239999,  347000,  329999,  699900,  259900,  449900,
+                             299900,  199900,  499998,  599000,  252900,  255000,  242900,  259900,
+                             573900,  249900,  464500,  469000,  475000,  299900,  349900,  169900,
+                             314900,  579900,  285900,  249900,  229900,  345000,  549000,  287000,
+                             368500,  329900,  314000,  299000,  179900,  299900,  239500 ])
+
+# Test a data set
+
+size_data_test = numpy.asarray([ 1600, 1494, 1236, 1100, 3137, 2238 ])
+price_data_test = numpy.asarray([ 329900, 242500, 199900, 249900, 579900, 329900 ])
+
+# Normalizing your data helps to improve the performance of gradient descent, especially in the case of multivariate linear regression.
+def normalize(array):
+    return (array - array.mean()) / array.std()
+
+# Normalize a data set
+
+size_data_n = normalize(size_data)
+price_data_n = normalize(price_data)
+
+size_data_test_n = normalize(size_data_test)
+price_data_test_n = normalize(price_data_test)
+
+checkpoint_path = "models/saved_checkpoint"
+checkpoint_state_name = "saved_checkpoint"
+
+
+# Display a plot
+# plt.plot(size_data, price_data, 'ro', label='Samples data')
+# plt.legend()
+# plt.draw()
+
+samples_number = price_data_n.size
+
+# TF graph input
+X = tf.placeholder("float")
+Y = tf.placeholder("float")
+
+# Create a model
+
+# Set model weights
+W = tf.Variable(numpy.random.randn(), name="weight")
+b = tf.Variable(numpy.random.randn(), name="bias")
+
+# Set parameters
+learning_rate = 0.1
+training_iteration = 200
+
+# Construct a linear model
+model = tf.add(tf.multiply(X, W), b, name="model")
+
+# Minimize squared errors
+cost_function = tf.reduce_sum(tf.pow(model - Y, 2))/(2 * samples_number) #L2 loss
+optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost_function, name='optimizer') #Gradient descent
+
+# Initialize variables
+init = tf.initialize_all_variables()
+
+# 'Saver' op to save and restore all the variables
+saver = tf.train.Saver()
+
+# Launch a graph
+with tf.Session() as sess:
+    sess.run(init)
+
+    display_step = 20
+
+    # Fit all training data
+    for iteration in range(training_iteration):
+        for (x, y) in zip(size_data_n, price_data_n):
+            sess.run(optimizer, feed_dict={X: x, Y: y})
+
+        # Display logs per iteration step
+        if iteration % display_step == 0:
+            print "Iteration:", '%04d' % (iteration + 1), "cost=", "{:.9f}".format(sess.run(cost_function, feed_dict={X:size_data_n, Y:price_data_n})), \
+                    "W=", sess.run(W), "b=", sess.run(b)
+
+    tuning_cost = sess.run(cost_function, feed_dict={X: normalize(size_data_n), Y: normalize(price_data_n)})
+
+    print "Tuning completed:", "cost=", "{:.9f}".format(tuning_cost), "W=", sess.run(W), "b=", sess.run(b)
+
+    # Validate a tuning model
+
+    testing_cost = sess.run(cost_function, feed_dict={X: size_data_test_n, Y: price_data_test_n})
+
+    print "Testing data cost:" , testing_cost
+
+    # Display a plot
+#    plt.figure()
+#    plt.plot(size_data_n, price_data_n, 'ro', label='Normalized samples')
+#    plt.plot(size_data_test_n, price_data_test_n, 'go', label='Normalized testing samples')
+#    plt.plot(size_data_n, sess.run(W) * size_data_n + sess.run(b), label='Fitted line')
+#    plt.legend()
+
+#    plt.show()
+
+# Save model weights and graph to disk
+    save_path = saver.save(sess, checkpoint_path, global_step=0, latest_filename=checkpoint_state_name)
+    print "Save path:" , save_path
+    graph_path = tf.train.write_graph(sess.graph_def, 'models/', 'graph.pb', as_text=True)
+    print "Graph path:" , graph_path
+
+# Merge grapt with weights
+#    input_saver_def_path = ""
+#    input_binary = False
+#    output_node_names = "weight/read"
+#    restore_op_name = "save/restore_all"
+#    filename_tensor_name = "save/Const:0"
+#    clear_devices = False
+#    output_graph_path = "models/graphwithdata.pb"
+
+#    freeze_graph.freeze_graph(graph_path, input_saver_def_path,
+#                              input_binary, save_path, output_node_names,
+#                              filename_tensor_name, filename_tensor_name,
+#                              output_graph_path, clear_devices, "")

Diff for: LogisticRegression.py

@@ -0,0 +1,36 @@
+# from https://github.com/nlintz/TensorFlow-Tutorials/blob/master/02_logistic_regression.py
+
+import tensorflow as tf
+import numpy as np
+from tensorflow.examples.tutorials.mnist import input_data
+
+def init_weights(shape):
+    return tf.Variable(tf.random_normal(shape, stddev=0.01))
+
+def model(X, w):
+    return tf.matmul(X, w) # notice we use the same model as linear regression, this is because there is a baked in cost function which performs softmax and cross entropy
+
+
+mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
+trX, trY, teX, teY = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels
+
+X = tf.placeholder("float", [None, 784]) # create symbolic variables
+Y = tf.placeholder("float", [None, 10])
+
+w = init_weights([784, 10]) # like in linear regression, we need a shared variable weight matrix for logistic regression
+
+py_x = model(X, w)
+
+cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=py_x, labels=Y)) # compute mean cross entropy (softmax is applied internally)
+train_op = tf.train.GradientDescentOptimizer(0.05).minimize(cost) # construct optimizer
+predict_op = tf.argmax(py_x, 1) # at predict time, evaluate the argmax of the logistic regression
+
+# Launch the graph in a session
+with tf.Session() as sess:
+    # you need to initialize all variables
+    tf.global_variables_initializer().run()
+
+    for i in range(100):
+        for start, end in zip(range(0, len(trX), 128), range(128, len(trX)+1, 128)):
+            sess.run(train_op, feed_dict={X: trX[start:end], Y: trY[start:end]})
+        print(i, np.mean(np.argmax(teY, axis=1) == sess.run(predict_op, feed_dict={X: teX})))

Diff for: MNIST_data/t10k-images-idx3-ubyte.gz

@@ -0,0 +1,12 @@
+Tensorflow Python
+===========
+
+This is my simple playground for Tensorflow Python
+follow https://www.tensorflow.org/install/install_mac for installation on Mac. I am using Python 2.7 
+with the latest Tensorflow version
+Most of the examples are my attempts to learn Tensorflow and specyfy where they come from.
+The most useful ones are in Tensorflow tutorial folder and show how to export graph
+reflecting the training results.
+Additioanally TensorflowBoard and Polynomial regression ones are integrated with
+Tensorflow board - Python application very usefull for viewing graph and results
+of training.

Diff for: MNIST_data/t10k-labels-idx1-ubyte.gz

@@ -0,0 +1,130 @@
+# from https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/4_Utils/save_restore_model.py
+
+import tensorflow as tf
+
+# Import MNIST data
+from tensorflow.examples.tutorials.mnist import input_data
+mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
+
+# Parameters
+learning_rate = 0.001
+batch_size = 100
+display_step = 1
+model_path = "/Users/boris/Projects/TensorFlowPython/models/MNISTNueralNet.ckpt"
+
+# Network Parameters
+n_hidden_1 = 256 # 1st layer number of features
+n_hidden_2 = 256 # 2nd layer number of features
+n_input = 784 # MNIST data input (img shape: 28*28)
+n_classes = 10 # MNIST total classes (0-9 digits)
+
+# tf Graph input
+x = tf.placeholder("float", [None, n_input])
+y = tf.placeholder("float", [None, n_classes])
+
+
+# Create model
+def multilayer_perceptron(x, weights, biases):
+    # Hidden layer with RELU activation
+    layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
+    layer_1 = tf.nn.relu(layer_1)
+    # Hidden layer with RELU activation
+    layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])
+    layer_2 = tf.nn.relu(layer_2)
+    # Output layer with linear activation
+    out_layer = tf.matmul(layer_2, weights['out']) + biases['out']
+    return out_layer
+
+# Store layers weight & bias
+weights = {
+    'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])),
+    'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),
+    'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes]))
+}
+biases = {
+    'b1': tf.Variable(tf.random_normal([n_hidden_1])),
+    'b2': tf.Variable(tf.random_normal([n_hidden_2])),
+    'out': tf.Variable(tf.random_normal([n_classes]))
+}
+
+# Construct model
+pred = multilayer_perceptron(x, weights, biases)
+
+# Define loss and optimizer
+cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
+optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
+
+# Initializing the variables
+init = tf.global_variables_initializer()
+
+# 'Saver' op to save and restore all the variables
+saver = tf.train.Saver()
+
+# Running first session - learning
+print("Starting 1st session...")
+with tf.Session() as sess:
+    # Initialize variables
+    sess.run(init)
+
+    # Training cycle
+    for epoch in range(3):
+        avg_cost = 0.
+        total_batch = int(mnist.train.num_examples/batch_size)
+        # Loop over all batches
+        for i in range(total_batch):
+            batch_x, batch_y = mnist.train.next_batch(batch_size)
+            # Run optimization op (backprop) and cost op (to get loss value)
+            _, c = sess.run([optimizer, cost], feed_dict={x: batch_x,
+                                                          y: batch_y})
+            # Compute average loss
+            avg_cost += c / total_batch
+        # Display logs per epoch step
+        if epoch % display_step == 0:
+            print("Epoch:", '%04d' % (epoch+1), "cost=", \
+                  "{:.9f}".format(avg_cost))
+    print("First Optimization Finished!")
+
+    # Test model
+    correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
+    # Calculate accuracy
+    accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
+    print("Accuracy:", accuracy.eval({x: mnist.test.images, y: mnist.test.labels}))
+
+    # Save model weights to disk
+    save_path = saver.save(sess, model_path)
+    print("Model saved in file: %s" % save_path)
+
+# Running a new session - continue training
+print("Starting 2nd session...")
+with tf.Session() as sess:
+    # Initialize variables
+    sess.run(init)
+
+    # Restore model weights from previously saved model
+    saver.restore(sess, model_path)
+    print("Model restored from file: %s" % save_path)
+
+    # Resume training
+    for epoch in range(7):
+        avg_cost = 0.
+        total_batch = int(mnist.train.num_examples / batch_size)
+        # Loop over all batches
+        for i in range(total_batch):
+            batch_x, batch_y = mnist.train.next_batch(batch_size)
+            # Run optimization op (backprop) and cost op (to get loss value)
+            _, c = sess.run([optimizer, cost], feed_dict={x: batch_x,
+                                                          y: batch_y})
+            # Compute average loss
+            avg_cost += c / total_batch
+        # Display logs per epoch step
+        if epoch % display_step == 0:
+            print("Epoch:", '%04d' % (epoch + 1), "cost=", \
+                  "{:.9f}".format(avg_cost))
+    print("Second Optimization Finished!")
+
+    # Test model
+    correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
+    # Calculate accuracy
+    accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
+    print("Accuracy:", accuracy.eval(
+        {x: mnist.test.images, y: mnist.test.labels}))