add KucukelbirOptimizer

edward/optimizers/__init__.py

@@ -9,7 +9,7 @@
 from tensorflow.python.util.all_util import remove_undocumented
 
 _allowed_symbols = [
-    'alp_optimizer_apply_gradients',
+    'KucukelbirOptimizer',
 ]
 
 remove_undocumented(__name__, allowed_exception_list=_allowed_symbols)

edward/optimizers/sgd.py

@@ -5,15 +5,26 @@
 import tensorflow as tf
 
 
-def alp_optimizer_apply_gradients(n, s_n, grads_and_vars):
-  ops = []
-  for i, (grad, var) in enumerate(grads_and_vars):
-    updated_s_n = s_n[i].assign( (t * grad**2) + (1 - t) * s_n[i] )
-
-    p_n_first = eta * n**(-.5 + delta)
-    p_n_second = (1 + tf.sqrt(updated_s_n[i]))**(-1)
-    p_n = p_n_first * p_n_second
-
-    updated_var = var.assign_add(-p_n * grad)
-    ops.append(updated_var)
-  return ops
+class KucukelbirOptimizer:
+
+  """
+  # TODO: add me
+  """
+
+  def __init__(self, t, delta, eta):
+    self.t = t
+    self.delta = delta
+    self.eta = eta
+
+  def apply_gradients(self, n, s_n, grads_and_vars):
+    ops = []
+    for i, (grad, var) in enumerate(grads_and_vars):
+      updated_s_n = s_n[i].assign( (self.t * grad**2) + (1 - self.t) * s_n[i] )
+
+      p_n_first = self.eta * n**(-.5 + self.delta)
+      p_n_second = (1 + tf.sqrt(updated_s_n[i]))**(-1)
+      p_n = p_n_first * p_n_second
+
+      updated_var = var.assign_add(-p_n * grad)
+      ops.append(updated_var)
+    return ops

tests/optimizers/test_sgd.py

@@ -4,7 +4,7 @@
 
 import tensorflow as tf
 
-from edward.optimizers import alp_optimizer_apply_gradients
+from edward.optimizers import KucukelbirOptimizer
 
 
 class test_sgd_class(tf.test.TestCase):
@@ -15,23 +15,11 @@ def test_kucukelbir_grad(self):
       [(2.7902498, 0.8434107), (1.241244, 1.8959416)],
       [(2.6070995, 0.7563643), (1.0711095, 1.8410041)]
     ]
+
     t = 0.1
     delta = 10e-3
     eta = 1e-1
 
-    def alp_optimizer_apply_gradients(n, s_n, grads_and_vars):
-      ops = []
-      for i, (grad, var) in enumerate(grads_and_vars):
-        updated_s_n = s_n[i].assign( (t * grad**2) + (1 - t) * s_n[i] )
-
-        p_n_first = eta * n**(-.5 + delta)
-        p_n_second = (1 + tf.sqrt(updated_s_n[i]))**(-1)
-        p_n = p_n_first * p_n_second
-
-        updated_var = var.assign_add(-p_n * grad)
-        ops.append(updated_var)
-      return ops
-
     w1 = tf.Variable(tf.constant(1.))
     w2 = tf.Variable(tf.constant(2.))
     var_list = [w1, w2]
@@ -47,7 +35,8 @@ def alp_optimizer_apply_gradients(n, s_n, grads_and_vars):
     s_n = tf.Variable(tf.zeros(2))
     n = tf.Variable(tf.constant(1.))
 
-    train = alp_optimizer_apply_gradients(n, s_n, grads_and_vars)
+    optimizer = KucukelbirOptimizer(t=t, delta=delta, eta=eta)
+    train = optimizer.apply_gradients(n, s_n, grads_and_vars)
     increment_n = n.assign_add(1.)
 
     actual_grads_and_vars = []