SINGA-9 Add Support for Restricted Boltzman Machine (RBM) model

* Refactor the CDWorker::TrainOneBatch. * Replace Phase with int flag in ComputeFeature ComputeGradient * functions. The flag can be a combination for multiple phases, e.g., * kTrain|kForward, where each phase is set to have only one none zero bit, e.g., 1, 2, 4, 8, etc. * Remove compliation dependency on opencv. * Refactor JobProto to create a AlgProto for TrainOneBatch. * Create a RBMLayer as the base layer for RBM layers. * Updated all configurations of all examples.
jaejaejae · Aug 28, 2015 · fbbcaaf · fbbcaaf
1 parent ef4de79
commit fbbcaaf
Show file tree

Hide file tree

Showing 22 changed files with 497 additions and 700 deletions.
diff --git a/Makefile.am b/Makefile.am
@@ -95,9 +95,6 @@ singa_LDFLAGS = -I./include \
                 -lglog  \
                 -lprotobuf \
                 -lrt \
-                -lopencv_highgui \
-                -lopencv_imgproc \
-                -lopencv_core \
                 -lopenblas \
                 -lzmq \
                 -lczmq \

diff --git a/configure.ac b/configure.ac
@@ -44,15 +44,15 @@ if test x"$enable_lmdb" = x"yes"; then
 	AC_DEFINE(LMDB, 1, [Enable Option layer])
 fi
 
-AC_CHECK_LIB([opencv_imgproc], [main], [], [
-  AC_MSG_ERROR([unable to find opencv_imgproc lib])
-  ])
-AC_CHECK_LIB([opencv_highgui], [main], [], [
-  AC_MSG_ERROR([unable to find opencv_highgui lib])
-  ])
-AC_CHECK_LIB([opencv_core], [main], [], [
-  AC_MSG_ERROR([unable to find opencv_core lib])
-  ])
+#AC_CHECK_LIB([opencv_imgproc], [main], [], [
+#  AC_MSG_ERROR([unable to find opencv_imgproc lib])
+#  ])
+#AC_CHECK_LIB([opencv_highgui], [main], [], [
+#  AC_MSG_ERROR([unable to find opencv_highgui lib])
+#  ])
+#AC_CHECK_LIB([opencv_core], [main], [], [
+#  AC_MSG_ERROR([unable to find opencv_core lib])
+#  ])
 AC_CHECK_LIB([zookeeper_mt], [main], [], [
   AC_MSG_ERROR([unable to find zookeeper])
   ])

diff --git a/examples/cifar10/job.conf b/examples/cifar10/job.conf
@@ -3,7 +3,10 @@ train_steps: 1000
 test_steps: 100
 test_freq:300
 disp_freq:30
-alg: kBP
+debug: true
+train_one_batch {
+  alg: kBP
+}
 updater{
   type: kSGD
   weight_decay:0.004

diff --git a/examples/mnist/conv.conf b/examples/mnist/conv.conf
@@ -3,7 +3,9 @@ train_steps: 10000
 test_steps:100
 test_freq:500
 disp_freq:50
-alg: kBP
+train_one_batch {
+  alg: kBP
+}
 updater {
   momentum:0.9
   weight_decay:0.0005

diff --git a/examples/mnist/job.conf b/examples/mnist/job.conf
@@ -3,7 +3,9 @@ train_steps: 1000
 test_steps:10
 test_freq:60
 disp_freq:10
-alg: kBP
+train_one_batch {
+  alg: kBP
+}
 updater{
   type: kSGD
   learning_rate{
@@ -82,6 +84,10 @@ neuralnet {
   layer{
     name: "tanh1"
     type: kTanh
+    tanh_conf {
+      outer_scale: 1.7159047
+      inner_scale: 0.6666667
+    }
     srclayers:"fc1"
   }
   layer{
@@ -112,6 +118,11 @@ neuralnet {
   layer{
     name: "tanh2"
     type: kTanh
+    tanh_conf {
+      outer_scale: 1.7159047
+      inner_scale: 0.6666667
+    }
+
     srclayers:"fc2"
   }
   layer{
@@ -143,6 +154,11 @@ neuralnet {
   layer{
     name: "tanh3"
     type: kTanh
+    tanh_conf {
+      outer_scale: 1.7159047
+      inner_scale: 0.6666667
+    }
+
     srclayers:"fc3"
   }
   layer{
@@ -174,6 +190,11 @@ neuralnet {
   layer{
     name: "tanh4"
     type: kTanh
+    tanh_conf {
+      outer_scale: 1.7159047
+      inner_scale: 0.6666667
+    }
+
     srclayers:"fc4"
   }
   layer{
@@ -205,6 +226,11 @@ neuralnet {
   layer{
     name: "tanh5"
     type: kTanh
+    tanh_conf {
+      outer_scale: 1.7159047
+      inner_scale: 0.6666667
+    }
+
     srclayers:"fc5"
   }
   layer{

diff --git a/examples/rbm/autoencoder.conf b/examples/rbm/autoencoder.conf
@@ -1,15 +1,15 @@
-name: "deep-big-simple-mlp"
+name: "auto-encoder"
 train_steps: 12200
 test_steps:100
-test_freq:100
-disp_freq:20
-checkpoint_after: 1000
-checkpoint_freq: 1000
-checkpoint_path: "examples/rbm/checkpoint/rbm0/checkpoint/step6000-worker0.bin"
-checkpoint_path: "examples/rbm/checkpoint/rbm1/checkpoint/step6000-worker0.bin"
-checkpoint_path: "examples/rbm/checkpoint/rbm2/checkpoint/step6000-worker0.bin"
-checkpoint_path: "examples/rbm/checkpoint/rbm3/checkpoint/step6000-worker0.bin"
-alg: kBP
+test_freq:1000
+disp_freq:100
+checkpoint_path: "examples/rbm/rbm0/checkpoint/step6000-worker0.bin"
+checkpoint_path: "examples/rbm/rbm1/checkpoint/step6000-worker0.bin"
+checkpoint_path: "examples/rbm/rbm2/checkpoint/step6000-worker0.bin"
+checkpoint_path: "examples/rbm/rbm3/checkpoint/step6000-worker0.bin"
+train_one_batch{
+  alg: kBP
+}
 updater{
   type: kAdaGrad
   learning_rate{
@@ -23,7 +23,7 @@ neuralnet {
     name: "data"
     type: kShardData
     sharddata_conf {
-      path: "examples/rbm/mnist_train_shard"
+      path: "examples/mnist/mnist_train_shard"
       batchsize: 1000
     }
     exclude: kTest
@@ -33,7 +33,7 @@ neuralnet {
     name: "data"
     type: kShardData
     sharddata_conf {
-      path: "examples/rbm/mnist_test_shard"
+      path: "examples/mnist/mnist_test_shard"
       batchsize: 1000
     }
     exclude: kTrain
@@ -64,19 +64,9 @@ neuralnet {
     }
     param{
       name: "w1"
-      init{
-      type: kUniform
-      low:-0.05
-      high:0.05
-      }
     }
     param{
       name: "rb12"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
     }
   }
 
@@ -94,19 +84,9 @@ neuralnet {
     }
     param{
       name: "w2"
-      init{
-      type: kUniform
-      low:-0.05
-      high:0.05
-      }
     }
     param{
       name: "rb22"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
     }
   }
 
@@ -125,19 +105,9 @@ neuralnet {
     }
     param{
       name: "w3"
-      init{
-      type: kUniform
-      low:-0.05
-      high:0.05
-      }
     }
     param{
       name: "rb32"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
     }
   }
 
@@ -156,19 +126,10 @@ neuralnet {
     }
     param{
       name: "w4"
-      init{
-      type: kUniform
-      low:-0.05
-      high:0.05
-      }
     }
     param{
       name: "rb42"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
+
     }
   }
 
@@ -187,11 +148,6 @@ neuralnet {
     }
     param{
       name: "rb41"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
     }
   }
 
@@ -214,13 +170,7 @@ neuralnet {
     }
     param{
       name: "rb31"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
     }
-
   }
 
   layer{
@@ -242,11 +192,6 @@ neuralnet {
     }
     param{
       name: "rb21"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
     }
 
   }
@@ -270,13 +215,7 @@ neuralnet {
     }
     param{
       name: "rb11"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
     }
-
   }
 
   layer{
@@ -295,5 +234,5 @@ neuralnet {
 cluster {
   nworker_groups: 1
   nserver_groups: 1
-  workspace: "examples/rbm/checkpoint/autoencoder/"
+  workspace: "examples/rbm/autoencoder/"
 }