Transfer code from nusinga repo to singa apache repo.

New commuinication framework is implemented to unify the frameworks of existing distributed deep learning systems.
Communication is now implmented using ZeroMQ. APIs are general to replace the implementation using MPI.
Tested on single node using examples in example/ folder. Todo test with other cluster configurations (e.g., multiple nodes).

.gitignore

@@ -0,0 +1,28 @@
+*.orig
+*.swp
+*.o
+*.bin
+*.a
+*.so
+*.dat
+*~
+*.bak
+*.P
+*.odp
+*.project
+*.cproject
+*.log
+*.nfs*
+script/*
+!script/*.sh
+!script/*.awk
+src/test/data/*
+tmp
+log*
+build/
+tmp/
+include/proto/*.h
+src/proto/*.cc
+.sync
+*lmdb
+*.binaryproto

Makefile

@@ -0,0 +1,90 @@
+###################User Config Varaibles #############################
+HOME_DIR := /usr/
+# Lib folder for system and external libs. You may need to change it.
+LIBRARY_DIRS := $(HOME_DIR)/lib64 $(HOME_DIR)/lib $(HOME_DIR)/local/lib
+# Header folder for system and external libs. You may need to change it.
+INCLUDE_DIRS := $(HOME_DIR)/include ./include
+# g++ location, should support c++11, tested with 4.8.1
+CXX := g++
+
+######################Setting Varialbes#######################################
+LIBRARIES := glog gflags protobuf rt opencv_highgui opencv_imgproc opencv_core\
+	lmdb openblas zmq czmq
+
+LDFLAGS := $(foreach librarydir, $(LIBRARY_DIRS), -L$(librarydir))\
+	$(foreach library, $(LIBRARIES), -l$(library))
+# Folder to store compiled files
+BUILD_DIR := build
+MSHADOW_FLAGS :=-DMSHADOW_USE_CUDA=0 -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0
+CXXFLAGS := -O2 -Wall -pthread -fPIC -std=c++11 -Wno-unknown-pragmas \
+	$(MSHADOW_FLAGS) -DCPU_ONLY=1 \
+	-funroll-loops $(foreach includedir, $(INCLUDE_DIRS), -I$(includedir))
+
+# find user defined .proto file, and then compute the corresponding .h, .cc
+# files, which cannot be found by shell find, because they haven't been
+# generated currently
+PROTOS := $(shell find src/proto/ -name "*.proto")
+PROTO_SRCS :=$(PROTOS:.proto=.pb.cc)
+PROTO_HDRS :=$(patsubst src%, include%, $(PROTOS:.proto=.pb.h))
+PROTO_OBJS :=$(addprefix $(BUILD_DIR)/, $(PROTO_SRCS:.cc=.o))
+
+# each singa src file will generate a .o file
+SINGA_SRCS := $(shell find src/ \( -path "src/test" -o -path "src/main.cc" \) \
+	-prune -o \( -name "*.cc" -type f \) -print )
+SINGA_OBJS := $(sort $(addprefix $(BUILD_DIR)/, $(SINGA_SRCS:.cc=.o)) \
+	$(PROTO_OBJS) )
+-include $(SINGA_OBJS:%.o=%.P)
+
+TEST_SRCS :=$(shell find src/test/ -maxdepth 1 -name "*.cc")
+TEST_OBJS := $(sort $(addprefix $(BUILD_DIR)/, $(TEST_SRCS:.cc=.o)))
+-include $(TEST_OBJS:%.o=%.P)
+
+GTEST_SRC := include/gtest/gtest-all.cc
+GTEST_HDR := include/gtest/gtest.h
+GTEST_LIB := $(BUILD_DIR)/libgtest.a
+
+OBJS := $(sort $(SINGA_OBJS) $(TEST_OBJS) )
+
+########################Compilation Section###################################
+.PHONY: singa test
+
+singa: $(PROTO_OBJS) $(SINGA_OBJS)
+	$(CXX) $(SINGA_OBJS) src/main.cc -o $(BUILD_DIR)/singa $(CXXFLAGS) $(LDFLAGS)
+	@echo
+
+loader: proto $(LOADER_OBJS)
+	$(CXX) $(LOADER_OBJS) -o $(BUILD_DIR)/loader $(CXXFLAGS) $(LDFLAGS)
+	@echo
+
+test:  proto $(GTEST_LIB) $(TEST_OBJS) $(SINGA_OBJS)
+	$(CXX) $(TEST_OBJS) include/gtest/gtest_main.cc $(GTEST_LIB) \
+		$(SINGA_OBJS) -o $(BUILD_DIR)/test $(CXXFLAGS) $(LDFLAGS)
+	@echo
+
+$(GTEST_LIB): $(GTEST_HDR) $(GTEST_SRC)
+	$(CXX) $(GTEST_SRC) -c -o $(BUILD_DIR)/gtest-all.o $(CXXFLAGS)
+	ar -rv $(GTEST_LIB) $(BUILD_DIR)/gtest-all.o
+
+# compile all files
+$(OBJS):$(BUILD_DIR)/%.o : %.cc
+	@mkdir -p $(dir $@)
+	$(CXX) $<  $(CXXFLAGS) -MMD -c -o $@
+	cp $(BUILD_DIR)/$*.d $(BUILD_DIR)/$*.P; \
+	sed -e 's/#.*//' -e 's/^[^:]*: *//' -e 's/ *\\$$//' \
+		-e '/^$$/ d' -e 's/$$/ :/' < $(BUILD_DIR)/$*.d >> $(BUILD_DIR)/$*.P; \
+	rm -f $*.d
+
+proto: $(PROTO_OBJS)
+
+$(PROTO_SRCS): $(PROTOS)
+	protoc --proto_path=src/proto --cpp_out=src/proto $(PROTOS)
+	mkdir -p include/proto/
+	cp src/proto/*.pb.h include/proto/
+	@echo
+
+clean:
+	rm -rf *.a *.so
+	rm -rf include/proto/*
+	rm -rf src/proto/*.pb.h src/proto/*.pb.cc
+	rm -rf $(BUILD_DIR)
+	@echo

README.md

@@ -1,4 +1,6 @@
-SINGA
+Apache SINGA
 =====
 
 Distributed deep learning system
+
+[Project Page](http://singa.incubator.apache.org)

examples/cifar10/cluster.conf

@@ -0,0 +1,3 @@
+nworker_groups: 1
+nserver_groups: 1
+workspace: "/data1/wangwei/singa/data/mnist"

examples/cifar10/hostfile

@@ -0,0 +1,20 @@
+awan-2-26-0
+awan-2-27-0
+awan-2-28-0
+awan-2-29-0
+awan-2-30-0
+awan-2-31-0
+awan-2-32-0
+awan-2-33-0
+awan-2-34-0
+awan-2-35-0
+awan-2-36-0
+awan-2-37-0
+awan-2-38-0
+awan-2-39-0
+awan-2-40-0
+awan-2-41-0
+awan-2-42-0
+awan-2-43-0
+awan-2-44-0
+awan-2-45-0

examples/cifar10/model.conf

@@ -0,0 +1,219 @@
+name: "cifar10-convnet"
+train_steps: 70000
+test_steps:100
+test_frequency:1000
+display_frequency:50
+updater{
+  momentum:0.9
+  weight_decay:0.004
+  learning_rate_change_method:kFixedStep
+  step:0
+  step:60000
+  step:65000
+  step_lr:0.001
+  step_lr:0.0001
+  step_lr:0.00001
+}
+neuralnet {
+layer {
+  name: "data"
+  type: "kLMDBData"
+  data_param {
+    path: "/home/wangwei/program/singa/examples/cifar10/cifar10_train_lmdb"
+    batchsize: 100
+    random_skip:10000
+  }
+  exclude: kTest
+}
+
+layer {
+  name: "data"
+  type: "kLMDBData"
+  data_param {
+    path: "/home/wangwei/program/singa/examples/cifar10/cifar10_test_lmdb"
+    batchsize: 100
+  }
+  exclude: kTrain
+}
+
+layer{
+  name:"rgb"
+  type: "kRGBImage"
+  srclayers: "data"
+  rgbimage_param {
+    meanfile: "/home/wangwei/program/singa/examples/cifar10/mean.binaryproto"
+  }
+}
+
+layer{
+  name: "label"
+  type: "kLabel"
+  srclayers: "data"
+}
+layer {
+  name: "conv1"
+  type: "kConvolution"
+  srclayers: "rgb"
+  convolution_param {
+    num_filters: 32
+    kernel: 5
+    stride: 1
+    pad:2
+  }
+  param{
+      name: "weight"
+      init_method:kGaussian
+      std:0.0001
+      learning_rate_multiplier:1.0
+    }
+  param{
+      name: "bias"
+      init_method: kConstant
+      learning_rate_multiplier:2.0
+      value:0
+    }
+}
+layer {
+  name: "pool1"
+  type: "kPooling"
+  srclayers: "conv1"
+  pooling_param {
+    pool: MAX
+    kernel: 3
+    stride: 2
+  }
+}
+layer {
+  name: "relu1"
+  type: "kReLU"
+  srclayers:"pool1"
+}
+layer {
+  name: "norm1"
+  type: "kLRN"
+  lrn_param {
+    norm_region: WITHIN_CHANNEL
+    local_size: 3
+    alpha: 5e-05
+    beta: 0.75
+  }
+  srclayers:"relu1"
+}
+layer {
+  name: "conv2"
+  type: "kConvolution"
+  srclayers: "norm1"
+  convolution_param {
+    num_filters: 32
+    kernel: 5
+    stride: 1
+    pad:2
+  }
+  param{
+      name: "weight"
+      init_method:kGaussian
+      std:0.01
+      learning_rate_multiplier:1.0
+    }
+  param{
+      name: "bias"
+      init_method: kConstant
+      learning_rate_multiplier:2.0
+      value:0
+    }
+}
+layer {
+  name: "relu2"
+  type: "kReLU"
+  srclayers:"conv2"
+}
+layer {
+  name: "pool2"
+  type: "kPooling"
+  srclayers: "relu2"
+  pooling_param {
+    pool: MAX
+    kernel: 3
+    stride: 2
+  }
+}
+layer {
+  name: "norm2"
+  type: "kLRN"
+  lrn_param {
+    norm_region: WITHIN_CHANNEL
+    local_size: 3
+    alpha: 5e-05
+    beta: 0.75
+  }
+  srclayers:"pool2"
+}
+layer {
+  name: "conv3"
+  type: "kConvolution"
+  srclayers: "norm2"
+  convolution_param {
+    num_filters: 64
+    kernel: 5
+    stride: 1
+    pad:2
+  }
+  param{
+      name: "weight"
+      init_method:kGaussian
+      std:0.01
+    }
+  param{
+      name: "bias"
+      init_method: kConstant
+      value:0
+    }
+}
+layer {
+  name: "relu3"
+  type: "kReLU"
+  srclayers:"conv3"
+}
+layer {
+  name: "pool3"
+  type: "kPooling"
+  srclayers: "relu3"
+  pooling_param {
+    pool: AVE
+    kernel: 3
+    stride: 2
+  }
+}
+layer {
+  name: "ip1"
+  type: "kInnerProduct"
+  srclayers:"pool3"
+  inner_product_param {
+    num_output: 10
+  }
+  param{
+      name: "weight"
+      init_method:kGaussian
+      std:0.01
+      learning_rate_multiplier:1.0
+      weight_decay_multiplier:250
+    }
+  param{
+      name: "bias"
+      init_method: kConstant
+      learning_rate_multiplier:2.0
+      weight_decay_multiplier:0
+      value:0
+  }
+}
+
+layer{
+  name: "loss"
+  type:"kSoftmaxLoss"
+  softmaxloss_param{
+    topk:1
+  }
+  srclayers:"ip1"
+  srclayers:"label"
+}
+}