Generate the accuracy of each label

Author: Duc Hoang

python-analytics/muvr/converters/neon2iosmlp.py

@@ -2,6 +2,10 @@
 import struct
 import cPickle as pkl
 
+from neon.initializers import Uniform, Constant
+from neon.layers import Affine, Dropout
+from neon.transforms import Rectlin, Logistic, Tanh
+
 
 def extract_weights(file_name):
     """Load weights from the file_name. Data in the file should be stored neon model."""
@@ -45,3 +49,40 @@ def write_layers_to_file(layers, output_filename):
     with open(output_filename, 'w') as f:
         data = " ".join(str(e) for e in layers)
         f.write(data)
+
+
+def activation_from_name(name):
+    if name == "relu":
+        activation = Rectlin()
+    elif name == "logistic" or name == "sigmoid":
+        activation = Logistic()
+    elif name == "tanh":
+        activation = Tanh()
+    else:
+        raise Exception("Bad activation name: " + name)
+    return activation
+
+
+def parsing_layer(layer_str):
+    init_norm = Uniform(low=-0.1, high=0.1)
+    bias_init = Constant(val=1.0)
+
+    elements = layer_str.split(" ")
+    i = 2
+    layers = []
+    while i < len(elements):
+        number_nodes = elements[i]
+        activation_func = activation_from_name(elements [i+1])
+        layers.append(Affine(
+            nout=int(number_nodes),
+            init=init_norm,
+            bias=bias_init,
+            activation=activation_func))
+        if i < len(elements) - 2:
+            # don't add this Dropout at the last layer
+            layers.append(Dropout(
+                name="do_" + str(i),
+                keep=0.9))
+        i += 2
+
+    return layers

python-analytics/start_training.py

@@ -10,6 +10,9 @@
 from muvr.converters import neon2iosmlp
 from muvr.training.default_models import generate_default_activity_model
 from muvr.training.default_models import generate_default_exercise_model
+from muvr.dataset.labelmappers import generate_activity_labelmapper
+from muvr.dataset.labelmappers import generate_exercise_labelmapper
+from muvr.visualization.datastats import dataset_statistics
 from pylab import *
 
 
@@ -51,11 +54,14 @@ def label_of_example(index):
     savefig(output_image)
 
 
-def learn_model_from_data(dataset, working_directory, model_name, epoch):
+def learn_model_from_data(dataset, working_directory, model_name, epoch, layer_filename):
     """Use MLP to train the dataset and generate result in working_directory"""
     model_trainer = MLPMeasurementModelTrainer(working_directory, max_epochs=epoch)
 
-    if model_name == "slacking":
+    if layer_filename:
+        layers = neon2iosmlp.parsing_layer(read_file(layer_filename))
+        model = Model(layers=layers)
+    elif model_name == "slacking":
         print "Using slacking model"
         model = generate_default_activity_model(dataset.num_labels)
     else:
@@ -111,8 +117,24 @@ def show_evaluation(model, dataset):
         table[i+1][0] = s
     table[0][0] = "actual \ predicted"
 
+    # Add 3 more last column: Total | Accuracy (%) | ExerciseId
+    table[0].extend(["Total", "Accuracy (%)", "Exercise"])
+    exerId = 1
+    while exerId < len(table):
+        row = table[exerId]
+        total = sum(row[1:len(row)])
+        print row[exerId], " - ", total
+        accuracy = "%.2f" % (float(row[exerId]) / float(total) * 100.0)
+        exerName = table[0][exerId]
+        table[exerId].extend([total, accuracy + "%", exerName])
+        exerId += 1
     return table
 
+def read_file(filename):
+    f = open(filename, 'r')
+    result = f.readline().strip()
+    f.close()
+    return result
 
 def write_to_csv(filename, data):
     """Write csv data to filename"""
@@ -122,25 +144,32 @@ def write_to_csv(filename, data):
     csvfile.close()
 
 
-def main(dataset_directory, working_directory, evaluation_file, visualise_image, model_name, test_directory, is_analysis, epoch):
+def main(dataset_directory, working_directory, evaluation_file, visualise_image, model_name, test_directory, is_analysis, epoch, layer_filename):
     """Main entry point."""
 
+    if model_name == "slacking":
+        mapping_label = generate_activity_labelmapper()
+    else:
+        mapping_label = generate_exercise_labelmapper()
+
     # 1/ Load the dataset
-    dataset = CSVAccelerationDataset(dataset_directory, test_directory)
+    dataset = CSVAccelerationDataset(dataset_directory, test_directory, label_mapper=mapping_label)
     print "Number of training examples:", dataset.num_train_examples
     print "Number of test examples:", dataset.num_test_examples
     print "Number of features:", dataset.num_features
     print "Number of labels:", dataset.num_labels
 
+    # 2a/ Write statistic of the dataset (in terms of window samples)
+    stats = dataset_statistics(dataset)
+    write_to_csv(os.path.join(working_directory, "dataset_stats.csv"), stats)
+
+    # 2b/ Print statistic in term of csv files
     dataset.train_examples.print_statistic("train", dataset.label_id_mapping)
     dataset.test_examples.print_statistic("test", dataset.label_id_mapping)
 
-    # 2/ Visualise the dataset
-    visualise_dataset(dataset, visualise_image)
-
     if not is_analysis:
         # 3/ Train the dataset using MLP
-        mlpmodel, trained_model = learn_model_from_data(dataset, working_directory, model_name, epoch)
+        mlpmodel, trained_model = learn_model_from_data(dataset, working_directory, model_name, epoch, layer_filename)
 
         # 4/ Evaluate the trained model
         table = show_evaluation(trained_model, dataset)
@@ -158,11 +187,12 @@ def main(dataset_directory, working_directory, evaluation_file, visualise_image,
     parser.add_argument('-v', metavar='visualise', default='./output/visualisation.png', type=str, help="visualisation dataset image output")
     parser.add_argument('-m', metavar='modelname', default='demo', type=str, help="prefix name of model")
     parser.add_argument('-loop', metavar='epoch', default=30, type=int, help="number of training epoch")
+    parser.add_argument('-shape', metavar='shape', type=str, help="filename containing the shape of model")
     parser.add_argument('-analysis', action='store_true', default=False)
     args = parser.parse_args()
 
     #
     # A good example of command-line params is
     # -m core -d ../../muvr-training-data/labelled/core -o ../output/ -v ../output/v.png -e  ../output/e.csv
     #
-    sys.exit(main(args.d, args.o, args.e, args.v, args.m, args.t, args.analysis, args.loop))
+    sys.exit(main(args.d, args.o, args.e, args.v, args.m, args.t, args.analysis, args.loop, args.shape))

run_training.sh

@@ -58,21 +58,28 @@ printf "\n\nSTART TRAINING & EVALUATION with parameter:\n\tDataset: %s\n\tTest:
 
 if ! [ -z $IS_ANALYSIS ]
 then
-    python python-analytics/start_training.py -d $DATASET -o $OUTPUT -e $EVAL -v $VISUAL -m $MODEL_NAME -loop $EPOCH -analysis | tee $LOG_FILE
+    if [ -z $TEST_FOLDER ]
+    then
+        python python-analytics/start_training.py -d "$DATASET" -o $OUTPUT -e $EVAL -v $VISUAL -m $MODEL_NAME -loop $EPOCH -analysis | tee $LOG_FILE
+    else
+        python python-analytics/start_training.py -d "$DATASET" -o $OUTPUT -e $EVAL -v $VISUAL -t $TEST_FOLDER -m $MODEL_NAME -loop $EPOCH -analysis | tee $LOG_FILE
+    fi
 else
     remove_output
     if [ -z $TEST_FOLDER ]
     then
-        python python-analytics/start_training.py -d $DATASET -o $OUTPUT -e $EVAL -v $VISUAL -m $MODEL_NAME -loop $EPOCH | tee $LOG_FILE
+        python python-analytics/start_training.py -d "$DATASET" -o $OUTPUT -e $EVAL -v $VISUAL -m $MODEL_NAME -loop $EPOCH | tee $LOG_FILE
     else
-        python python-analytics/start_training.py -d $DATASET -o $OUTPUT -e $EVAL -v $VISUAL -t $TEST_FOLDER -m $MODEL_NAME -loop $EPOCH | tee $LOG_FILE
+        python python-analytics/start_training.py -d "$DATASET" -o $OUTPUT -e $EVAL -v $VISUAL -t $TEST_FOLDER -m $MODEL_NAME -loop $EPOCH | tee $LOG_FILE
     fi
     EXIT_CODE=$?
     if [[ $EXIT_CODE != 0 ]]
     then
         exit $EXIT_CODE
     else
+        open $OUTPUT/dataset_stats.csv
+        open $OUTPUT/evaluation.csv
 #        open $OUTPUT/visualisation.png
-        column -s, -t < $OUTPUT/evaluation.csv
+#        column -s, -t < $OUTPUT/evaluation.csv
     fi
 fi