Add FaceNet FP32 Inference (#259)

mjkyung · dmsuehir · commit 32cbd693f87d · 2019-04-01T14:08:10.000-07:00
* Facenet update

* Refactor model_init and place the model files in models directory

* Update start.sh for FaceNet

* Move Facenet to proper directory

* Fix a typo

* Add datafile

* Add custum arg

* model_init update

* README.md update

* Updatd licensing header

* Add unit test for latency/accuracy

* Add example log and unit test

* Update dataset link in README.md

* Fix indentation error

* Update unit test command to reflect correct location

* Use model-source-dir

* add model-source-dir

* Update dataset download instruction

* Remove misplaced lines
diff --git a/benchmarks/README.md b/benchmarks/README.md
@@ -17,6 +17,7 @@ dependencies to be installed:
 | -----------------------| --------------| ------------------- | --------- |------------------------------|
 | Adversarial Networks   | TensorFlow    | [DCGAN](https://arxiv.org/pdf/1511.06434.pdf)               | Inference | [FP32](adversarial_networks/tensorflow/dcgan/README.md#fp32-inference-instructions) |
 | Content Creation       | TensorFlow    | [DRAW](https://arxiv.org/pdf/1502.04623.pdf)               | Inference | [FP32](content_creation/tensorflow/draw/README.md#fp32-inference-instructions) |
+| Face Detection and Alignment | Tensorflow    | [FaceNet](https://arxiv.org/pdf/1503.03832.pdf)             | Inference | [FP32](face_detection_and_alignment/tensorflow/facenet/README.md#fp32-inference-instructions) |
 | Face Detection and Alignment | TensorFlow    | [MTCC](https://arxiv.org/pdf/1604.02878.pdf)               | Inference | [FP32](face_detection_and_alignment/tensorflow/mtcc/README.md#fp32-inference-instructions) |
 | Image Recognition      | TensorFlow    | [Inception ResNet V2](https://arxiv.org/pdf/1602.07261.pdf) | Inference | [Int8](image_recognition/tensorflow/inception_resnet_v2/README.md#int8-inference-instructions) [FP32](image_recognition/tensorflow/inception_resnet_v2/README.md#fp32-inference-instructions) |
 | Image Recognition      | TensorFlow    | [Inception V3](https://arxiv.org/pdf/1512.00567.pdf)        | Inference | [Int8](image_recognition/tensorflow/inceptionv3/README.md#int8-inference-instructions) [FP32](image_recognition/tensorflow/inceptionv3/README.md#fp32-inference-instructions) |
diff --git a/benchmarks/common/tensorflow/start.sh b/benchmarks/common/tensorflow/start.sh
@@ -288,6 +288,21 @@ function draw() {
   fi
 }
 
+# FaceNet model
+function facenet() {
+  if [ ${PRECISION} == "fp32" ]; then
+    cp ${MOUNT_INTELAI_MODELS_SOURCE}/${PRECISION}/validate_on_lfw.py \
+        ${MOUNT_EXTERNAL_MODELS_SOURCE}/src/validate_on_lfw.py
+
+    CMD="${CMD} $(add_arg "--lfw_pairs" ${lfw_pairs})"
+    PYTHONPATH=${PYTHONPATH}:${MOUNT_BENCHMARK}:${MOUNT_EXTERNAL_MODELS_SOURCE}
+    PYTHONPATH=${PYTHONPATH} CMD=${CMD} run_model
+  else
+    echo "PRECISION=${PRECISION} is not supported for ${MODEL_NAME}"
+    exit
+  fi
+}
+
 # Faster R-CNN (ResNet50) model
 function faster_rcnn() {
     export PYTHONPATH=$PYTHONPATH:${MOUNT_EXTERNAL_MODELS_SOURCE}/research:${MOUNT_EXTERNAL_MODELS_SOURCE}/research/slim
@@ -574,7 +589,7 @@ function ssd-resnet34() {
     if [ ${PRECISION} == "fp32" ]; then
       if [ ${NOINSTALL} != "True" ]; then
         for line in $(cat ${MOUNT_BENCHMARK}/object_detection/tensorflow/ssd-resnet34/requirements.txt)
-        do 
+        do
           pip install $line
         done
       fi
@@ -669,7 +684,7 @@ function transformer_lt_official() {
     cp ${MOUNT_INTELAI_MODELS_SOURCE}/${MODE}/${PRECISION}/infer_ab.py \
         ${MOUNT_EXTERNAL_MODELS_SOURCE}/official/transformer/infer_ab.py
 
-    CMD="${CMD} 
+    CMD="${CMD}
     --in_graph=${IN_GRAPH} \
     --vocab_file=${DATASET_LOCATION}/${vocab_file} \
     --file=${DATASET_LOCATION}/${file} \
@@ -776,6 +791,8 @@ if [ ${MODEL_NAME} == "dcgan" ]; then
   dcgan
 elif [ ${MODEL_NAME} == "draw" ]; then
   draw
+elif [ ${MODEL_NAME} == "facenet" ]; then
+  facenet
 elif [ ${MODEL_NAME} == "faster_rcnn" ]; then
   faster_rcnn
 elif [ ${MODEL_NAME} == "gnmt" ]; then
diff --git a/benchmarks/face_detection_and_alignment/tensorflow/facenet/README.md b/benchmarks/face_detection_and_alignment/tensorflow/facenet/README.md
@@ -0,0 +1,156 @@
+# FaceNet
+
+This document has instructions for how to run FaceNet for the
+following modes/precisions:
+* [FP32 inference](#fp32-inference-instructions)
+
+Benchmarking instructions and scripts for model training and inference
+other precisions are coming later.
+
+## FP32 Inference Instructions
+
+1. Clone the [davidsandberg/facenet](https://github.com/davidsandberg/facenet) repository:
+
+```
+$ git clone https://github.com/davidsandberg/facenet.git
+```
+
+2. Clone this [intelai/models](https://github.com/IntelAI/models)
+repository:
+
+```
+$ git clone https://github.com/IntelAI/models.git
+```
+
+3. Download and extract the pre-trained model.
+```
+$ wget https://storage.googleapis.com/intel-optimized-tensorflow/models/facenet_fp32_pretrained_model.tar.gz
+$ tar -zxvf facenet_fp32_pretrained_model.tar.gz
+$ ls checkpoint
+model-20181015-153825.ckpt-90.data-00000-of-00001  model-20181015-153825.ckpt-90.index  model-20181015-153825.meta
+```
+
+4. If you would like to run FaceNet FP32 inference, you will need the aligned LFW dataset.
+Instructions for downloading the dataset and converting it can be found in the documentation
+[here](https://github.com/davidsandberg/facenet/wiki/Validate-on-lfw), step 2 to step 4.
+
+5. Navigate to the `benchmarks` directory in your local clone of
+the [intelai/models](https://github.com/IntelAI/models) repo from step 2.
+The `launch_benchmark.py` script in the `benchmarks` directory is
+used for starting a benchmarking run in a optimized TensorFlow docker
+container. It has arguments to specify which model, framework, mode,
+precision, and docker image.
+
+Substitute in your own `--checkpoint` pretrained model file path (from step 3),
+and `--data-location` (from step 4).
+
+FaceNet can be run for latency benchmarking, throughput
+benchmarking, or accuracy. Use one of the following examples below,
+depending on your use case.
+
+* For latency (using `--batch-size 1`):
+
+```
+python launch_benchmark.py \
+    --model-name facenet \
+    --precision fp32 \
+    --mode inference \
+    --framework tensorflow \
+    --batch-size 1 \
+    --socket-id 0 \
+    --checkpoint /home/<user>/checkpoints \
+    --data-location  /home/<user>/dataset \
+    --model-source-dir /home/<user>/facenet/ \
+    --docker-image intelaipg/intel-optimized-tensorflow:latest-devel-mkl
+```
+Example log tail when benchmarking for latency:
+```
+Batch 979 elapsed Time 0.0297989845276
+Batch 989 elapsed Time 0.029657125473
+Batch 999 elapsed Time 0.0295519828796
+Batchsize: 1
+Time spent per BATCH: 30.1561 ms
+Total samples/sec: 33.1608 samples/s
+2019-03-28 21:00:02.725536: W tensorflow/core/kernels/queue_base.cc:277] _2_input_producer: Skipping cancelled enqueue attempt with queue not closed
+2019-03-28 21:00:02.725672: W tensorflow/core/kernels/queue_base.cc:277] _1_batch_join/fifo_queue: Skipping cancelled enqueue attempt with queue not closed
+2019-03-28 21:00:02.725683: W tensorflow/core/kernels/queue_base.cc:277] _1_batch_join/fifo_queue: Skipping cancelled enqueue attempt with queue not closed
+2019-03-28 21:00:02.725693: W tensorflow/core/kernels/queue_base.cc:277] _1_batch_join/fifo_queue: Skipping cancelled enqueue attempt with queue not closed
+2019-03-28 21:00:02.725713: W tensorflow/core/kernels/queue_base.cc:277] _1_batch_join/fifo_queue: Skipping cancelled enqueue attempt with queue not closed
+2019-03-28 21:00:02.725722: W tensorflow/core/kernels/queue_base.cc:277] _1_batch_join/fifo_queue: Skipping cancelled enqueue attempt with queue not closed
+2019-03-28 21:00:02.725746: W tensorflow/core/kernels/queue_base.cc:277] _1_batch_join/fifo_queue: Skipping cancelled enqueue attempt with queue not closed
+2019-03-28 21:00:02.725776: W tensorflow/core/kernels/queue_base.cc:277] _1_batch_join/fifo_queue: Skipping cancelled enqueue attempt with queue not closed
+lscpu_path_cmd = command -v lscpu
+lscpu located here: /usr/bin/lscpu
+Ran inference with batch size 1
+Log location outside container: {--output-dir value}/benchmark_facenet_inference_fp32_20190328_205911.log
+```
+
+* For throughput (using `--batch-size 100`):
+
+```
+python launch_benchmark.py \
+    --model-name facenet \
+    --precision fp32 \
+    --mode inference \
+    --framework tensorflow \
+    --batch-size 100 \
+    --socket-id 0 \
+    --checkpoint /home/<user>/checkpoints \
+    --data-location  /home/<user>/dataset \
+    --model-source-dir /home/<user>/facenet/ \
+    --docker-image intelaipg/intel-optimized-tensorflow:latest-devel-mkl
+```
+Example log tail when benchmarking for throughput:
+```
+Batch 219 elapsed Time 0.446497917175
+Batch 229 elapsed Time 0.422048091888
+Batch 239 elapsed Time 0.433968067169
+Batchsize: 100
+Time spent per BATCH: 434.5414 ms
+Total samples/sec: 230.1277 samples/s
+Accuracy: 0.98833+-0.00489
+Validation rate: 0.96200+-0.01968 @ FAR=0.00100
+Area Under Curve (AUC): 0.999
+Equal Error Rate (EER): 0.011
+lscpu_path_cmd = command -v lscpu
+lscpu located here: /usr/bin/lscpu
+Ran inference with batch size 100
+Log location outside container: {--output-dir value}/benchmark_facenet_inference_fp32_20190329_002623.log
+```
+
+* For accuracy (using `--accuracy-only`, and `--batch-size 100`):
+
+```
+python launch_benchmark.py \
+    --model-name facenet \
+    --precision fp32 \
+    --mode inference \
+    --framework tensorflow \
+    --accuracy-only \
+    --batch-size 100 \
+    --socket-id 0 \
+    --checkpoint /home/<user>/checkpoints \
+    --data-location  /home/<user>/dataset \
+    --model-source-dir /home/<user>/facenet/ \
+    --docker-image intelaipg/intel-optimized-tensorflow:latest-devel-mkl
+```
+Example log tail when benchmarking for accuracy:
+```
+Batch 219 elapsed Time 0.398629188538
+Batch 229 elapsed Time 0.354953050613
+Batch 239 elapsed Time 0.366483926773
+Batchsize: 100
+Time spent per BATCH: 388.5419 ms
+Total samples/sec: 257.3725 samples/s
+Accuracy: 0.98833+-0.00489
+Validation rate: 0.96200+-0.01968 @ FAR=0.00100
+Area Under Curve (AUC): 0.999
+Equal Error Rate (EER): 0.011
+lscpu_path_cmd = command -v lscpu
+lscpu located here: /usr/bin/lscpu
+Ran inference with batch size 100
+Log location outside container: {--output-dir value}/benchmark_facenet_inference_fp32_20190328_214145.log
+```
+
+Note that the `--verbose` or `--output-dir` flag can be added to any of the above commands
+to get additional debug output or change the default output location..
diff --git a/benchmarks/face_detection_and_alignment/tensorflow/facenet/__init__.py b/benchmarks/face_detection_and_alignment/tensorflow/facenet/__init__.py
@@ -0,0 +1,19 @@
+#
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2019 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: EPL-2.0
+#
diff --git a/benchmarks/face_detection_and_alignment/tensorflow/facenet/inference/__init__.py b/benchmarks/face_detection_and_alignment/tensorflow/facenet/inference/__init__.py
@@ -0,0 +1,19 @@
+#
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2019 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: EPL-2.0
+#
diff --git a/benchmarks/face_detection_and_alignment/tensorflow/facenet/inference/fp32/__init__.py b/benchmarks/face_detection_and_alignment/tensorflow/facenet/inference/fp32/__init__.py
@@ -0,0 +1,19 @@
+#
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2019 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: EPL-2.0
+#
diff --git a/benchmarks/face_detection_and_alignment/tensorflow/facenet/inference/fp32/model_init.py b/benchmarks/face_detection_and_alignment/tensorflow/facenet/inference/fp32/model_init.py
@@ -0,0 +1,91 @@
+#
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2019 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: EPL-2.0
+#
+
+from common.base_model_init import BaseModelInitializer
+from common.base_model_init import set_env_var
+
+import os
+from argparse import ArgumentParser
+
+
+class ModelInitializer(BaseModelInitializer):
+    """initialize mode and run benchmark for FaceNet model"""
+
+    def __init__(self, args, custom_args=[], platform_util=None):
+        super(ModelInitializer, self).__init__(args, custom_args, platform_util)
+        self.cmd = self.get_numactl_command(self.args.socket_id) + \
+            self.python_exe + " "
+
+        # Set KMP env vars, if they haven't already been set
+        self.set_kmp_vars()
+
+        pairs_file = os.path.join(self.args.model_source_dir,
+                                  "data/pairs.txt")
+        arg_parser = ArgumentParser(description='Parse custom args')
+        arg_parser.add_argument(
+            "--lfw_pairs", type=str,
+            help="The file containing the pairs to use for validation.",
+            dest="lfw_pairs", default=pairs_file)
+        self.args = arg_parser.parse_args(
+            self.custom_args, namespace=self.args)
+
+        # use default batch size if -1
+        if self.args.batch_size == -1 or self.args.accuracy_only:
+            self.args.batch_size = 100
+
+        # set num_inter_threads and num_intra_threads
+        if self.args.batch_size > 32:
+            self.set_num_inter_intra_threads(num_inter_threads=2)
+        else:
+            self.set_num_inter_intra_threads(num_inter_threads=1)
+
+        set_env_var("OMP_NUM_THREADS", self.args.num_intra_threads)
+
+        run_script = os.path.join(self.args.model_source_dir,
+                                  "src/validate_on_lfw.py")
+
+        warmup_steps = 40
+        max_steps = 1000
+        if self.args.batch_size == 1:
+            warmup_steps = 200
+
+        cmd_args = ' ' + self.args.data_location + \
+                   ' ' + self.args.checkpoint + ' --distance_metric 1' + \
+                   ' --use_flipped_images' + ' --subtract_mean' + \
+                   ' --use_fixed_image_standardization' + \
+                   ' --num_inter_threads=' + \
+                   str(self.args.num_inter_threads) + \
+                   ' --num_intra_threads=' + \
+                   str(self.args.num_intra_threads) + \
+                   ' --lfw_batch_size=' + str(self.args.batch_size) + \
+                   ' --lfw_pairs=' + self.args.lfw_pairs + \
+                   ' --warmup_steps=' + str(warmup_steps) + \
+                   ' --max_steps=' + str(max_steps)
+
+        self.cmd = self.cmd + run_script + cmd_args
+
+    def run(self):
+        """run command to enable model benchmark or accuracy measurement"""
+
+        original_dir = os.getcwd()
+        os.chdir(self.args.model_source_dir)
+        if self.cmd:
+            self.run_command(self.cmd)
+        os.chdir(original_dir)
diff --git a/models/face_detection_and_alignment/tensorflow/facenet/__init__.py b/models/face_detection_and_alignment/tensorflow/facenet/__init__.py
@@ -0,0 +1,20 @@
+#
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2019 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: EPL-2.0
+#
+
diff --git a/models/face_detection_and_alignment/tensorflow/facenet/fp32/validate_on_lfw.py b/models/face_detection_and_alignment/tensorflow/facenet/fp32/validate_on_lfw.py
diff --git a/tests/unit/common/tensorflow/tf_model_args.txt b/tests/unit/common/tensorflow/tf_model_args.txt
