Refactoring, adding common utils library w/ method for saving/caching PyTorch embeddings

Author: MarkhamLee

benchmarking/common/__init__.py benchmarking/benchmarking_utils/__init__.py

@@ -8,6 +8,7 @@
 # precomputed tensors. The script will return a dataframe with your testing
 # data in it as well as write it to a log file.
 import os
+import sys
 import torch
 import pandas as pd
 import torch.nn.functional as F
@@ -16,7 +17,11 @@
 from PIL import Image
 from time import time
 
-from common.logging_util import LoggingUtilities  # noqa: E402
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from common_utils.logging_util import LoggingUtilities  # noqa: E402
+from common_utils.general_utilities import GeneralUtils  # noqa: E402
 
 
 class FacenetBenchmarking:
@@ -26,15 +31,17 @@ def __init__(self, photo_path: str, tensor_path: str):
         self.logger = LoggingUtilities.\
             log_file_logger("facenet_benchmarking")
 
+        self.utilities = GeneralUtils()
+
         # check for cuda, configure cudnn
         self.cuda_check()
 
         # get models
         self.mtcnn, self.resnet = self.get_models()\
 
         # get files
-        self.photo_files = self.get_file_lists(photo_path)
-        self.tensor_files = self.get_file_lists(tensor_path)
+        self.photo_files = self.utilities.get_file_list(photo_path)
+        self.tensor_files = self.utilities.get_file_list(tensor_path)
 
         self.logger.info('File lists created')
 

benchmarking/facenet_benchmarking.py

@@ -0,0 +1,31 @@
+import os
+from torch import save as ts
+from common_utils.logging_util import LoggingUtilities
+
+logger = LoggingUtilities.console_out_logger("general utils")
+
+
+class GeneralUtils():
+
+    def __init__(self):
+
+        pass
+
+    @staticmethod
+    def get_file_list(path: str):
+
+        file_list = list()
+
+        for (dirpath, dirnames, filenames) in os.walk(path):
+            filenames.sort()
+            file_list += [os.path.join(dirpath, file) for file in filenames]
+
+        return file_list
+
+    # save PyTorch tensors
+    @staticmethod
+    def save_pytorch_tensors(tensor: object, path: str):
+
+        ts(tensor,  path)
+
+        logger.info(f'Tensor saved to: {path}')