Boosted inference speed

Author: antoinecharbel-inmind

src/main/deep_learning_service.py

@@ -1,7 +1,7 @@
 import os
+import re
 import json
 import uuid
-import re
 from inference.inference_engines_factory import InferenceEngineFactory
 from inference.exceptions import ModelNotFound, InvalidModelConfiguration, ModelNotLoaded, InferenceEngineNotFound, \
     InvalidInputData, ApplicationError
@@ -75,14 +75,14 @@ def load_models(self, model_names):
         for model in model_names:
             self.load_model(model)
 
-    async def run_model(self, model_name, input_data, draw_boxes, predict_batch):
+    async def run_model(self, model_name, input_data, draw, predict_batch):
         """
         Loads the model in case it was never loaded and calls the inference engine class to get a prediction.
         :param model_name: Model name
         :param input_data: Batch of images or a single image
-        :param draw_boxes: Boolean to specify if we need to draw the response on the input image
+        :param draw: Boolean to specify if we need to draw the response on the input image
         :param predict_batch: Boolean to specify if there is a batch of images in a request or not
-        :return: Model response in case draw_boxes was set to False, else an actual image
+        :return: Model response in case draw was set to False, else an actual image
         """
         if re.match(r'[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}', model_name,
                     flags=0):
@@ -92,18 +92,18 @@ async def run_model(self, model_name, input_data, draw_boxes, predict_batch):
         if self.model_loaded(model_name):
             try:
                 if predict_batch:
-                    return await self.models_dict[model_name].run_batch(input_data, draw_boxes, predict_batch)
+                    return await self.models_dict[model_name].run_batch(input_data, draw, predict_batch)
                 else:
-                    if not draw_boxes:
-                        return await self.models_dict[model_name].run(input_data, draw_boxes, predict_batch)
+                    if not draw:
+                        return await self.models_dict[model_name].infer(input_data, draw, predict_batch)
                     else:
-                        await self.models_dict[model_name].run(input_data, draw_boxes, predict_batch)
+                        await self.models_dict[model_name].infer(input_data, draw, predict_batch)
             except ApplicationError as e:
                 raise e
         else:
             try:
                 self.load_model(model_name)
-                return await self.run_model(model_name, input_data, draw_boxes, predict_batch)
+                return await self.run_model(model_name, input_data, draw, predict_batch)
             except ApplicationError as e:
                 raise e
 

src/main/inference/__init__.py

@@ -12,7 +12,7 @@ def __init__(self):
         """
         self.logger = logging.getLogger('logger')
         date = datetime.now().strftime('%Y-%m-%d')
-        file_path = 'logs/yolov3_inference_engine_' + date + '.log'
+        file_path = 'logs/' + date + '.log'
         self.handler = logging.FileHandler(file_path)
         self.handler.setLevel(logging.INFO)
         self.handler.setFormatter(logging.Formatter("%(levelname)s;%(asctime)s;%(message)s"))

src/main/inference/base_error.py

@@ -4,89 +4,89 @@
 
 class AbstractInferenceEngine(ABC):
 
-    def __init__(self, model_path):
-        """
-        Takes a model path and calls the load function.
-        :param model_path: The model's path
-        :return:
-        """
-        self.labels = []
-        self.configuration = {}
-        self.model_path = model_path
-        try:
-            self.validate_configuration()
-        except ApplicationError as e:
-            raise e
-        try:
-            self.load()
-        except ApplicationError as e:
-            raise e
-        except Exception as e:
-            raise ModelNotLoaded()
+	def __init__(self, model_path):
+		"""
+		Takes a model path and calls the load function.
+		:param model_path: The model's path
+		:return:
+		"""
+		self.labels = []
+		self.configuration = {}
+		self.model_path = model_path
+		try:
+			self.validate_configuration()
+		except ApplicationError as e:
+			raise e
+		try:
+			self.load()
+		except ApplicationError as e:
+			raise e
+		except Exception as e:
+			raise ModelNotLoaded()
 
-    @abstractmethod
-    def load(self):
-        """
-        Loads the model based on the underlying implementation.
-        """
-        pass
+	@abstractmethod
+	def load(self):
+		"""
+		Loads the model based on the underlying implementation.
+		"""
+		pass
 
-    @abstractmethod
-    def free(self):
-        """
-        Performs any manual memory implementation required to when unloading a model.
-        Will be called when the class's destructor is called.
-        """
-        pass
+	@abstractmethod
+	def free(self):
+		"""
+		Performs any manual memory implementation required to when unloading a model.
+		Will be called when the class's destructor is called.
+		"""
+		pass
 
-    @abstractmethod
-    async def run(self, input_data, draw_boxes, predict_batch):
-        """
-        Performs the required inference based on the underlying implementation of this class.
-        Could be used to return classification predictions, object detection coordinates...
-        :param predict_batch: Boolean
-        :param input_data: A single image
-        :param draw_boxes: Used to draw bounding boxes on image instead of returning them
-        :return: A bounding-box
-        """
-        pass
+	@abstractmethod
+	async def infer(self, input_data, draw, predict_batch):
+		"""
+		Performs the required inference based on the underlying implementation of this class.
+		Could be used to return classification predictions, object detection coordinates...
+		:param predict_batch: Boolean
+		:param input_data: A single image
+		:param draw: Used to draw bounding boxes on image instead of returning them
+		:return: A bounding-box
+		"""
+		pass
 
-    @abstractmethod
-    async def run_batch(self, input_data, draw_boxes, predict_batch):
-        """
-        Iterates over images and returns a prediction for each one.
-        :param predict_batch: Boolean
-        :param input_data: List of images
-        :param draw_boxes: Used to draw bounding boxes on image instead of returning them
-        :return: List of bounding-boxes
-        """
-        pass
+	@abstractmethod
+	async def run_batch(self, input_data, draw, predict_batch):
+		"""
+		Iterates over images and returns a prediction for each one.
+		:param predict_batch: Boolean
+		:param input_data: List of images
+		:param draw: Used to draw bounding boxes on image instead of returning them
+		:return: List of bounding-boxes
+		"""
+		pass
 
-    @abstractmethod
-    def validate_configuration(self):
-        """
-        Validates that the model and its files are valid based on the underlying implementation's requirements.
-        Can check for configuration values, folder structure...
-        """
-        pass
+	@abstractmethod
+	def validate_configuration(self):
+		"""
+		Validates that the model and its files are valid based on the underlying implementation's requirements.
+		Can check for configuration values, folder structure...
+		"""
+		pass
 
-    @abstractmethod
-    def set_configuration(self, data):
-        """
-        Takes the configuration from the config.json file
-        :param data: Json data
-        :return:
-        """
-        pass
+	@abstractmethod
+	def set_model_configuration(self, data):
+		"""
+		Takes the configuration from the config.json file
+		:param data: Json data
+		:return:
+		"""
+		pass
 
-    @abstractmethod
-    def validate_json_configuration(self, data):
-        """
-        Validates the configuration of the config.json file.
-        :param data: Json data
-        :return:
-        """
-        pass
+	@abstractmethod
+	def validate_json_configuration(self, data):
+		"""
+		Validates the configuration of the config.json file.
+		:param data: Json data
+		:return:
+		"""
+		pass
 
-    def __del__(self):
-        self.free()
+	def __del__(self):
+		self.free()

src/main/inference/base_inference_engine.py

@@ -1,6 +1,6 @@
 import os
 import logging
-from datetime import datetime, date
+from datetime import datetime
 from inference.base_error import AbstractError
 
 
@@ -31,15 +31,15 @@ def check_date(self):
         :return:
         """
         self.date = datetime.now().strftime('%Y-%m-%d')
-        file_path = 'yolov3_inference_engine_' + self.date + '.log'
+        file_path = self.date + '.log'
         if file_path not in os.listdir('logs'):
             self.logger.removeHandler(self.handler)
             self.handler = logging.FileHandler('logs/' + file_path)
             self.handler.setLevel(logging.INFO)
             self.handler.setFormatter(logging.Formatter("%(levelname)s;%(asctime)s;%(message)s"))
             self.logger.addHandler(self.handler)
         oldest_log_file = os.listdir('logs')[0]
-        oldest_date = oldest_log_file.split("_")[3].split('.')[0]
+        oldest_date = oldest_log_file.split('.')[0]
         a = datetime.strptime(datetime.now().strftime('%Y-%m-%d'), '%Y-%m-%d')
         b = datetime.strptime(oldest_date, '%Y-%m-%d')
         delta = a - b

src/main/inference/errors.py

@@ -2,55 +2,50 @@
 
 
 class ApplicationError(Exception):
-    """Base class for other exceptions"""
+	"""Base class for other exceptions"""
 
-    def __init__(self, default_message, additional_message=''):
-        self.default_message = default_message
-        self.additional_message = additional_message
+	def __init__(self, default_message, additional_message=''):
+		self.default_message = default_message
+		self.additional_message = additional_message
 
-    def __str__(self):
-        return self.get_message()
+	def __str__(self):
+		return self.get_message()
 
-    def get_message(self):
-        return self.default_message if self.additional_message == '' else "{}: {}".format(self.default_message,
-                                                                                          self.additional_message)
+	def get_message(self):
+		return self.default_message if self.additional_message == '' else "{}: {}".format(self.default_message,
+																						  self.additional_message)
 
 
 class InvalidModelConfiguration(ApplicationError):
-    """Raised when the model's configuration is corrupted"""
+	"""Raised when the model's configuration is corrupted"""
 
-    def __init__(self, additional_message=''):
-        # super('Invalid model configuration', additional_message)
-        super().__init__('Invalid model configuration', additional_message)
+	def __init__(self, additional_message=''):
+		super().__init__('Invalid model configuration', additional_message)
 
 
 class ModelNotFound(ApplicationError):
-    """Raised when the model is not found"""
+	"""Raised when the model is not found"""
 
-    def __init__(self, additional_message=''):
-        # super('Model not found', additional_message)
-        super().__init__('Model not found', additional_message)
+	def __init__(self, additional_message=''):
+		super().__init__('Model not found', additional_message)
 
 
 class ModelNotLoaded(ApplicationError):
-    """Raised when the model is not loaded"""
+	"""Raised when the model is not loaded"""
 
-    def __init__(self, additional_message=''):
-        # super('Error loading model', additional_message)
-        super().__init__('Error loading model', additional_message)
+	def __init__(self, additional_message=''):
+		super().__init__('Error loading model', additional_message)
 
 
 class InvalidInputData(ApplicationError):
-    """Raised when the input data is corrupted"""
+	"""Raised when the input data is corrupted"""
 
-    def __init__(self, additional_message=''):
-        # super('Invalid input data', additional_message)
-        super().__init__('Invalid input data', additional_message)
+	def __init__(self, additional_message=''):
+		super().__init__('Invalid input data', additional_message)
 
 
 class InferenceEngineNotFound(ApplicationError):
-    """Raised when the Inference Engine is not found"""
+	"""Raised when the Inference Engine is not found"""
 
-    def __init__(self, additional_message=''):
-        # super('Inference engine not found', additional_message)
-        super().__init__('Inference engine not found', additional_message)
+	def __init__(self, additional_message=''):
+		super().__init__('Inference engine not found', additional_message)

src/main/inference/exceptions.py

@@ -1,32 +1,33 @@
 import os
 import json
-from inference.exceptions import ModelNotFound, ApplicationError, InvalidModelConfiguration, InferenceEngineNotFound, ModelNotLoaded
+from inference.exceptions import ModelNotFound, ApplicationError, InvalidModelConfiguration, InferenceEngineNotFound, \
+	ModelNotLoaded
 
 
 class InferenceEngineFactory:
 
-    @staticmethod
-    def get_engine(path_to_model):
-        """
-        Reads the model's inference engine from the model's configuration and calls the right inference engine class.
-        :param path_to_model: Model's path
-        :return: The model's instance
-        """
-        if not os.path.exists(path_to_model):
-            raise ModelNotFound()
-        try:
-            configuration = json.loads(open(os.path.join(path_to_model, 'config.json')).read())
-        except Exception:
-            raise InvalidModelConfiguration('config.json not found or corrupted')
-        try:
-            inference_engine_name = configuration['inference_engine_name']
-        except Exception:
-            raise InvalidModelConfiguration('missing inference engine name in config.json')
-        try:
-            # import one of the available inference engine class (in this project there's only one), and return a
-            # model instance
-            return getattr(__import__(inference_engine_name), 'InferenceEngine')(path_to_model)
-        except ApplicationError as e:
-            raise e
-        except Exception as e:
-            raise InferenceEngineNotFound(inference_engine_name)
+	@staticmethod
+	def get_engine(path_to_model):
+		"""
+		Reads the model's inference engine from the model's configuration and calls the right inference engine class.
+		:param path_to_model: Model's path
+		:return: The model's instance
+		"""
+		if not os.path.exists(path_to_model):
+			raise ModelNotFound()
+		try:
+			configuration = json.loads(open(os.path.join(path_to_model, 'config.json')).read())
+		except Exception:
+			raise InvalidModelConfiguration('config.json not found or corrupted')
+		try:
+			inference_engine_name = configuration['inference_engine_name']
+		except Exception:
+			raise InvalidModelConfiguration('missing inference engine name in config.json')
+		try:
+			# import one of the available inference engine class (in this project there's only one), and return a
+			# model instance
+			return getattr(__import__(inference_engine_name), 'InferenceEngine')(path_to_model)
+		except ApplicationError as e:
+			raise e
+		except Exception as e:
+			raise InferenceEngineNotFound(inference_engine_name)