Merge pull request #25 from Paperspace/PS-10385-Refactor-logger-module

Refactor logger module

Author: mkulaczkowski

gradient/client.py

@@ -8,12 +8,13 @@
 
 
 class API(object):
-    def __init__(self, api_url, headers=None, api_key=None):
+    def __init__(self, api_url, headers=None, api_key=None, logger_=logger.Logger()):
         self.api_url = api_url
         headers = headers or default_headers
         self.headers = headers.copy()
         if api_key:
             self.api_key = api_key
+        self.logger = logger_
 
     @property
     def api_key(self):
@@ -30,36 +31,36 @@ def get_path(self, url):
 
     def post(self, url, json=None, params=None, files=None, data=None):
         path = self.get_path(url)
-        logger.debug("POST request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}\n\tfiles: {}\n\tdata: {}"
+        self.logger.debug("POST request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}\n\tfiles: {}\n\tdata: {}"
                      .format(path, self.headers, json, params, files, data))
         response = requests.post(path, json=json, params=params, headers=self.headers, files=files, data=data)
-        logger.debug("Response status code: {}".format(response.status_code))
-        logger.debug("Response content: {}".format(response.content))
+        self.logger.debug("Response status code: {}".format(response.status_code))
+        self.logger.debug("Response content: {}".format(response.content))
         return response
 
     def put(self, url, json=None, params=None):
         path = self.get_path(url)
-        logger.debug("PUT request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}"
+        self.logger.debug("PUT request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}"
                      .format(path, self.headers, json, params))
         response = requests.put(path, json=json, params=params, headers=self.headers)
-        logger.debug("Response status code: {}".format(response.status_code))
-        logger.debug("Response content: {}".format(response.content))
+        self.logger.debug("Response status code: {}".format(response.status_code))
+        self.logger.debug("Response content: {}".format(response.content))
         return response
 
     def get(self, url, json=None, params=None):
         path = self.get_path(url)
-        logger.debug("GET request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}"
+        self.logger.debug("GET request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}"
                      .format(path, self.headers, json, params))
         response = requests.get(path, params=params, headers=self.headers, json=json)
-        logger.debug("Response status code: {}".format(response.status_code))
-        logger.debug("Response content: {}".format(response.content))
+        self.logger.debug("Response status code: {}".format(response.status_code))
+        self.logger.debug("Response content: {}".format(response.content))
         return response
 
     def delete(self, url, json=None, params=None):
         path = self.get_path(url)
         response = requests.delete(path, params=params, headers=self.headers, json=json)
-        logger.debug("DELETE request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}"
+        self.logger.debug("DELETE request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}"
                      .format(response.url, self.headers, json, params))
-        logger.debug("Response status code: {}".format(response.status_code))
-        logger.debug("Response content: {}".format(response.content))
+        self.logger.debug("Response status code: {}".format(response.status_code))
+        self.logger.debug("Response content: {}".format(response.content))
         return response

gradient/commands/common.py

@@ -9,7 +9,7 @@
 
 
 class CommandBase(object):
-    def __init__(self, api=None, logger_=logger):
+    def __init__(self, api=None, logger_=logger.Logger()):
         self.api = api
         self.logger = logger_
 

gradient/commands/experiments.py

@@ -6,7 +6,6 @@
 
 from gradient import logger, constants, client, config
 from gradient.commands import common
-from gradient.logger import log_response
 from gradient.utils import get_terminal_lines
 from gradient.workspace import S3WorkspaceHandler
 
@@ -16,7 +15,7 @@
 class ExperimentCommand(common.CommandBase):
     def __init__(self, workspace_handler=None, **kwargs):
         super(ExperimentCommand, self).__init__(**kwargs)
-        self._workspace_handler = workspace_handler or S3WorkspaceHandler(experiments_api=self.api, logger=self.logger)
+        self._workspace_handler = workspace_handler or S3WorkspaceHandler(experiments_api=self.api, logger_=self.logger)
 
     def _log_create_experiment(self, response, success_msg_template, error_msg):
         if response.ok:
@@ -62,16 +61,16 @@ def execute(self, json_):
         return None
 
 
-def start_experiment(experiment_id, api=experiments_api):
+def start_experiment(experiment_id, api=experiments_api, logger_=logger.Logger()):
     url = "/experiments/{}/start/".format(experiment_id)
     response = api.put(url)
-    log_response(response, "Experiment started", "Unknown error while starting the experiment")
+    logger_.log_response(response, "Experiment started", "Unknown error while starting the experiment")
 
 
-def stop_experiment(experiment_id, api=experiments_api):
+def stop_experiment(experiment_id, api=experiments_api, logger_=logger.Logger()):
     url = "/experiments/{}/stop/".format(experiment_id)
     response = api.put(url)
-    log_response(response, "Experiment stopped", "Unknown error while stopping the experiment")
+    logger_.log_response(response, "Experiment stopped", "Unknown error while stopping the experiment")
 
 
 class ListExperimentsCommand(common.ListCommand):
@@ -165,7 +164,7 @@ def _make_details_table(experiment):
     return table_string
 
 
-def get_experiment_details(experiment_id, api=experiments_api):
+def get_experiment_details(experiment_id, api=experiments_api, logger_=logger.Logger()):
     url = "/experiments/{}/".format(experiment_id)
     response = api.get(url)
     details = response.content
@@ -174,9 +173,9 @@ def get_experiment_details(experiment_id, api=experiments_api):
             experiment = response.json()["data"]
             details = _make_details_table(experiment)
         except (ValueError, KeyError) as e:
-            logger.error("Error parsing response data")
+            logger_.error("Error parsing response data")
 
-    log_response(response, details, "Unknown error while retrieving details of the experiment")
+    logger_.log_response(response, details, "Unknown error while retrieving details of the experiment")
 
 
 class ExperimentLogsCommand(common.CommandBase):

gradient/commands/jobs.py

@@ -146,7 +146,7 @@ def _make_table(self, logs, table, table_data):
 class CreateJobCommand(JobsCommandBase):
     def __init__(self, workspace_handler=None, **kwargs):
         super(CreateJobCommand, self).__init__(**kwargs)
-        self._workspace_handler = workspace_handler or WorkspaceHandler(logger=self.logger)
+        self._workspace_handler = workspace_handler or WorkspaceHandler(logger_=self.logger)
 
     def execute(self, json_):
         url = "/jobs/createJob/"

gradient/commands/run.py

@@ -9,7 +9,7 @@
 
 class RunCommand(object):
 
-    def __init__(self, api=None, logger_=logger):
+    def __init__(self, api=None, logger_=logger.Logger()):
         self.api = api
         self.logger = logger_
 

gradient/logger.py

@@ -1,70 +1,70 @@
 import click
-from six import string_types
+import six
 
 from .config import config
 
 
-def _log(message, color=None, err=False):
-    message = str(message)
-    color = color if config.USE_CONSOLE_COLORS else None
-    click.secho(message, fg=color, err=err)
-
-
-def log(message, color=None, err=False):
-    _log(message, color=color, err=err)
-
-
-def error(message):
-    color = "red" if config.USE_CONSOLE_COLORS else None
-    _log(message, color=color, err=True)
-
-
-def warning(message):
-    color = "yellow" if config.USE_CONSOLE_COLORS else None
-    _log(message, color=color)
-
-
-def log_error_response(data):
-    error_str = data.get("error")
-    details = data.get("details")
-    message = data.get("message")
-
-    if not any((error_str, details, message)):
-        raise ValueError("No error messages found")
-
-    if error_str:
-        try:
-            error(error_str["message"])
-        except (KeyError, TypeError):
-            error(str(error_str))
-
-    if details:
-        if isinstance(details, dict):
-            for key, val in details.items():
-                if isinstance(val, string_types):
-                    val = [val]
-
-                for v in val:
-                    msg = "{}: {}".format(key, str(v))
-                    error(msg)
+class Logger(object):
+    def _log(self, message, color=None, err=False):
+        message = str(message)
+        color = color if config.USE_CONSOLE_COLORS else None
+        click.secho(message, fg=color, err=err)
+
+    def log(self, message, color=None, err=False):
+        self._log(message, color=color, err=err)
+
+    def error(self, message):
+        color = "red" if config.USE_CONSOLE_COLORS else None
+        self._log(message, color=color, err=True)
+
+    def warning(self, message):
+        color = "yellow" if config.USE_CONSOLE_COLORS else None
+        self._log(message, color=color)
+
+    def log_error_response(self, data):
+        messages = list(self._get_error_messages(data))
+        msg = "\n".join(messages)
+
+        if not msg:
+            raise ValueError("No error messages found")
+
+        self.error(msg)
+
+    def _get_error_messages(self, data, add_prefix=False):
+        if isinstance(data, dict):
+            for key, value in sorted(data.items()):
+                if key in ("error", "errors", "message", "messages"):
+                    for message in self._get_error_messages(value):
+                        yield message
+
+                # when key == "details" and value is a dict then values should be prefixed with key
+                if add_prefix:
+                    for message in self._get_error_messages(value):
+                        # there is some useless message in data["context"]
+                        if key != "context":
+                            yield key + ": " + message
+                if key == "details":
+                    for message in self._get_error_messages(value, add_prefix=True):
+                        yield message
+
+        if isinstance(data, list):
+            for element in data:
+                for message in self._get_error_messages(element):
+                    yield message
+
+        if isinstance(data, six.string_types):
+            yield data
+
+    def debug(self, message):
+        if config.DEBUG:
+            self._log("DEBUG: {}".format(message))
+
+    def log_response(self, response, success_msg, error_msg):
+        if response.ok:
+            self._log(success_msg)
         else:
-            error(details)
-
-    if message:
-        error(str(message))
-
-
-def debug(message):
-    if config.DEBUG:
-        log("DEBUG: {}".format(message))
-
-
-def log_response(response, success_msg, error_msg):
-    if response.ok:
-        log(success_msg)
-    else:
-        try:
-            data = response.json()
-            log_error_response(data)
-        except ValueError:
-            error(error_msg)
+            try:
+                data = response.json()
+                self.log_error_response(data)
+            except ValueError:
+                self.error(error_msg)

gradient/version_checker.py

@@ -13,6 +13,9 @@
     import xmlrpc.client as xmlrpclib
 
 
+logger = logger.Logger()
+
+
 class PackageNotFoundError(Exception):
     pass