Add ability for clients to send error log to server

docs/resources/log.config

@@ -21,7 +21,7 @@ args=(sys.stdout,)
 class=FileHandler
 level=ERROR
 formatter=fullFormatter
-args=('error.log', 'a')
+args=('error_log.txt', 'a')
 
 [formatter_fullFormatter]
 format=%(asctime)s - %(name)s - %(levelname)s - %(message)s

nvflare/apis/fl_constant.py

@@ -375,6 +375,7 @@ class WorkspaceConstants:
     DEFAULT_LOGGING_CONFIG = LOGGING_CONFIG + ".default"
     AUDIT_LOG = "audit.log"
     LOG_FILE_NAME = "log.txt"
+    ERROR_LOG_FILE_NAME = "error_log.txt"
     STATS_POOL_SUMMARY_FILE_NAME = "stats_pool_summary.json"
     STATS_POOL_RECORDS_FILE_NAME = "stats_pool_records.csv"
 

nvflare/apis/workspace.py

@@ -153,6 +153,9 @@ def get_app_dir(self, job_id: str) -> str:
     def get_app_log_file_path(self, job_id: str) -> str:
         return os.path.join(self.get_run_dir(job_id), WorkspaceConstants.LOG_FILE_NAME)
 
+    def get_app_error_log_file_path(self, job_id: str) -> str:
+        return os.path.join(self.get_run_dir(job_id), WorkspaceConstants.ERROR_LOG_FILE_NAME)
+
     def get_app_config_dir(self, job_id: str) -> str:
         return os.path.join(self.get_app_dir(job_id), self.config_folder)
 

nvflare/fuel/f3/cellnet/defs.py

@@ -170,6 +170,7 @@ class CellChannelTopic:
     EXECUTE_RESULT = "execute_result"
     FIRE_EVENT = "fire_event"
     REPORT_JOB_FAILURE = "report_job_failure"
+    TRANSMIT_ERROR_LOG = "transmit_error_log"
 
     SIMULATOR_WORKER_INIT = "simulator_worker_init"
 

nvflare/private/fed/app/simulator/log.config

@@ -21,7 +21,7 @@ args=(sys.stdout,)
 class=FileHandler
 level=ERROR
 formatter=fullFormatter
-args=('error.log', 'a')
+args=('error_log.txt', 'a')
 
 [formatter_fullFormatter]
 format=%(asctime)s - %(name)s - %(levelname)s - %(message)s

nvflare/private/fed/client/client_executor.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import logging
+import os
 import threading
 import time
 from abc import ABC, abstractmethod
@@ -22,6 +23,7 @@
 from nvflare.apis.fl_context import FLContext
 from nvflare.apis.job_launcher_spec import JobLauncherSpec
 from nvflare.apis.resource_manager_spec import ResourceManagerSpec
+from nvflare.apis.workspace import Workspace
 from nvflare.fuel.common.exit_codes import PROCESS_EXIT_REASON, ProcessExitCode
 from nvflare.fuel.f3.cellnet.core_cell import FQCN
 from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode
@@ -393,6 +395,31 @@ def _wait_child_process_finish(self, client, job_id, allocated_resource, token,
             return_code = get_return_code(job_handle, job_id, workspace, self.logger)
 
             self.logger.info(f"run ({job_id}): child worker process finished with RC {return_code}")
+
+            should_report_error_log = (
+                False  # set this to True to report error log to server, todo: get value from client config
+            )
+            if should_report_error_log:
+                error_log_contents = None
+                workspace_object = Workspace(root_dir=workspace, site_name=client.client_name)
+                error_log_path = workspace_object.get_app_error_log_file_path(job_id=job_id)
+                if os.path.exists(error_log_path):
+                    with open(error_log_path, "r") as f:
+                        error_log_contents = f.read()
+                if error_log_contents:
+                    request = new_cell_message(
+                        headers={},
+                        payload=error_log_contents,
+                    )
+                    self.client.cell.fire_and_forget(
+                        targets=[FQCN.ROOT_SERVER],
+                        channel=CellChannel.SERVER_MAIN,
+                        topic=CellChannelTopic.TRANSMIT_ERROR_LOG,
+                        message=request,
+                        optional=True,
+                    )
+                    self.logger.info("Reported contents of error log to server!")
+
             if return_code in [ProcessExitCode.UNSAFE_COMPONENT, ProcessExitCode.CONFIG_ERROR]:
                 request = new_cell_message(
                     headers={},

nvflare/private/fed/server/fed_server.py

@@ -357,6 +357,12 @@ def _register_cellnet_cbs(self):
             cb=self.client_heartbeat,
         )
 
+        self.cell.register_request_cb(
+            channel=CellChannel.SERVER_MAIN,
+            topic=CellChannelTopic.TRANSMIT_ERROR_LOG,
+            cb=self.process_error_log,
+        )
+
         self.cell.register_request_cb(
             channel=CellChannel.SERVER_MAIN,
             topic=CellChannelTopic.REPORT_JOB_FAILURE,
@@ -665,6 +671,11 @@ def quit_client(self, request: Message) -> Message:
             headers = {CellMessageHeaderKeys.MESSAGE: "Removed client"}
             return self._generate_reply(headers=headers, payload=None, fl_ctx=fl_ctx)
 
+    def process_error_log(self, request: Message):
+        payload = request.payload
+        client = request.get_header(key=MessageHeaderKey.ORIGIN)
+        print(f"Error log from {client}: {payload}")
+
     def process_job_failure(self, request: Message):
         payload = request.payload
         client = request.get_header(key=MessageHeaderKey.ORIGIN)

nvflare/private/fed/utils/fed_utils.py

@@ -65,7 +65,7 @@ def add_logfile_handler(log_file: str):
     The purpose for this is to handle dynamic log file locations.
 
     If a handler named errorFileHandler is found, it will be used as a template to
-    create a new handler for writing to the error.log file at the same directory as log_file.
+    create a new handler for writing to the error log file at the same directory as log_file.
     The original errorFileHandler will be removed and replaced by the new handler.
 
     Each log file will be rotated when it reaches 20MB.
@@ -90,7 +90,7 @@ def add_logfile_handler(log_file: str):
     if not configured_error_handler:
         return
 
-    error_log_file = os.path.join(os.path.dirname(log_file), "error.log")
+    error_log_file = os.path.join(os.path.dirname(log_file), WorkspaceConstants.ERROR_LOG_FILE_NAME)
     error_file_handler = RotatingFileHandler(error_log_file, maxBytes=20 * 1024 * 1024, backupCount=10)
     error_file_handler.setLevel(configured_error_handler.level)
     error_file_handler.setFormatter(configured_error_handler.formatter)