Fix mark lost jobs, log exceptions inside worker

Author: davegaeddert

bolt-jobs/bolt/jobs/models.py

@@ -98,11 +98,7 @@ def mark_lost_jobs(self):
         # Note that this will save it in the results,
         # but lost jobs are only retried if they have a retry!
         for job in lost_jobs:
-            job.convert_to_result(
-                ended_at=now,
-                error="",
-                status=JobResultStatuses.LOST,
-            )
+            job.convert_to_result(status=JobResultStatuses.LOST)
 
 
 class Job(models.Model):

bolt-jobs/bolt/jobs/workers.py

@@ -35,8 +35,13 @@ def run(self):
 
         try:
             while True:
-                self.maybe_log_stats()
-                self.maybe_check_job_results()
+                try:
+                    self.maybe_log_stats()
+                    self.maybe_check_job_results()
+                except Exception as e:
+                    # Log the issue, but don't stop the worker
+                    # (these tasks are kind of ancilarry to the main job processing)
+                    logger.exception(e)
 
                 with transaction.atomic():
                     job_request = JobRequest.objects.next_up()
@@ -63,14 +68,9 @@ def run(self):
                 del job
 
                 self.executor.submit(process_job, job_uuid)
-
         except (KeyboardInterrupt, SystemExit):
             self.executor.shutdown(wait=True, cancel_futures=True)
 
-        except Exception as e:
-            # Some kind of error in the job system...
-            logger.exception(e)
-
     def maybe_log_stats(self):
         if not self.stats_every:
             return