add execution for failure processing

lib/chrono_forge/executor.rb

@@ -53,7 +53,7 @@ def perform(key, attempt: 0, options: {}, **kwargs)
         if should_retry?(e, attempt)
           self.class::RetryStrategy.schedule_retry(workflow, attempt: attempt)
         else
-          workflow.failed!
+          fail_workflow! e
         end
       ensure
         context.save!
@@ -104,6 +104,47 @@ def complete_workflow!
       end
     end
 
+    def fail_workflow!(error)
+      # Create an execution log for workflow failure
+      execution_log = ExecutionLog.create_or_find_by!(
+        workflow: workflow,
+        step_name: "$workflow_failure$"
+      ) do |log|
+        log.started_at = Time.current
+        log.metadata = {
+          workflow_id: workflow.id,
+          error_class: error.class.to_s,
+          error_message: error.message
+        }
+      end
+
+      begin
+        execution_log.update!(
+          attempts: execution_log.attempts + 1,
+          last_executed_at: Time.current
+        )
+
+        workflow.failed!
+
+        # Mark execution log as completed
+        execution_log.update!(
+          state: :completed,
+          completed_at: Time.current
+        )
+
+        # Return the execution log for tracking
+        execution_log
+      rescue => e
+        # Log any completion errors
+        execution_log.update!(
+          state: :failed,
+          error_message: e.message,
+          error_class: e.class.name
+        )
+        raise
+      end
+    end
+
     def setup_workflow(key, options, kwargs)
       @workflow = find_workflow(key, options, kwargs)
       @context = Context.new(@workflow)

test/chrono_forge_test.rb

@@ -8,43 +8,110 @@ def test_version
   end
 
   def test_kitchen_sink_runs_successfully
-    KitchenSink.perform_later("identifier", kwarg: "durable", options: {option1: 1})
+    KitchenSink.perform_later("happy_path", kwarg: "durable", options: {option1: 1})
 
     perform_all_jobs
 
-    assert_equal ChronoForge::Workflow.count, 1, "ONLY one workflow exists"
+    workflow = ChronoForge::Workflow.last
 
-    workfow = ChronoForge::Workflow.first
+    assert workflow.completed?, "workflow should be completed"
 
-    assert workfow.completed?, "workflow should be completed"
+    assert_equal "happy_path", workflow.key
+    assert_equal "KitchenSink", workflow.job_class
+    assert_equal({"kwarg" => "durable"}, workflow.kwargs)
+    assert_equal({"option1" => 1}, workflow.options)
 
-    assert_equal workfow.key, "identifier"
-    assert_equal workfow.job_class, "KitchenSink"
-    assert_equal workfow.kwargs, {"kwarg" => "durable"}
-    assert_equal workfow.options, {"option1" => 1}
+    assert workflow.context["order_id"], "order_id should be set"
+    assert workflow.context["processed_at"], "processed_at should be set"
+    assert workflow.context["completed_at"], "completed_at should be set"
 
-    assert workfow.context["order_id"], "order_id should be set"
-    assert workfow.context["processed_at"], "processed_at should be set"
-    assert workfow.context["completed_at"], "completed_at should be set"
+    assert workflow.started_at, "workflow tracking dates should be set: started_at"
+    assert workflow.completed_at, "workflow tracking dates should be set: completed_at"
 
-    refute workfow.locked_by, "workflow should be unlocked: locked_by"
+    refute workflow.locked_at, "workflow should be unlocked: locked_at"
+    refute workflow.locked_by, "workflow should be unlocked: locked_by"
 
-    assert workfow.started_at, "workflow tracking dates should be set: started_at"
-    assert workfow.completed_at, "workflow tracking dates should be set: completed_at"
+    assert_equal 5, workflow.execution_logs.size, "there should be 5 executions"
+    assert_equal [
+      "wait_until$payment_confirmed?",
+      "wait$fraud_check_delay",
+      "durably_execute$process_order",
+      "durably_execute$complete_order",
+      "$workflow_completion$"
+    ], workflow.execution_logs.pluck(:step_name)
+
+    assert_equal 0, workflow.error_logs.size, "no errors should have occurred"
+  end
 
-    refute workfow.locked_at, "workflow should be unlocked: locked_at"
-    refute workfow.locked_by, "workflow should be unlocked: locked_by"
+  def test_kitchen_sink_experiences_a_glitch
+    workflow = KitchenSink.new("glitched")
+    run_scenario(
+      workflow,
+      glitch: ["before", "#{workflow.method(:process_order).source_location[0]}:17"]
+    )
 
-    assert_equal workfow.execution_logs.size, 5, "there should be 5 executions"
-    assert_equal workfow.execution_logs.pluck(:step_name), [
+    workflow = ChronoForge::Workflow.last
+
+    assert workflow.completed?, "workflow should be completed"
+
+    assert_equal "glitched", workflow.key
+
+    assert workflow.context["order_id"], "order_id should be set"
+    assert workflow.context["processed_at"], "processed_at should be set"
+    assert workflow.context["completed_at"], "completed_at should be set"
+
+    assert workflow.started_at, "workflow tracking dates should be set: started_at"
+    assert workflow.completed_at, "workflow tracking dates should be set: completed_at"
+
+    refute workflow.locked_at, "workflow should be unlocked: locked_at"
+    refute workflow.locked_by, "workflow should be unlocked: locked_by"
+
+    assert_equal 5, workflow.execution_logs.size, "there should be 5 executions"
+    assert_equal [
       "wait_until$payment_confirmed?",
       "wait$fraud_check_delay",
       "durably_execute$process_order",
       "durably_execute$complete_order",
       "$workflow_completion$"
-    ]
+    ], workflow.execution_logs.pluck(:step_name)
+
+    assert_equal 1, workflow.error_logs.size, "a single glitch should have occurred"
+    assert_equal ["ChaoticJob::RetryableError"], workflow.error_logs.pluck(:error_class).uniq
+  end
+
+  def test_kitchen_sink_permanenty_fails
+    KitchenSink.perform_later("permanent_failed", permanently_fail: true)
+
+    perform_all_jobs
+
+    workflow = ChronoForge::Workflow.last
+
+    assert workflow.failed?, "workflow should be failed"
+
+    assert_equal "permanent_failed", workflow.key
+    assert_equal "KitchenSink", workflow.job_class
+    assert_equal({"permanently_fail" => true}, workflow.kwargs)
+    assert_equal({}, workflow.options)
+
+    assert workflow.context["order_id"], "order_id should be set"
+    assert workflow.context["processed_at"], "processed_at should be set"
+    refute workflow.context["completed_at"], "completed_at should NOT be set"
+
+    assert workflow.started_at, "workflow tracking dates should be set: started_at"
+    refute workflow.completed_at, "workflow tracking dates should NOT be set: completed_at"
+
+    refute workflow.locked_at, "workflow should be unlocked: locked_at"
+    refute workflow.locked_by, "workflow should be unlocked: locked_by"
+
+    assert_equal 4, workflow.execution_logs.size, "there should be 5 executions"
+    assert_equal [
+      "wait_until$payment_confirmed?",
+      "wait$fraud_check_delay",
+      "durably_execute$process_order",
+      "$workflow_failure$"
+    ], workflow.execution_logs.pluck(:step_name)
 
-    assert_equal workfow.error_logs.size, 0, "no errors should have occurred"
-    assert ChronoForge::Workflow.last.completed?
+    assert_equal 4, workflow.error_logs.size, "workflow should have failed after 4 runs. 1 + 3 retries."
+    assert_equal ["Permanent Failure"], workflow.error_logs.pluck(:error_message).uniq
   end
 end

test/internal/app/jobs/kitchen_sink.rb

@@ -1,7 +1,7 @@
 class KitchenSink < WorkflowJob
   prepend ChronoForge::Executor
 
-  def perform(kwarg: nil, succeed: true)
+  def perform(kwarg: nil, permanently_fail: false)
     # Context can be used to pass and store data between executions
     context[:order_id] ||= SecureRandom.hex
 
@@ -16,7 +16,7 @@ def perform(kwarg: nil, succeed: true)
     # Durably execute order processing
     durably_execute :process_order
 
-    raise "Permanent Failure" unless succeed
+    raise "Permanent Failure" if permanently_fail
 
     # Final steps
     durably_execute :complete_order