feat: health gate before autopull + delivery backoff retry (closes #22)

Author: wolverin0

app/services/agent_auto_runner_service.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "net/http"
+require "uri"
+
 class AgentAutoRunnerService
   NO_FAKE_IN_PROGRESS_GRACE = 10.minutes
   ZOMBIE_STALE_AFTER = 30.minutes
@@ -37,6 +40,16 @@ def run!
       plan = selector.plan
       plan.skip_reasons.each { |reason, amount| stats[:queue_skip_reasons][reason] += amount.to_i }
 
+      unless gateway_ready?(user)
+        cache_key = "openclaw_health_alert:#{user.id}"
+        unless @cache.exist?(cache_key)
+          @cache.write(cache_key, Time.current.to_i, expires_in: 30.minutes)
+        end
+        @logger.warn("[AgentAutoRunner] gateway not ready user=#{user.id}, skipping")
+        stats[:queue_skip_reasons][:gateway_not_ready] += 1
+        next
+      end
+
 woke = 0
 if plan.tasks.any?
   woke = wake_tasks!(user, plan.tasks)
@@ -63,6 +76,35 @@ def openclaw_configured?(user)
     user.openclaw_gateway_url.present? && (hooks_token.present? || user.openclaw_gateway_token.present?)
   end
 
+  def gateway_ready?(user)
+    return true if Rails.env.test? && ENV["OPENCLAW_GATEWAY_HEALTHCHECK"] != "true"
+
+    base_url = gateway_base_url(user)
+    return false if base_url.blank?
+
+    uri = URI.parse(base_url)
+    uri.path = "/ready"
+    uri.query = nil
+
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = uri.scheme == "https"
+    http.open_timeout = 3
+    http.read_timeout = 3
+
+    response = http.request(Net::HTTP::Get.new(uri.request_uri))
+    code = response.code.to_i
+    code >= 200 && code < 300
+  rescue StandardError
+    false
+  end
+
+  def gateway_base_url(user)
+    url = user.respond_to?(:openclaw_gateway_url) ? user.openclaw_gateway_url.to_s.strip : ""
+    url = ENV["OPENCLAW_GATEWAY_URL"].to_s.strip if url.blank?
+    url = "http://192.168.100.186:18789" if url.blank?
+    url
+  end
+
   # Pipeline: process tasks that need pipeline advancement
   # Simplified: skip triage, go straight to context compilation + routing
   def process_pipeline_tasks!(user)

app/services/external_notification_service.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "json"
 require "net/http"
 require "uri"
 
@@ -68,9 +69,24 @@ def send_telegram
     thread_id = telegram_thread_id
     params[:message_thread_id] = thread_id if thread_id.present?
 
-    Net::HTTP.post_form(uri, params)
-  rescue StandardError => e
-    Rails.logger.warn("[ExternalNotification] Telegram failed: #{e.message}")
+    delays = [1, 5, 30]
+    retries = 0
+
+    loop do
+      begin
+        response = Net::HTTP.post_form(uri, params)
+        message_id = extract_telegram_message_id(response)
+        return response if message_id.present?
+      rescue StandardError
+      end
+
+      break if retries >= delays.length
+      Kernel.sleep(delays[retries])
+      retries += 1
+    end
+
+    Rails.logger.warn("[DeliveryBackoff] failed after 3 attempts")
+    nil
   end
 
   def send_webhook
@@ -122,4 +138,13 @@ def send_webhook
   rescue StandardError => e
     Rails.logger.warn("[ExternalNotification] Webhook failed: #{e.message}")
   end
+
+  def extract_telegram_message_id(response)
+    return nil unless response.respond_to?(:body)
+
+    payload = JSON.parse(response.body.to_s) rescue nil
+    return nil unless payload.is_a?(Hash)
+
+    payload.dig("result", "message_id") || payload["message_id"]
+  end
 end

test/services/agent_auto_runner_health_gate_test.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require "test_helper"
+require "minitest/mock"
+
+class AgentAutoRunnerHealthGateTest < ActiveSupport::TestCase
+  include ActiveSupport::Testing::TimeHelpers
+
+  class FakeWebhookService
+    cattr_accessor :wakes, default: []
+
+    def initialize(_user)
+    end
+
+    def notify_auto_pull_ready(task)
+      self.class.wakes << task.id
+      true
+    end
+
+    def notify_auto_pull_ready_with_pipeline(task)
+      self.class.wakes << task.id
+      true
+    end
+
+    def notify_runner_summary(_message)
+      true
+    end
+  end
+
+  setup do
+    Rails.cache.clear
+    User.update_all(agent_auto_mode: false)
+    @user = User.create!(
+      email_address: "auto_runner_health_#{SecureRandom.hex(4)}@example.test",
+      password: "password123",
+      password_confirmation: "password123",
+      agent_auto_mode: true,
+      openclaw_gateway_url: "http://example.test",
+      openclaw_gateway_token: "tok"
+    )
+    @board = Board.create!(user: @user, name: "Health Gate Board")
+    @task = Task.create!(
+      user: @user,
+      board: @board,
+      name: "Health Gate Task",
+      status: :up_next,
+      assigned_to_agent: true,
+      blocked: false,
+      pipeline_enabled: false,
+      model: "gemini"
+    )
+  end
+
+  test "gateway 503 skips wakes and records skip reason" do
+    FakeWebhookService.wakes = []
+    cache = ActiveSupport::Cache::MemoryStore.new
+
+    with_healthcheck_enabled do
+      stub_gateway_ready(503) do
+        travel_to Time.find_zone!("America/Argentina/Buenos_Aires").local(2026, 2, 8, 23, 30, 0) do
+          stats = AgentAutoRunnerService.new(openclaw_webhook_service: FakeWebhookService, cache: cache).run!
+          assert_equal 0, FakeWebhookService.wakes.length
+          assert_equal 1, stats[:queue_skip_reasons][:gateway_not_ready]
+        end
+      end
+    end
+  end
+
+  test "gateway 200 allows wakes" do
+    FakeWebhookService.wakes = []
+    cache = ActiveSupport::Cache::MemoryStore.new
+
+    with_healthcheck_enabled do
+      stub_gateway_ready(200) do
+        travel_to Time.find_zone!("America/Argentina/Buenos_Aires").local(2026, 2, 8, 23, 30, 0) do
+          stats = AgentAutoRunnerService.new(openclaw_webhook_service: FakeWebhookService, cache: cache).run!
+          assert_includes FakeWebhookService.wakes, @task.id
+          assert stats[:tasks_woken] >= 1
+        end
+      end
+    end
+  end
+
+  private
+
+  def with_healthcheck_enabled
+    original = ENV["OPENCLAW_GATEWAY_HEALTHCHECK"]
+    ENV["OPENCLAW_GATEWAY_HEALTHCHECK"] = "true"
+    yield
+  ensure
+    ENV["OPENCLAW_GATEWAY_HEALTHCHECK"] = original
+  end
+
+  def stub_gateway_ready(code)
+    response = Minitest::Mock.new
+    response.expect(:code, code.to_s)
+
+    http = Minitest::Mock.new
+    http.expect(:use_ssl=, nil, [false])
+    http.expect(:open_timeout=, nil, [3])
+    http.expect(:read_timeout=, nil, [3])
+    http.expect(:request, response) { true }
+
+    Net::HTTP.stub(:new, ->(_host, _port) { http }) do
+      yield
+    end
+  ensure
+    response.verify
+    http.verify
+  end
+end

test/services/delivery_backoff_test.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class DeliveryBackoffTest < ActiveSupport::TestCase
+  setup do
+    @user = users(:default)
+    @board = boards(:default)
+    @task = @board.tasks.create!(
+      user: @user,
+      name: "Delivery Backoff Task",
+      description: "Test delivery",
+      status: :in_review,
+      origin_chat_id: "12345"
+    )
+    ENV["CLAWTROL_TELEGRAM_BOT_TOKEN"] = "test_token"
+  end
+
+  teardown do
+    ENV.delete("CLAWTROL_TELEGRAM_BOT_TOKEN")
+  end
+
+  test "missing message_id triggers retries" do
+    responses = [
+      build_response({ "ok" => true, "result" => {} }),
+      build_response({ "ok" => true }),
+      build_response({ "ok" => true, "result" => { "message_id" => 123 } })
+    ]
+    call_count = 0
+    sleeps = []
+
+    Net::HTTP.stub(:post_form, ->(_uri, _params) {
+      response = responses[call_count]
+      call_count += 1
+      response
+    }) do
+      Kernel.stub(:sleep, ->(duration) { sleeps << duration }) do
+        ExternalNotificationService.new(@task).send(:send_telegram)
+      end
+    end
+
+    assert_equal 3, call_count
+    assert_equal [1, 5], sleeps
+  end
+
+  test "logs after three failures and continues" do
+    response = build_response({ "ok" => true })
+    call_count = 0
+    sleeps = []
+    logged = []
+
+    Net::HTTP.stub(:post_form, ->(_uri, _params) {
+      call_count += 1
+      response
+    }) do
+      Kernel.stub(:sleep, ->(duration) { sleeps << duration }) do
+        Rails.logger.stub(:warn, ->(message) { logged << message }) do
+          assert_nothing_raised { ExternalNotificationService.new(@task).send(:send_telegram) }
+        end
+      end
+    end
+
+    assert_equal 4, call_count
+    assert_equal [1, 5, 30], sleeps
+    assert_includes logged, "[DeliveryBackoff] failed after 3 attempts"
+  end
+
+  private
+
+  def build_response(payload)
+    Struct.new(:body).new(payload.to_json)
+  end
+end

test/services/external_notification_service_test.rb

@@ -121,7 +121,9 @@ class ExternalNotificationServiceTest < ActiveSupport::TestCase
       service = ExternalNotificationService.new(@task)
       assert service.send(:telegram_configured?)
       # send_telegram catches all exceptions
-      assert_nothing_raised { service.send(:send_telegram) }
+      Kernel.stub(:sleep, nil) do
+        assert_nothing_raised { service.send(:send_telegram) }
+      end
     else
       skip "Task does not have origin_chat_id column"
     end