fix: global hard stop coverage, 1s save task, exhaustion detail logging

Author: CaptainMirage

src/bin/ui.rs

@@ -1224,17 +1224,19 @@ impl eframe::App for App {
 
                     // Use quota-tracked daily capacity when available, fall back to
                     // the free-tier default for display purposes.
+                    // quota_used: total relay() calls (exit node + Apps Script combined)
+                    // so "fetches today" reflects all proxied traffic, not just Apps Script.
                     let (quota_cap, quota_used, _quota_remaining, any_exhausted, global_stop) =
                         if let Some(q) = &quota_state {
                             (
                                 q.daily_capacity_total.max(1),
-                                q.requests_used_total,
+                                s.total_relay_calls,
                                 q.requests_remaining_total,
                                 q.exhausted_count > 0,
                                 q.global_hard_stop,
                             )
                         } else {
-                            (20_000u64, s.today_calls, 20_000u64.saturating_sub(s.today_calls), false, false)
+                            (20_000u64, s.total_relay_calls, 20_000u64.saturating_sub(s.total_relay_calls), false, false)
                         };
 
                     let pct = (quota_used as f64 / quota_cap as f64) * 100.0;

src/domain_fronter.rs

@@ -365,6 +365,11 @@ pub struct DomainFronter {
     /// strike state is per-deployment health bookkeeping, not the
     /// permanent ban list.
     script_timeouts: Arc<std::sync::Mutex<HashMap<String, (Instant, u32)>>>,
+    /// Every call to `relay()` increments this — exit node AND Apps Script.
+    /// Use this for UI "fetches today" display. Distinct from `relay_calls`
+    /// (Apps-Script-direct only) and from the quota tracker's `requests_used`
+    /// (also Apps-Script-only).
+    total_relay_calls: AtomicU64,
     relay_calls: AtomicU64,
     relay_failures: AtomicU64,
     bytes_relayed: AtomicU64,
@@ -633,6 +638,7 @@ impl DomainFronter {
             coalesced: AtomicU64::new(0),
             blacklist: Arc::new(std::sync::Mutex::new(HashMap::new())),
             script_timeouts: Arc::new(std::sync::Mutex::new(HashMap::new())),
+            total_relay_calls: AtomicU64::new(0),
             relay_calls: AtomicU64::new(0),
             relay_failures: AtomicU64::new(0),
             bytes_relayed: AtomicU64::new(0),
@@ -775,6 +781,7 @@ impl DomainFronter {
             guard.clone()
         };
         StatsSnapshot {
+            total_relay_calls: self.total_relay_calls.load(Ordering::Relaxed),
             relay_calls: self.relay_calls.load(Ordering::Relaxed),
             relay_failures: self.relay_failures.load(Ordering::Relaxed),
             coalesced: self.coalesced.load(Ordering::Relaxed),
@@ -1784,6 +1791,23 @@ impl DomainFronter {
         headers: &[(String, String)],
         body: &[u8],
     ) -> Vec<u8> {
+        self.total_relay_calls.fetch_add(1, Ordering::Relaxed);
+
+        // Block ALL relay paths (exit node + Apps Script) when every account
+        // bucket is quota-exhausted. Checked here so the exit node short-circuit
+        // below can't bypass the global hard stop.
+        if self.quota_tracker.is_globally_hard_stopped() {
+            self.relay_failures.fetch_add(1, Ordering::Relaxed);
+            tracing::error!(
+                "[quota] global hard stop active — all Apps Script account buckets exhausted"
+            );
+            return error_response(
+                502,
+                "All Apps Script accounts quota exhausted; hard stop active. \
+                 Quota resets on a rolling 24-hour window per account.",
+            );
+        }
+
         // Optional URL rewrite for X/Twitter GraphQL (issue #16). Applied
         // here, at the top of relay(), so it affects BOTH the cache key
         // (so matching requests collapse into one entry) AND the URL that
@@ -4907,6 +4931,9 @@ fn decode_js_string_escapes(s: &str) -> Option<String> {
 
 #[derive(Debug, Clone)]
 pub struct StatsSnapshot {
+    /// Total calls to `relay()` — all traffic through this fronter including
+    /// exit node and Apps Script. Use for "fetches today" display.
+    pub total_relay_calls: u64,
     pub relay_calls: u64,
     pub relay_failures: u64,
     pub coalesced: u64,

src/main.rs

@@ -357,11 +357,6 @@ async fn main() -> ExitCode {
         }
     };
 
-    // Log quota state on startup so the user knows where their daily budget stands.
-    if let Some(summary) = server.quota_startup_summary() {
-        tracing::info!("{}", summary);
-    }
-
     let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel();
 
     let run = server.run(shutdown_rx);

src/proxy_server.rs

@@ -564,6 +564,9 @@ impl ProxyServer {
             "Listening SOCKS5 on {} — xray / Telegram / app-level SOCKS5 clients use this.",
             socks_addr
         );
+        if let Some(summary) = self.quota_startup_summary() {
+            tracing::info!("{}", summary);
+        }
         // Pre-warm the outbound connection pool so the user's first request
         // doesn't pay a fresh TLS handshake to Google edge. Best-effort;
         // failures are logged and ignored. Skipped in `direct` mode —
@@ -616,22 +619,25 @@ impl ProxyServer {
 
         let stats_task = if let Some(stats_fronter) = self.fronter.clone() {
             tokio::spawn(async move {
-                let mut ticker = tokio::time::interval(std::time::Duration::from_secs(60));
+                let mut ticker = tokio::time::interval(std::time::Duration::from_secs(15));
                 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
-                ticker.tick().await;
+                let mut was_hard_stopped = false;
                 loop {
                     ticker.tick().await;
                     let s = stats_fronter.snapshot_stats();
                     if s.relay_calls > 0 || s.cache_hits > 0 {
                         tracing::info!("{}", s.fmt_line());
                     }
-                    // Log quota warnings and flush persisted state.
                     let q = &s.quota;
                     if q.global_hard_stop {
                         tracing::error!(
                             "[quota] GLOBAL HARD STOP — all {} account(s) exhausted",
                             q.account_count
                         );
+                        // Log per-account reasons only on the first tick after transition.
+                        if !was_hard_stopped {
+                            stats_fronter.quota_tracker().log_exhaustion_details();
+                        }
                     } else if q.exhausted_count > 0 {
                         tracing::warn!(
                             "[quota] {}/{} account(s) exhausted  used={}/{}  remaining={}",
@@ -642,13 +648,29 @@ impl ProxyServer {
                             q.requests_remaining_total,
                         );
                     }
+                    was_hard_stopped = q.global_hard_stop;
                     // Roll any expired 24-hour windows so idle accounts come
                     // back online even without inbound traffic.
                     stats_fronter.quota_tracker().roll_expired_windows();
-                    // Always flush so the file is up-to-date even when idle.
-                    // save_if_needed() skips the write when dirty_count == 0,
-                    // which means zero-traffic sessions never update the file.
-                    stats_fronter.quota_tracker().save();
+                    // Safety-net flush for idle sessions (1s save task handles
+                    // active-traffic saves; this covers the zero-traffic case).
+                    stats_fronter.quota_tracker().save_if_needed();
+                }
+            })
+        } else {
+            tokio::spawn(async move { std::future::pending::<()>().await })
+        };
+
+        // Flush quota state to disk every second so the JSON file stays within
+        // ~1s of in-memory state. The Mutex-backed in-memory state is always
+        // real-time; this just keeps the on-disk snapshot current.
+        let save_task = if let Some(save_fronter) = self.fronter.clone() {
+            tokio::spawn(async move {
+                let mut ticker = tokio::time::interval(std::time::Duration::from_secs(1));
+                ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+                loop {
+                    ticker.tick().await;
+                    save_fronter.quota_tracker().save_if_needed();
                 }
             })
         } else {
@@ -744,6 +766,7 @@ impl ProxyServer {
             biased;
             _ = &mut shutdown_rx => {
                 tracing::info!("Shutdown signal received, stopping listeners");
+                save_task.abort();
                 stats_task.abort();
                 keepalive_task.abort();
                 refill_task.abort();

src/quota_tracker.rs

@@ -487,21 +487,48 @@ impl QuotaTracker {
     }
 
     /// Build a human-readable startup summary line.
+    /// Log the masked ID and exhaustion reason for every hard-stopped bucket.
+    /// Called once when global hard stop transitions from false to true.
+    pub fn log_exhaustion_details(&self) {
+        let st = self.state.lock().unwrap();
+        for sid in &self.script_ids {
+            let Some(b) = st.buckets.get(sid) else { continue };
+            if b.hard_stopped {
+                let reason = b.exhaustion_reason.as_deref().unwrap_or("no reason recorded");
+                tracing::warn!("[quota]   {} exhausted: {}", b.masked_id, reason);
+            }
+        }
+    }
+
     pub fn startup_summary(&self) -> String {
         let s = self.summary();
         let now = now_unix();
         let reset_str = s.next_reset_at.map(|r| {
             let secs = r.saturating_sub(now);
             format!("  next_reset=in {}h {}m", secs / 3600, (secs / 60) % 60)
         }).unwrap_or_default();
+        let stop_suffix = if s.global_hard_stop {
+            format!("  exhausted={}/{} HARD-STOP", s.exhausted_count, s.account_count)
+        } else if s.exhausted_count > 0 {
+            format!("  exhausted={}/{}", s.exhausted_count, s.account_count)
+        } else {
+            String::new()
+        };
 
         format!(
-            "[quota] {} account(s)  capacity={}/day  used={}  remaining={}{}",
+            "[quota] {} account(s)  capacity={}/day  used={}  remaining={}{}{}",
             s.account_count,
             s.daily_capacity_total,
             s.requests_used_total,
             s.requests_remaining_total,
             reset_str,
+            stop_suffix,
         )
     }
 }
+
+impl Drop for QuotaTracker {
+    fn drop(&mut self) {
+        self.save();
+    }
+}