Identify the caller by counting proxy hops, not by trusting the first header entry

.env.example

@@ -86,7 +86,16 @@ POOL_ADMIN_ADDRESSES=
 
 # === Global Rate Limiting ===
 # Per-IP rate limiting for all endpoints (skipped when x402 is enabled).
-# RATE_LIMIT_ENABLED=true                     # Master switch (default: true)
+# # Number of proxies between the internet and this application, used to locate
+# the caller in X-Forwarded-For. 1 is correct behind a single reverse proxy.
+# Set 0 if exposed directly, in which case forwarding headers are ignored.
+#
+# This value decides who the per-IP rate limits and the daily spend budget are
+# applied to. Too high groups callers together; too low lets a caller choose
+# their own identity by sending the header themselves.
+TRUSTED_PROXY_HOPS=1
+
+RATE_LIMIT_ENABLED=true                     # Master switch (default: true)
 # RATE_LIMIT_PER_MINUTE=60                    # Requests per minute per IP (default: 60)
 # RATE_LIMIT_BURST=10                         # Extra burst capacity (default: 10)
 

.github/workflows/deploy.yml

@@ -64,6 +64,7 @@ jobs:
           X402_ALLOW_TESTNET_PAID_BYPASS=${{ vars.X402_ALLOW_TESTNET_PAID_BYPASS || 'false' }}
           X402_MAX_STAMP_BZZ=${{ vars.X402_MAX_STAMP_BZZ || '5.0' }}
           STAMP_DAILY_BZZ_PER_CALLER=${{ vars.STAMP_DAILY_BZZ_PER_CALLER || '0.5' }}
+          TRUSTED_PROXY_HOPS=${{ vars.TRUSTED_PROXY_HOPS || '1' }}
           STAMP_POOL_CHECK_INTERVAL_SECONDS=${{ vars.STAMP_POOL_CHECK_INTERVAL_SECONDS || '900' }}
           STAMP_POOL_MIN_TTL_HOURS=${{ vars.STAMP_POOL_MIN_TTL_HOURS || '24' }}
           STAMP_POOL_TOPUP_HOURS=${{ vars.STAMP_POOL_TOPUP_HOURS || '168' }}
@@ -129,6 +130,7 @@ jobs:
           X402_ALLOW_TESTNET_PAID_BYPASS=${{ vars.X402_ALLOW_TESTNET_PAID_BYPASS || 'false' }}
           X402_MAX_STAMP_BZZ=${{ vars.X402_MAX_STAMP_BZZ || '5.0' }}
           STAMP_DAILY_BZZ_PER_CALLER=${{ vars.STAMP_DAILY_BZZ_PER_CALLER || '0.5' }}
+          TRUSTED_PROXY_HOPS=${{ vars.TRUSTED_PROXY_HOPS || '1' }}
           STAMP_POOL_CHECK_INTERVAL_SECONDS=${{ vars.STAMP_POOL_CHECK_INTERVAL_SECONDS || '900' }}
           STAMP_POOL_MIN_TTL_HOURS=${{ vars.STAMP_POOL_MIN_TTL_HOURS || '24' }}
           STAMP_POOL_TOPUP_HOURS=${{ vars.STAMP_POOL_TOPUP_HOURS || '168' }}

app/core/client_ip.py

@@ -0,0 +1,106 @@
+# app/core/client_ip.py
+"""Identify the calling client, for limits that are keyed on who is calling.
+
+Three controls key on this value: the global per-IP rate limit, the x402
+free-tier rate limit, and the daily spend budget (#102). A caller who can change
+the value at will is not limited by any of them.
+
+## Why the previous approach was a trap rather than a bug
+
+There were two near-identical copies of this function, in `app/x402/middleware.py`
+and `app/middleware/rate_limit.py`. Both took the FIRST entry of
+`X-Forwarded-For` and trusted it:
+
+    return forwarded_for.split(",")[0].strip()
+
+`X-Forwarded-For` is a list, appended to by each proxy it passes through, so the
+leftmost entry is the one FURTHEST from us — and if any proxy appends rather than
+replaces, that entry is whatever the client sent. A caller rotating the header
+would then get a fresh rate-limit window and a fresh spend budget on every
+request. That is the same failure the pool allowance had when every distinct
+`Origin` minted a new budget.
+
+It was not exploitable: Caddy is configured to replace the header rather than
+append, verified on 2026-09-09 by exhausting the free-tier limit and retrying
+with three different forged values — all still refused. But the safety lived
+entirely in the proxy's configuration, nothing in this repository recorded the
+dependency, and no test pinned it. Setting `trusted_proxies` in Caddy, putting a
+CDN in front, or moving to a load balancer that appends would have made all three
+controls bypassable with nothing failing to say so.
+
+## What this does instead
+
+Count back from the RIGHT by the number of proxies we actually have in front of
+us. The rightmost entry is the one our own proxy added and is therefore the only
+entry we know to be truthful; each further step left is one hop further out.
+
+With `TRUSTED_PROXY_HOPS = 1` this is correct whether the proxy replaces or
+appends, which is the property worth having — it stops depending on a behaviour
+nobody here controls:
+
+    header "1.2.3.4"                    (Caddy replaced)  -> 1.2.3.4
+    header "9.9.9.9, 1.2.3.4"           (Caddy appended)  -> 1.2.3.4
+    header "9.9.9.9, 8.8.8.8, 1.2.3.4"  (forged chain)    -> 1.2.3.4
+
+A forged chain cannot reach past the hop count, because entries to the right of
+the client are added by infrastructure we control.
+"""
+import logging
+from typing import Optional
+
+from fastapi import Request
+
+from app.core.config import settings
+
+logger = logging.getLogger(__name__)
+
+UNKNOWN = "unknown"
+
+
+def _peer(request: Request) -> str:
+    """The direct TCP peer — never forgeable, but it is the proxy when behind one."""
+    if request.client and request.client.host:
+        return request.client.host
+    return UNKNOWN
+
+
+def get_client_ip(request: Request) -> str:
+    """The calling client's address, as far as the proxy configuration allows.
+
+    Reads `TRUSTED_PROXY_HOPS`: the number of proxies between the internet and
+    the application. 0 means the application is directly exposed, in which case
+    forwarding headers are ignored entirely rather than half-trusted.
+    """
+    hops = settings.TRUSTED_PROXY_HOPS
+
+    # Directly exposed: the headers carry no authority at all, so reading them
+    # would be strictly worse than using the connection we actually have.
+    if hops <= 0:
+        return _peer(request)
+
+    forwarded_for = request.headers.get("X-Forwarded-For")
+    if forwarded_for:
+        parts = [p.strip() for p in forwarded_for.split(",") if p.strip()]
+        if parts:
+            index = len(parts) - hops
+            if index >= 0:
+                return parts[index]
+            # Fewer entries than configured hops: the header disagrees with the
+            # deployment. Falling back to the peer is the safe direction — it may
+            # group callers together behind a proxy, which over-limits, rather
+            # than trusting a value that is short by a hop, which under-limits.
+            logger.warning(
+                "X-Forwarded-For has %d entries but TRUSTED_PROXY_HOPS is %d; "
+                "using the direct peer instead. Check the setting against the "
+                "actual number of proxies.",
+                len(parts), hops,
+            )
+            return _peer(request)
+
+    # Single-valued alternative set by some proxies. Equally forgeable, so it is
+    # read only where a proxy is expected to be setting it.
+    real_ip = request.headers.get("X-Real-IP")
+    if real_ip and real_ip.strip():
+        return real_ip.strip()
+
+    return _peer(request)

app/core/config.py

@@ -280,6 +280,19 @@ class Settings(BaseSettings):
     MAX_JSON_DEPTH: int = 20  # Maximum JSON nesting depth
 
     # === Global Rate Limiting ===
+    # Number of proxies between the internet and this application, used to
+    # locate the caller in X-Forwarded-For. See app/core/client_ip.py.
+    #
+    # 1 is correct for the standard deployment: Caddy terminates TLS and
+    # forwards to the gateway on loopback. Raise it if a CDN or load balancer is
+    # added in front, and set it to 0 if the application is exposed directly, in
+    # which case forwarding headers are ignored rather than half-trusted.
+    #
+    # Getting this too HIGH groups callers together, which over-limits and is
+    # visible as complaints. Too LOW lets a caller pick their own identity, which
+    # under-limits and is visible as nothing at all.
+    TRUSTED_PROXY_HOPS: int = 1
+
     RATE_LIMIT_ENABLED: bool = True  # Enable global rate limiting
     RATE_LIMIT_PER_MINUTE: int = 60  # Requests per minute per IP
     RATE_LIMIT_BURST: int = 10  # Extra burst capacity above per-minute limit

app/middleware/rate_limit.py

@@ -97,20 +97,11 @@ def cleanup_stale(self, max_age: float = 300):
 _counter = SlidingWindowCounter()
 
 
-def get_client_ip(request: Request) -> str:
-    """Extract client IP from request, handling proxies."""
-    forwarded_for = request.headers.get("X-Forwarded-For")
-    if forwarded_for:
-        return forwarded_for.split(",")[0].strip()
-
-    real_ip = request.headers.get("X-Real-IP")
-    if real_ip:
-        return real_ip.strip()
-
-    if request.client:
-        return request.client.host
-
-    return "unknown"
+# Single implementation in app/core/client_ip. Re-exported here because
+# callers and tests import it from this module. Both copies used to take the
+# FIRST X-Forwarded-For entry, which is the one furthest from us and is
+# caller-controlled if any proxy appends rather than replaces.
+from app.core.client_ip import get_client_ip  # noqa: F401,E402
 
 
 class RateLimitMiddleware(BaseHTTPMiddleware):

app/x402/middleware.py

@@ -82,23 +82,11 @@ def is_protected_endpoint(method: str, path: str) -> bool:
     return False
 
 
-def get_client_ip(request: Request) -> str:
-    """Extract client IP from request, handling proxies."""
-    # Check for forwarded headers first
-    forwarded_for = request.headers.get("X-Forwarded-For")
-    if forwarded_for:
-        # Take the first IP in the chain
-        return forwarded_for.split(",")[0].strip()
-
-    real_ip = request.headers.get("X-Real-IP")
-    if real_ip:
-        return real_ip.strip()
-
-    # Fall back to direct connection
-    if request.client:
-        return request.client.host
-
-    return "unknown"
+# Single implementation in app/core/client_ip. Re-exported here because
+# callers and tests import it from this module. Both copies used to take the
+# FIRST X-Forwarded-For entry, which is the one furthest from us and is
+# caller-controlled if any proxy appends rather than replaces.
+from app.core.client_ip import get_client_ip  # noqa: F401,E402
 
 
 def create_payment_requirements(

deploy/Caddyfile

@@ -24,10 +24,18 @@
 }
 
 {$GATEWAY_HOST} {
-	# Caddy sets X-Forwarded-For by default. Both get_client_ip()
-	# implementations read it, and without it every caller collapses into one
-	# bucket — per-IP rate limiting and the x402 free-tier allowance would then
-	# be shared globally rather than per client.
+	# Caddy sets X-Forwarded-For by default. The gateway reads it to identify
+	# the caller, and without it every caller collapses into one bucket —
+	# per-IP rate limiting, the x402 free-tier allowance and the daily spend
+	# budget would all be shared globally rather than per client.
+	#
+	# The gateway counts back from the RIGHT of that header by
+	# TRUSTED_PROXY_HOPS (default 1), so it is correct whether Caddy replaces
+	# the header or appends to it. IF YOU ADD A PROXY OR CDN IN FRONT OF THIS
+	# ONE, raise TRUSTED_PROXY_HOPS to match, or the gateway will read the new
+	# proxy's address as the caller and every request will share one budget.
+	# Lowering it below the real number of hops is worse: the caller then gets
+	# to pick their own identity by sending the header themselves.
 	# Prometheus metrics are NOT public (#188). The endpoint exposed 146 series to
 	# anyone who asked: wallet addresses, BZZ and xDAI balances, gateway version,
 	# which features are enabled, and request volumes — a funding position, in
@@ -66,6 +74,9 @@
 }
 
 {$GATEWAY_DEV_HOST} {
+	# The X-Forwarded-For note on the site above applies here too: the gateway
+	# identifies callers from that header, and TRUSTED_PROXY_HOPS must match the
+	# real number of proxies in front of it.
 	# Prometheus metrics are NOT public (#188). The endpoint exposed 146 series to
 	# anyone who asked: wallet addresses, BZZ and xDAI balances, gateway version,
 	# which features are enabled, and request volumes — a funding position, in