Harden search egress auditing and disable telemetry defaults

Author: peruna

README.md

@@ -84,6 +84,8 @@ const test=(h)=>new Promise(r=>{\
 
 Edit the allowlist and restart the proxy to change which domains are permitted.
 
+When the local-search overlay is enabled, SearXNG/Firecrawl egress is intentionally routed through Squid with full access logging so outbound search fetches are auditable.
+
 ---
 
 ## Model Presets
@@ -184,7 +186,7 @@ Search context is bounded by default (3 results, 2 scraped sources, 2 highlights
 
 - SearXNG config (`optional/local-search/searxng/settings.yml`): JSON output enabled (LibreChat requires it), noisy engines removed, timeout lowered to 4 s.
 - Rate limiting uses Valkey with a private-IP allowlist so LibreChat doesn't trip bot detection.
-- Firecrawl, its Redis, RabbitMQ, and Postgres sit on a dedicated `search` network. Only SearXNG, the Firecrawl API, and Playwright get WAN access (they need it to fetch pages).
+- Firecrawl, its Redis, RabbitMQ, and Postgres sit on a dedicated `search` network. SearXNG + Firecrawl web-fetch traffic is routed through Squid on a dedicated `search_egress` subnet so requests are auditable in proxy logs.
 - The Jina compatibility patch makes `batch_size` optional — LibreChat's client omits it.
 - A mounted search patch caps scraped text, requests `markdown` + `onlyMainContent`, and strips raw `content` from the artifact returned to the model.
 - After editing files under `optional/local-search/jina/`, recreate the container to pick up changes.
@@ -245,6 +247,9 @@ MEILI_LOG_LEVEL=WARN
 CODE_INTERPRETER_LOG_LEVEL=WARNING
 FIRECRAWL_LOG_LEVEL=warn
 JINA_RERANKER_LOG_LEVEL=WARNING
+DO_NOT_TRACK=1
+FIRECRAWL_NO_TELEMETRY=1
+HF_HUB_DISABLE_TELEMETRY=1
 ```
 
 **Useful log commands:**
@@ -253,7 +258,7 @@ docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'
 docker logs -f --tail=200 LibreChat
 docker logs -f --tail=200 egress-proxy
 
-# Squid logs denied destinations by default (allowed CONNECTs are suppressed).
+# Squid logs all local-search egress (auditable) plus denied requests elsewhere.
 # Look for TCP_DENIED/403 when debugging allowlist misses.
 docker exec -u proxy egress-proxy sh -lc 'tail -f /var/log/squid/access.log'
 ```

docker-compose.yml

@@ -124,6 +124,7 @@ services:
       http_proxy: ${EGRESS_PROXY_URL:-http://egress-proxy:3128}
       https_proxy: ${EGRESS_PROXY_URL:-http://egress-proxy:3128}
       NODE_OPTIONS: --require /app/proxy-bootstrap.cjs
+      DO_NOT_TRACK: ${DO_NOT_TRACK:-1}
       NO_PROXY: ${NO_PROXY:-localhost,127.0.0.1,::1,mongodb,chat-mongodb,meilisearch,vectordb,rag_api,sandpack,sandpack-static,caddy-static-proxy,api-proxy,egress-proxy}
       no_proxy: ${NO_PROXY:-localhost,127.0.0.1,::1,mongodb,chat-mongodb,meilisearch,vectordb,rag_api,sandpack,sandpack-static,caddy-static-proxy,api-proxy,egress-proxy}
       CONFIG_PATH: /run/secrets/librechat_yaml
@@ -289,6 +290,8 @@ services:
       RAG_HOST: 0.0.0.0
       DB_HOST: vectordb
       RAG_PORT: ${RAG_PORT:-8000}
+      DO_NOT_TRACK: ${DO_NOT_TRACK:-1}
+      HF_HUB_DISABLE_TELEMETRY: ${HF_HUB_DISABLE_TELEMETRY:-1}
       EGRESS_PROXY_URL: ${EGRESS_PROXY_URL:-http://egress-proxy:3128}
       HTTP_PROXY: ${EGRESS_PROXY_URL:-http://egress-proxy:3128}
       HTTPS_PROXY: ${EGRESS_PROXY_URL:-http://egress-proxy:3128}

optional/code-interpreter/compose.yml

@@ -43,6 +43,8 @@ services:
       API_KEY: ${LIBRECHAT_CODE_API_KEY:-local-code-interpreter-key-change-me}
       REDIS_HOST: code-interpreter-redis
       REDIS_PORT: 6379
+      DO_NOT_TRACK: ${DO_NOT_TRACK:-1}
+      HF_HUB_DISABLE_TELEMETRY: ${HF_HUB_DISABLE_TELEMETRY:-1}
       MINIO_ENDPOINT: code-interpreter-minio:9000
       MINIO_ACCESS_KEY: ${CODE_INTERPRETER_MINIO_ACCESS_KEY:-minioadmin}
       MINIO_SECRET_KEY: ${CODE_INTERPRETER_MINIO_SECRET_KEY:-minioadmin}

optional/egress-proxy/squid.conf

@@ -20,6 +20,9 @@ acl allowed_domains dstdomain "/run/secrets/squid_allowlist"
 # Vertex AI regional endpoints follow LOCATION-aiplatform.googleapis.com.
 # CONNECT checks may include ":443", so allow optional port suffix.
 acl allowed_vertex_regional dstdom_regex -i ^[a-z0-9-]+-aiplatform\.googleapis\.com(:[0-9]+)?$
+# Local-search overlay egress network (optional/local-search/compose.yml).
+# These clients are allowed outbound web access, but all requests are audited.
+acl search_clients src 172.29.240.0/24
 
 # Restrict ports/methods.
 acl SSL_ports port 443
@@ -31,11 +34,13 @@ http_access deny !Safe_ports
 http_access deny CONNECT !SSL_ports
 
 # Allow only explicit destinations.
+http_access allow search_clients
 http_access allow allowed_vertex_regional
 http_access allow allowed_domains
 http_access deny all
 
-# Keep runtime noise low: only log denied destinations.
-access_log stdio:/var/log/squid/access.log !allowed_vertex_regional !allowed_domains
+# Audit all local-search proxy traffic, plus denied traffic from other clients.
+access_log stdio:/var/log/squid/access.log search_clients
+access_log stdio:/var/log/squid/access.log !search_clients !allowed_vertex_regional !allowed_domains
 cache_log /dev/null
 cache_store_log none

optional/local-search/compose.yml

@@ -8,6 +8,11 @@ volumes:
 networks:
   search:
     internal: true
+  search_egress:
+    internal: true
+    ipam:
+      config:
+        - subnet: 172.29.240.0/24
 
 x-json-logging: &json_logging
   driver: json-file
@@ -16,6 +21,12 @@ x-json-logging: &json_logging
     max-file: "${DOCKER_LOG_MAX_FILE:-3}"
 
 services:
+  egress-proxy:
+    networks:
+      - lan
+      - wan
+      - search_egress
+
   api:
     depends_on:
       searxng:
@@ -51,13 +62,19 @@ services:
       SEARXNG_BASE_URL: ${SEARXNG_BASE_URL:-http://searxng:8080/}
       SEARXNG_LIMITER: "true"
       SEARXNG_SECRET: ${SEARXNG_SECRET:-change-me-search-secret}
+      HTTP_PROXY: ${EGRESS_PROXY_URL:-http://egress-proxy:3128}
+      HTTPS_PROXY: ${EGRESS_PROXY_URL:-http://egress-proxy:3128}
+      http_proxy: ${EGRESS_PROXY_URL:-http://egress-proxy:3128}
+      https_proxy: ${EGRESS_PROXY_URL:-http://egress-proxy:3128}
+      NO_PROXY: ${NO_PROXY:-localhost,127.0.0.1,::1,mongodb,chat-mongodb,meilisearch,vectordb,rag_api,sandpack,sandpack-static,caddy-static-proxy,api-proxy,egress-proxy},searxng,searxng-valkey
+      no_proxy: ${NO_PROXY:-localhost,127.0.0.1,::1,mongodb,chat-mongodb,meilisearch,vectordb,rag_api,sandpack,sandpack-static,caddy-static-proxy,api-proxy,egress-proxy},searxng,searxng-valkey
     volumes:
       - ./optional/local-search/searxng/settings.yml:/etc/searxng/settings.yml:ro
       - ./optional/local-search/searxng/limiter.toml:/etc/searxng/limiter.toml:ro
       - searxng_data:/var/cache/searxng
     networks:
       - search
-      - wan
+      - search_egress
     ports:
       - "127.0.0.1:${SEARXNG_PORT:-8080}:8080"
     depends_on:
@@ -91,8 +108,14 @@ services:
       PORT: 3002
       ENV: local
       LOGGING_LEVEL: ${FIRECRAWL_LOG_LEVEL:-warn}
-      FIRECRAWL_NO_TELEMETRY: 1
-      DO_NOT_TRACK: 1
+      FIRECRAWL_NO_TELEMETRY: ${FIRECRAWL_NO_TELEMETRY:-1}
+      DO_NOT_TRACK: ${DO_NOT_TRACK:-1}
+      HTTP_PROXY: ${EGRESS_PROXY_URL:-http://egress-proxy:3128}
+      HTTPS_PROXY: ${EGRESS_PROXY_URL:-http://egress-proxy:3128}
+      http_proxy: ${EGRESS_PROXY_URL:-http://egress-proxy:3128}
+      https_proxy: ${EGRESS_PROXY_URL:-http://egress-proxy:3128}
+      NO_PROXY: ${NO_PROXY:-localhost,127.0.0.1,::1,mongodb,chat-mongodb,meilisearch,vectordb,rag_api,sandpack,sandpack-static,caddy-static-proxy,api-proxy,egress-proxy},firecrawl-redis,firecrawl-rabbitmq,firecrawl-postgres,firecrawl-playwright,searxng,searxng-valkey
+      no_proxy: ${NO_PROXY:-localhost,127.0.0.1,::1,mongodb,chat-mongodb,meilisearch,vectordb,rag_api,sandpack,sandpack-static,caddy-static-proxy,api-proxy,egress-proxy},firecrawl-redis,firecrawl-rabbitmq,firecrawl-postgres,firecrawl-playwright,searxng,searxng-valkey
       REDIS_URL: redis://firecrawl-redis:6379
       REDIS_RATE_LIMIT_URL: redis://firecrawl-redis:6379
       NUQ_RABBITMQ_URL: amqp://firecrawl-rabbitmq:5672
@@ -110,7 +133,7 @@ services:
       EXTRACT_WORKER_PORT: 3004
       WORKER_PORT: 3005
       USE_DB_AUTHENTICATION: "false"
-      PROXY_SERVER: ""
+      PROXY_SERVER: ${EGRESS_PROXY_URL:-http://egress-proxy:3128}
       PROXY_USERNAME: ""
       PROXY_PASSWORD: ""
       OPENAI_API_KEY: ""
@@ -138,7 +161,7 @@ services:
         condition: service_healthy
     networks:
       - search
-      - wan
+      - search_egress
     healthcheck:
       test:
         [
@@ -158,15 +181,21 @@ services:
     environment:
       PORT: 3000
       MAX_CONCURRENT_PAGES: ${FIRECRAWL_MAX_CONCURRENT_PAGES:-4}
-      FIRECRAWL_NO_TELEMETRY: 1
-      DO_NOT_TRACK: 1
+      FIRECRAWL_NO_TELEMETRY: ${FIRECRAWL_NO_TELEMETRY:-1}
+      DO_NOT_TRACK: ${DO_NOT_TRACK:-1}
+      HTTP_PROXY: ${EGRESS_PROXY_URL:-http://egress-proxy:3128}
+      HTTPS_PROXY: ${EGRESS_PROXY_URL:-http://egress-proxy:3128}
+      http_proxy: ${EGRESS_PROXY_URL:-http://egress-proxy:3128}
+      https_proxy: ${EGRESS_PROXY_URL:-http://egress-proxy:3128}
+      NO_PROXY: ${NO_PROXY:-localhost,127.0.0.1,::1,mongodb,chat-mongodb,meilisearch,vectordb,rag_api,sandpack,sandpack-static,caddy-static-proxy,api-proxy,egress-proxy},firecrawl-api,firecrawl-redis,firecrawl-rabbitmq,firecrawl-postgres,searxng,searxng-valkey
+      no_proxy: ${NO_PROXY:-localhost,127.0.0.1,::1,mongodb,chat-mongodb,meilisearch,vectordb,rag_api,sandpack,sandpack-static,caddy-static-proxy,api-proxy,egress-proxy},firecrawl-api,firecrawl-redis,firecrawl-rabbitmq,firecrawl-postgres,searxng,searxng-valkey
       BLOCK_MEDIA: ${FIRECRAWL_BLOCK_MEDIA:-}
-      PROXY_SERVER: ""
+      PROXY_SERVER: ${EGRESS_PROXY_URL:-http://egress-proxy:3128}
       PROXY_USERNAME: ""
       PROXY_PASSWORD: ""
     networks:
       - search
-      - wan
+      - search_egress
     healthcheck:
       test:
         [
@@ -239,6 +268,8 @@ services:
       MODEL_NAME: ${JINA_RERANKER_MODEL_NAME:-jinaai/jina-reranker-v1-tiny-en}
       CACHE_DIR: /app/.cache
       TOKENIZERS_PARALLELISM: "false"
+      DO_NOT_TRACK: ${DO_NOT_TRACK:-1}
+      HF_HUB_DISABLE_TELEMETRY: ${HF_HUB_DISABLE_TELEMETRY:-1}
       OMP_NUM_THREADS: ${JINA_RERANKER_OMP_NUM_THREADS:-1}
       JINA_RERANKER_MAX_BATCH_SIZE: ${JINA_RERANKER_MAX_BATCH_SIZE:-2}
       JINA_RERANKER_LOG_LEVEL: ${JINA_RERANKER_LOG_LEVEL:-WARNING}

optional/local-search/searxng/settings.yml

@@ -8,6 +8,8 @@ use_default_settings:
 general:
   instance_name: "LibreChat Local Search"
   debug: false
+  enable_metrics: false
+  open_metrics: ""
 
 search:
   safe_search: 1
@@ -17,6 +19,9 @@ search:
 
 outgoing:
   request_timeout: 4.0
+  proxies:
+    all://:
+      - http://egress-proxy:3128
 
 server:
   base_url: http://searxng:8080/

template_dot_env

@@ -138,6 +138,11 @@ DOCKER_LOG_MAX_FILE=3
 # Internal hosts that should bypass proxying.
 NO_PROXY=localhost,127.0.0.1,::1,mongodb,chat-mongodb,meilisearch,vectordb,rag_api,sandpack,sandpack-static,caddy-static-proxy,api-proxy,egress-proxy
 
+# Disable telemetry where supported.
+DO_NOT_TRACK=1
+FIRECRAWL_NO_TELEMETRY=1
+HF_HUB_DISABLE_TELEMETRY=1
+
 # ------------------------------------------------------------------------------
 # Optional: LibreCodeInterpreter (custom execute_code backend)
 # Enabled by adding -f optional/code-interpreter/compose.yml or