From e55f76ee0a7342cddf92d27388d9e986898db491 Mon Sep 17 00:00:00 2001 From: Sakura <32687351+zifanzhou1024@users.noreply.github.com> Date: Sun, 3 May 2026 11:08:05 -0700 Subject: [PATCH 01/11] docs: design experimental flow cockpit --- ...-05-03-experimental-flow-cockpit-design.md | 430 ++++++++++++++++++ 1 file changed, 430 insertions(+) create mode 100644 docs/superpowers/specs/2026-05-03-experimental-flow-cockpit-design.md diff --git a/docs/superpowers/specs/2026-05-03-experimental-flow-cockpit-design.md b/docs/superpowers/specs/2026-05-03-experimental-flow-cockpit-design.md new file mode 100644 index 0000000..0b01a3f --- /dev/null +++ b/docs/superpowers/specs/2026-05-03-experimental-flow-cockpit-design.md @@ -0,0 +1,430 @@ +# Experimental 2 Estimated Flow Cockpit Design + +Date: 2026-05-03 +Status: User-approved design +Repository: gamma-scope + +## Summary + +GammaScope will add a separate `/experimental-2` page for estimated SPX 0DTE buy/sell flow, dealer pressure, and replay validation. This page must not change the main realtime dashboard, the current `/experimental` price-only research cockpit, or the `/heatmap` surface. The first version is SPX-only, with explicit room to add SPY, QQQ, IWM, NDX, and other symbols later. + +The page will use a dense Flow Cockpit layout: a KPI strip, a spot-centered strike ladder as the main object, a right rail with inference diagnostics, and a bottom contract-level audit table. All outputs must be labeled as estimates because free broker snapshots cannot reveal true customer, market-maker, buy-to-open, or sell-to-close classifications. + +## Goals + +- Add `/experimental-2` as a new isolated page for estimated 0DTE flow. +- Add a dedicated backend experimental-flow API and generated contract. +- Use current and previous snapshots to estimate volume delta, aggressor side, premium flow, Greek-weighted flow, dealer pressure, and open/close proxy scores. +- Support live estimate mode and bounded replay validation mode in v1. +- Keep the first version SPX-only while shaping the contract for later multi-symbol support. +- Make every estimate auditable through contract-level rows and diagnostics. +- Preserve existing main dashboard, current experimental tab, heatmap, replay, scenario, and saved-view behavior. + +## Non-Goals + +- Do not claim official customer or market-maker flow without licensed open/close data. +- Do not infer exact buy-to-open, sell-to-open, buy-to-close, or sell-to-close from free quote snapshots. +- Do not add paid Cboe DataShop integration in this slice. +- Do not add brokerage execution, order routing, or trading alerts. +- Do not replace the existing `/experimental` page or merge these flow panels into it. +- Do not expand to SPY, QQQ, IWM, NDX, or other symbols in v1. +- Do not build a full backtest lab in v1. + +## Current Project Context + +The repo already has: + +- A Next.js web app under `apps/web`. +- A FastAPI backend under `apps/api`. +- Shared JSON Schema contracts under `packages/contracts`. +- A main realtime dashboard at `/`. +- A replay workstation at `/replay`. +- A heatmap page at `/heatmap`. +- A price-only experimental page at `/experimental`. +- Collector events that already carry option `last`, `bid_size`, `ask_size`, `volume`, `open_interest`, `ibkr_delta`, `ibkr_gamma`, `ibkr_vega`, and `ibkr_theta`. +- Existing `AnalyticsSnapshot` rows that currently expose bid, ask, mid, open interest, custom IV, custom gamma, custom vanna, and broker comparison fields. +- Backend experimental analytics patterns that safely build partial panel payloads and validate through generated contracts. + +The new work should follow the current experimental API and frontend patterns while keeping the flow estimator as a separate module. + +## Selected Approach + +Use a dedicated backend experimental-flow API. + +Data flow: + +```text +Live/replay AnalyticsSnapshot + -> experimental_flow service + -> previous-snapshot comparator + -> flow estimator + -> confidence + diagnostics + -> typed ExperimentalFlow payload + -> /experimental-2 cockpit +``` + +Alternatives considered: + +- Frontend-only estimator: fastest, but weak for replay validation and loses state on refresh. +- Extending the existing experimental payload: less API surface, but mixes price-only research with flow inference and makes `/experimental` heavier. +- Dedicated backend API: best isolation, better tests, and cleaner replay validation. + +The selected approach is the dedicated backend API. + +## Backend Architecture + +Add a new package: + +```text +apps/api/gammascope_api/experimental_flow/ + __init__.py + estimator.py + service.py +``` + +Add a new route module: + +```text +apps/api/gammascope_api/routes/experimental_flow.py +``` + +Routes: + +```http +GET /api/spx/0dte/experimental-flow/latest +GET /api/spx/0dte/experimental-flow/replay +``` + +`latest` mode should compare the current live snapshot with the previous live snapshot available to the service. If no previous snapshot exists, return an `insufficient_data` style payload with diagnostics rather than failing. + +`replay` mode should run the same estimator over persisted replay snapshots. V1 should support enough replay validation to compare estimated pressure against the next selected SPX horizon, but it should not become a general-purpose research/backtest framework. + +## Contract Shape + +Add a new JSON Schema: + +```text +packages/contracts/schemas/experimental-flow.schema.json +``` + +Generate matching TypeScript and Python contracts. + +Payload outline: + +```ts +ExperimentalFlow { + schema_version: "1.0.0" + meta: { + mode: "latest" | "replay" + symbol: "SPX" + expiry: string + generatedAt: string + sourceSessionId: string + currentSnapshotTime: string + previousSnapshotTime: string | null + } + summary: { + estimatedBuyContracts: number + estimatedSellContracts: number + netEstimatedContracts: number + netPremiumFlow: number + netDeltaFlow: number | null + netGammaFlow: number | null + estimatedDealerGammaPressure: number | null + confidence: "high" | "medium" | "low" | "unknown" + } + strikeRows: StrikeFlowRow[] + contractRows: ContractFlowRow[] + replayValidation: ReplayValidation | null + diagnostics: Diagnostic[] +} +``` + +Strike rows: + +```ts +StrikeFlowRow { + strike: number + callBuyContracts: number + callSellContracts: number + putBuyContracts: number + putSellContracts: number + netPremiumFlow: number + netDeltaFlow: number | null + netGammaFlow: number | null + estimatedDealerGammaPressure: number | null + openingScore: number + closingScore: number + confidence: "high" | "medium" | "low" | "unknown" + tags: string[] +} +``` + +Contract rows: + +```ts +ContractFlowRow { + contractId: string + right: "call" | "put" + strike: number + volumeDelta: number + aggressor: "buy" | "weak_buy" | "sell" | "weak_sell" | "unknown" + signedContracts: number + premiumFlow: number + deltaFlow: number | null + gammaFlow: number | null + vannaFlow: number | null + thetaFlow: number | null + openingScore: number + closingScore: number + confidence: "high" | "medium" | "low" | "unknown" + diagnostics: string[] +} +``` + +Replay validation: + +```ts +ReplayValidation { + horizonMinutes: 5 | 15 | 30 + rows: ReplayValidationRow[] + hitRate: number | null +} +``` + +## Required Snapshot Inputs + +The estimator needs these row fields: + +- `last` +- `volume` +- `bid_size` +- `ask_size` +- `open_interest` +- `custom_iv` +- `custom_gamma` +- `custom_vanna` +- `ibkr_delta` +- `ibkr_vega` +- `ibkr_theta` + +The current collector event path already captures most of these fields. The first implementation should expose the needed optional fields through the flow API without changing the main dashboard rendering behavior. If `AnalyticsSnapshot.rows` is extended, the new fields must be optional and all existing consumers must continue to work. + +## Estimation Formulas + +For each contract matched between current and previous snapshots: + +```text +volumeDelta = max(0, current.volume - previous.volume) +priceChange = current.last_or_mid - previous.last_or_mid +spread = current.ask - current.bid +spreadRatio = spread / current.mid +``` + +Aggressor estimate: + +```text +if current.last >= previous.ask: buy +elif current.last <= previous.bid: sell +elif current.last >= current.ask: buy +elif current.last <= current.bid: sell +elif priceChange > 0: weak_buy +elif priceChange < 0: weak_sell +else: unknown +``` + +Weights: + +```text +buy = +1 +weak_buy = +0.5 +sell = -1 +weak_sell = -0.5 +unknown = 0 +``` + +Signed flow: + +```text +signedContracts = volumeDelta * aggressorWeight +premiumFlow = signedContracts * current.mid * 100 +deltaFlow = signedContracts * delta * spot * 100 +gammaFlow = signedContracts * gamma * spot^2 * 0.01 * 100 +vannaFlow = signedContracts * vanna * spot * 100 +thetaFlow = signedContracts * theta * 100 +estimatedDealerGammaPressure = -gammaFlow +``` + +When a Greek is missing, the corresponding flow should be `null` and the row should include a diagnostic tag. + +## Open/Close Proxy Scores + +The estimator cannot identify true open or close activity from free snapshots. It should provide proxy scores only. + +Opening score increases when: + +- `volumeDelta / openInterest` is high. +- Ask-side buying lifts IV. +- Bid-side put selling lifts IV. +- Same-direction flow repeats at the same strike. +- Spread is tight enough to trust side classification. + +Closing score increases when: + +- Volume hits bid while IV falls. +- Price decays while volume increases. +- Same-strike flow reverses from the prior interval. +- Quote quality is weak enough that open/close should remain unknown rather than directional. + +Scores should be normalized from `0` to `1`. When inputs are missing, use lower confidence rather than inventing certainty. + +## Confidence And Diagnostics + +Confidence should combine: + +```text +confidence = quoteQuality * volumeSignal * aggressorClarity * greekCoverage +``` + +Map numeric confidence into: + +- `high` +- `medium` +- `low` +- `unknown` + +Each contract row should carry diagnostics such as: + +- `missing_previous_snapshot` +- `missing_volume` +- `no_volume_delta` +- `missing_last` +- `wide_spread` +- `crossed_quote` +- `missing_delta` +- `missing_gamma` +- `missing_vanna` +- `missing_theta` +- `aggressor_unknown` +- `open_close_proxy_only` + +The payload-level diagnostics should summarize any systemic issue, such as insufficient prior snapshot, partial chain coverage, or low Greek coverage. + +## Frontend UI + +Add: + +```text +apps/web/app/experimental-2/page.tsx +apps/web/components/ExperimentalFlowDashboard.tsx +apps/web/components/experimental-flow/ +``` + +Use the Flow Cockpit layout: + +- Top navigation includes `Experimental 2`. +- KPI strip: + - estimated buy contracts + - estimated sell contracts + - net contracts + - net premium flow + - dealer gamma pressure + - confidence +- Main strike ladder: + - centered around spot + - rows by strike + - call buy/sell + - put buy/sell + - net premium + - dealer gamma pressure + - confidence + - color intensity by dealer gamma pressure +- Right rail: + - aggressor mix + - opening/closing proxy + - quote/Greek coverage diagnostics +- Bottom audit table: + - contract-level rows + - volume delta + - aggressor + - signed contracts + - premium flow + - Greek flow + - confidence + - diagnostics + +The page should be dense and operational, not explanatory or marketing-oriented. It should use concise labels and keep formulas in diagnostics/audit surfaces rather than long in-app prose. + +## Replay Validation + +Replay validation should stay bounded in v1. + +Supported modes: + +- Latest/live estimate: compare current snapshot to previous live snapshot. +- Replay validate: run the estimator across persisted replay snapshots and compare pressure direction to the next selected SPX horizon. + +Initial horizons: + +- 5 minutes +- 15 minutes +- 30 minutes + +Validation rows should include: + +- source snapshot time +- pressure direction +- pressure magnitude +- next spot +- realized move +- hit/miss/null classification + +This is meant to answer whether the estimated flow pointed in the right direction recently. It is not a full strategy backtester. + +## Data Integrity Rules + +- Never label inferred data as official customer or market-maker data. +- Keep estimated fields named with `estimated` or `proxy` where needed. +- Include diagnostics whenever an estimate depends on weak assumptions. +- Return partial payloads instead of failing the entire API when one panel cannot be computed. +- Treat missing previous snapshots as `insufficient_data`. +- Keep existing dashboard contracts backward compatible. + +## Testing + +Backend tests: + +- Estimator computes volume deltas from current and previous rows. +- Aggressor classification handles buy, sell, weak buy, weak sell, and unknown. +- Negative or reset cumulative volume is clamped to zero with diagnostics. +- Greek-weighted flow returns `null` and diagnostics when Greeks are missing. +- Strike aggregation sums calls and puts correctly. +- Dealer gamma pressure is the inverse of estimated gamma flow. +- Open/close proxy scores stay in `[0, 1]`. +- Low-quality quotes reduce confidence. +- Latest route returns a typed insufficient-data payload when previous snapshot is unavailable. +- Replay route computes bounded validation rows. + +Frontend tests: + +- `/experimental-2` renders with seed or fallback payload. +- KPI strip formats contracts, money, Greek flows, and confidence. +- Strike ladder sorts and centers rows around spot. +- Contract audit table shows diagnostics. +- Low-confidence and unknown rows render visibly. +- Navigation includes Experimental 2 without breaking existing tabs. + +Contract tests: + +- TypeScript and Python generated contracts accept the seed fixture. +- Invalid confidence labels and aggressor labels are rejected. +- Optional Greek flow fields accept `null`. + +## Acceptance Criteria + +- A user can open `/experimental-2` without affecting `/`, `/experimental`, `/replay`, or `/heatmap`. +- The page shows estimated buy/sell contracts, premium flow, Greek-weighted flow, dealer gamma pressure, and confidence. +- Every displayed estimate can be traced to contract-level audit rows. +- Missing or weak data is visible through diagnostics and confidence. +- Replay validation can compare estimated pressure direction against a selected future SPX horizon. +- The implementation remains SPX-only in v1 but does not block later multi-symbol expansion. From 70674eca7ab0a9aa7500843ccfd4a176e38098a4 Mon Sep 17 00:00:00 2001 From: Sakura <32687351+zifanzhou1024@users.noreply.github.com> Date: Sun, 3 May 2026 14:35:07 -0700 Subject: [PATCH 02/11] chore: add AMH nginx server setup --- .dockerignore | 29 ++ .gitignore | 2 + README.md | 2 +- apps/api/Dockerfile | 20 + apps/web/Dockerfile | 26 ++ docs/amh-nginx-server-setup.md | 440 ++++++++++++++++++ ops/amh-nginx/docker-compose.amh.yml | 95 ++++ .../gammascope.collector-client.env.example | 6 + ops/amh-nginx/gammascope.nginx.conf | 80 ++++ .../gammascope.production.env.example | 19 + .../gammascope_collector/publisher.py | 48 +- services/collector/tests/test_publisher.py | 54 +++ 12 files changed, 812 insertions(+), 9 deletions(-) create mode 100644 .dockerignore create mode 100644 apps/api/Dockerfile create mode 100644 apps/web/Dockerfile create mode 100644 docs/amh-nginx-server-setup.md create mode 100644 ops/amh-nginx/docker-compose.amh.yml create mode 100644 ops/amh-nginx/gammascope.collector-client.env.example create mode 100644 ops/amh-nginx/gammascope.nginx.conf create mode 100644 ops/amh-nginx/gammascope.production.env.example diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8921704 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,29 @@ +.git +.worktrees +.gammascope +.idea +.vscode +.superpowers +.DS_Store + +.env +.env.* + +node_modules +**/node_modules +.next +**/.next +coverage +dist +*.tsbuildinfo + +__pycache__ +*.py[cod] +.pytest_cache +.ruff_cache +.mypy_cache +.venv +venv +*.egg-info + +*.log diff --git a/.gitignore b/.gitignore index a223b91..4c43225 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ .env .env.* !.env.example +ops/amh-nginx/gammascope.production.env +ops/amh-nginx/gammascope.collector-client.env # Node node_modules/ diff --git a/README.md b/README.md index f1de4ae..6d5d5d0 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ GammaScope is being built in slices. The first slice establishes the local monorepo, shared contracts, seeded replay data, and smoke-testable API/web surfaces. -Deployment notes for the current Moomoo-backed dashboard and heatmap stack are in [docs/deployment.md](docs/deployment.md). +Deployment notes for the current Moomoo-backed dashboard and heatmap stack are in [docs/deployment.md](docs/deployment.md). For the AMH/Nginx remote server layout where your computer publishes Moomoo data to a server-hosted backend and frontend, use [docs/amh-nginx-server-setup.md](docs/amh-nginx-server-setup.md). Run: diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile new file mode 100644 index 0000000..13f8308 --- /dev/null +++ b/apps/api/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app/apps/api:/app/services/collector + +WORKDIR /app + +RUN python -m pip install --no-cache-dir --upgrade pip + +COPY apps/api/pyproject.toml apps/api/pyproject.toml +COPY apps/api/gammascope_api apps/api/gammascope_api +COPY packages/contracts/fixtures packages/contracts/fixtures +COPY services/collector/gammascope_collector services/collector/gammascope_collector + +RUN python -m pip install --no-cache-dir ./apps/api moomoo-api pandas + +EXPOSE 8000 + +CMD ["python", "-m", "uvicorn", "gammascope_api.main:app", "--app-dir", "apps/api", "--host", "0.0.0.0", "--port", "8000"] diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile new file mode 100644 index 0000000..e7eaa48 --- /dev/null +++ b/apps/web/Dockerfile @@ -0,0 +1,26 @@ +FROM node:22-slim + +ENV PNPM_HOME=/pnpm \ + PATH=/pnpm:$PATH \ + NEXT_TELEMETRY_DISABLED=1 + +WORKDIR /app + +RUN corepack enable + +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +COPY apps/web/package.json apps/web/package.json +COPY packages/contracts/package.json packages/contracts/package.json +RUN pnpm install --frozen-lockfile + +COPY apps/web apps/web +COPY packages/contracts packages/contracts + +ARG NEXT_PUBLIC_GAMMASCOPE_WS_URL=http://127.0.0.1:8000 +ENV NEXT_PUBLIC_GAMMASCOPE_WS_URL=$NEXT_PUBLIC_GAMMASCOPE_WS_URL + +RUN pnpm --filter @gammascope/web build + +EXPOSE 3000 + +CMD ["pnpm", "--filter", "@gammascope/web", "exec", "next", "start", "--hostname", "0.0.0.0", "--port", "3000"] diff --git a/docs/amh-nginx-server-setup.md b/docs/amh-nginx-server-setup.md new file mode 100644 index 0000000..369b229 --- /dev/null +++ b/docs/amh-nginx-server-setup.md @@ -0,0 +1,440 @@ +# AMH Nginx Server Setup Guide + +This guide moves GammaScope from a local-only setup to a server layout where AMH/Nginx is public, FastAPI and Next.js run on the server, and your computer keeps running Moomoo OpenD plus the collector. + +## Target Architecture + +```mermaid +flowchart LR + Mac["Your computer\nMoomoo OpenD + GammaScope collector"] -->|HTTPS bulk collector events + admin token| Nginx["AMH / Nginx\n80 and 443"] + Browser["Browser"] -->|HTTPS| Nginx + Nginx -->|collector endpoints and ws| API["FastAPI container\n127.0.0.1:8000"] + Nginx -->|web app and Next API routes| Web["Next.js container\n127.0.0.1:3000"] + Web -->|server-side API proxy + admin token| API + API --> Postgres["Postgres volume"] + API --> Redis["Redis container"] +``` + +The server does not need Moomoo OpenD. Keep OpenD on the computer that has your licensed data session, then publish snapshots to the server API. + +## What This Branch Adds + +- `apps/api/Dockerfile`: production container for FastAPI plus collector modules. +- `apps/web/Dockerfile`: production container for Next.js with a build-time public WebSocket origin. +- `ops/amh-nginx/docker-compose.amh.yml`: server compose stack for Postgres, Redis, API, and web. +- `ops/amh-nginx/gammascope.nginx.conf`: full Nginx vhost template for AMH/manual Nginx. +- `ops/amh-nginx/gammascope.production.env.example`: server environment template. +- `ops/amh-nginx/gammascope.collector-client.env.example`: local collector environment template. +- `services/collector/gammascope_collector/publisher.py`: collector publishing now reads `GAMMASCOPE_ADMIN_TOKEN` and sends `X-GammaScope-Admin-Token`. + +## Sources Checked + +- AMH official installation docs say AMH 7.3 should be installed on a clean Debian, CentOS, or Ubuntu server and supports Nginx-based environments: https://amh.sh/install.htm +- AMH official docs describe installing server/environment modules such as Nginx, LNMP/LNGX, and AMSSL from the panel: https://amh.sh/doc.htm +- Nginx official reverse proxy docs use `proxy_pass` and `proxy_set_header` to forward application requests: https://docs.nginx.com/nginx/admin-guide/web-server/reverse-proxy/ +- Certbot official instructions recommend the snap-based Certbot install for Nginx on Ubuntu and note that port 80 HTTP should already work before issuing the certificate: https://certbot.eff.org/instructions?ws=nginx&os=ubuntufocal +- Docker official docs recommend installing Docker Engine from Docker's apt repository and using the Compose plugin on Linux: https://docs.docker.com/engine/install/ubuntu/ and https://docs.docker.com/compose/install/linux/ + +I could open the shared ChatGPT URL, but the shared page did not expose the actual chat content without login in this environment. This guide is based on the repo and primary docs above. + +## Prerequisites + +You need: + +- A VPS with a clean supported Linux image. Ubuntu 24.04 LTS is the most straightforward choice. +- AMH installed with an Nginx-based environment such as LNGX or LNMP. +- A domain or subdomain, for example `gammascope.example.com`. +- DNS `A` record pointing that domain to the server public IP. +- Cloud firewall/security group opened for `80/tcp` and `443/tcp`. +- SSH access to the server. +- Docker Engine and Docker Compose plugin on the server. +- Moomoo OpenD running on your own computer. + +Do not open Postgres, Redis, FastAPI port `8000`, or Next.js port `3000` to the internet. The production compose file binds API and web to `127.0.0.1` only. + +## 1. Prepare AMH and Nginx + +Install AMH from the official AMH install page on a clean server. During AMH setup, choose an Nginx-capable environment. If AMH is already installed, install or enable: + +- Nginx server software. +- LNGX or LNMP environment. +- AMSSL or another SSL certificate module if you want AMH to manage certificates. + +In the server provider firewall, allow only: + +```text +22/tcp SSH, ideally locked to your IP +80/tcp HTTP certificate challenge and redirect +443/tcp HTTPS app +AMH panel port, only from your IP +``` + +Keep AMH's panel port restricted to your IP if the provider supports security group source IP rules. + +## 2. Install Docker on the Server + +Follow Docker's official Ubuntu repository instructions. The short version for Ubuntu is: + +```bash +sudo apt-get update +sudo apt-get install -y ca-certificates curl +sudo install -m 0755 -d /etc/apt/keyrings +sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +sudo chmod a+r /etc/apt/keyrings/docker.asc + +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" \ + | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + +sudo apt-get update +sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +sudo docker run hello-world +docker compose version +``` + +Use `sudo docker ...` unless you intentionally add your SSH user to the `docker` group. + +## 3. Put the Repo on the Server + +Use `/opt/gammascope` for the app: + +```bash +sudo mkdir -p /opt/gammascope +sudo chown "$USER":"$USER" /opt/gammascope +cd /opt/gammascope +``` + +If this branch has been pushed to GitHub: + +```bash +git clone . +git fetch origin +git switch codex/amh-nginx-server-setup +``` + +If the branch has not been pushed yet, send it from your computer: + +```bash +rsync -az --delete \ + --exclude .git \ + --exclude node_modules \ + --exclude .venv \ + --exclude .gammascope \ + /Users/sakura/WebstormProjects/gamma-scope/.worktrees/amh-nginx-server-setup/ \ + @:/opt/gammascope/ +``` + +## 4. Configure Server Secrets + +Create the production env file on the server: + +```bash +cd /opt/gammascope +cp ops/amh-nginx/gammascope.production.env.example ops/amh-nginx/gammascope.production.env +``` + +Generate secrets: + +```bash +openssl rand -hex 24 +openssl rand -hex 32 +openssl rand -base64 48 | tr -d '\n' && echo +``` + +Edit `ops/amh-nginx/gammascope.production.env`: + +```text +GAMMASCOPE_PUBLIC_ORIGIN=https://gammascope.example.com +GAMMASCOPE_POSTGRES_PASSWORD= +GAMMASCOPE_ADMIN_TOKEN= +GAMMASCOPE_WEB_ADMIN_PASSWORD= +GAMMASCOPE_WEB_ADMIN_SESSION_SECRET= +``` + +`GAMMASCOPE_PUBLIC_ORIGIN` is compiled into the Next.js image. If you change the domain later, rebuild the web image. + +## 5. Start Backend and Frontend Containers + +From `/opt/gammascope`: + +```bash +docker compose \ + --env-file ops/amh-nginx/gammascope.production.env \ + -f ops/amh-nginx/docker-compose.amh.yml \ + up -d --build +``` + +Check status: + +```bash +docker compose \ + --env-file ops/amh-nginx/gammascope.production.env \ + -f ops/amh-nginx/docker-compose.amh.yml \ + ps +``` + +Check local-only endpoints from the server: + +```bash +curl -fsS http://127.0.0.1:3000/ >/dev/null && echo web-ok +curl -fsS http://127.0.0.1:8000/api/spx/0dte/replay/sessions | python3 -m json.tool +``` + +Useful logs: + +```bash +docker compose --env-file ops/amh-nginx/gammascope.production.env -f ops/amh-nginx/docker-compose.amh.yml logs -f api +docker compose --env-file ops/amh-nginx/gammascope.production.env -f ops/amh-nginx/docker-compose.amh.yml logs -f web +``` + +## 6. Configure AMH/Nginx + +There are two workable paths. + +### Option A: AMH Panel Vhost + +Use AMH to create a site/vhost for `gammascope.example.com`, enable SSL with AMSSL, then add custom Nginx rules equivalent to these locations: + +```nginx +location = /api/spx/0dte/collector/events { + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 30s; + proxy_send_timeout 30s; +} + +location = /api/spx/0dte/collector/events/bulk { + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 60s; + proxy_send_timeout 60s; +} + +location ^~ /ws/ { + proxy_pass http://127.0.0.1:8000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 1h; + proxy_send_timeout 1h; + proxy_buffering off; +} + +location / { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 60s; + proxy_send_timeout 60s; + proxy_buffering off; +} +``` + +Use this option when AMH owns certificate renewal and vhost generation. + +### Option B: Full Nginx Template + +Copy the repo template and edit the domain/certificate paths: + +```bash +sudo cp /opt/gammascope/ops/amh-nginx/gammascope.nginx.conf /etc/nginx/conf.d/gammascope.conf +sudo sed -i 's/gammascope.example.com/your-real-domain.example/g' /etc/nginx/conf.d/gammascope.conf +``` + +If your AMH Nginx is not using `/etc/nginx/conf.d`, locate its active config: + +```bash +sudo nginx -T 2>/dev/null | grep -n "include .*conf" +``` + +Then place the file in an included directory, or paste the server blocks into the AMH-managed custom config area. + +Validate and reload: + +```bash +sudo nginx -t +sudo nginx -s reload +``` + +## 7. Configure HTTPS + +If AMH/AMSSL handles HTTPS, issue the certificate there and make sure the Nginx vhost points at that certificate. + +If using Certbot directly on Ubuntu: + +```bash +sudo snap install --classic certbot +sudo ln -sf /snap/bin/certbot /usr/bin/certbot +sudo certbot --nginx -d gammascope.example.com +sudo certbot renew --dry-run +``` + +Certbot expects your domain to already resolve to the server and port `80` to be reachable. + +## 8. Smoke Test the Public Server + +From your computer: + +```bash +curl -I https://gammascope.example.com/ +curl -fsS https://gammascope.example.com/api/spx/0dte/replay/sessions | python3 -m json.tool +``` + +Collector ingestion should require the admin token: + +```bash +curl -i -X POST https://gammascope.example.com/api/spx/0dte/collector/events/bulk \ + -H 'Content-Type: application/json' \ + --data '[]' +``` + +Expected without token: `403`. + +With the token: + +```bash +curl -i -X POST https://gammascope.example.com/api/spx/0dte/collector/events/bulk \ + -H "X-GammaScope-Admin-Token: " \ + -H 'Content-Type: application/json' \ + --data '[]' +``` + +Expected with an empty batch: `200` and `accepted_count: 0`. + +## 9. Configure Your Computer as the Collector Client + +On your computer, from the GammaScope repo: + +```bash +cp ops/amh-nginx/gammascope.collector-client.env.example ops/amh-nginx/gammascope.collector-client.env +``` + +Edit it: + +```text +GAMMASCOPE_SERVER_API=https://gammascope.example.com +GAMMASCOPE_ADMIN_TOKEN= +GAMMASCOPE_MOOMOO_HOST=127.0.0.1 +GAMMASCOPE_MOOMOO_PORT=11111 +GAMMASCOPE_RUT_SPOT=2050 +GAMMASCOPE_NDX_SPOT=18300 +``` + +Load it into your shell: + +```bash +set -a +. ops/amh-nginx/gammascope.collector-client.env +set +a +``` + +Make sure Moomoo OpenD is running locally, then run a one-loop smoke publish: + +```bash +pnpm collector:moomoo-snapshot -- \ + --host "$GAMMASCOPE_MOOMOO_HOST" \ + --port "$GAMMASCOPE_MOOMOO_PORT" \ + --api "$GAMMASCOPE_SERVER_API" \ + --spot RUT="$GAMMASCOPE_RUT_SPOT" \ + --spot NDX="$GAMMASCOPE_NDX_SPOT" \ + --max-loops 1 \ + --publish +``` + +Then run continuously: + +```bash +pnpm collector:moomoo-snapshot -- \ + --host "$GAMMASCOPE_MOOMOO_HOST" \ + --port "$GAMMASCOPE_MOOMOO_PORT" \ + --api "$GAMMASCOPE_SERVER_API" \ + --spot RUT="$GAMMASCOPE_RUT_SPOT" \ + --spot NDX="$GAMMASCOPE_NDX_SPOT" \ + --publish +``` + +Because this branch updates the publisher, the collector automatically reads `GAMMASCOPE_ADMIN_TOKEN` from the environment and sends it as `X-GammaScope-Admin-Token`. + +## 10. Confirm Server Data + +On the server: + +```bash +docker compose --env-file ops/amh-nginx/gammascope.production.env -f ops/amh-nginx/docker-compose.amh.yml exec postgres \ + psql -U gammascope -d gammascope -c " + select session_id, symbol, snapshot_count, end_time + from replay_sessions + where session_id like 'moomoo-%-0dte-live' + order by session_id; + " +``` + +From your computer: + +```bash +curl -fsS "https://gammascope.example.com/api/spx/0dte/heatmap/latest?metric=gex&symbol=SPX" | python3 -m json.tool +``` + +Open: + +```text +https://gammascope.example.com/ +https://gammascope.example.com/heatmap +``` + +## 11. Operating Commands + +Rebuild after code changes: + +```bash +cd /opt/gammascope +git pull +docker compose --env-file ops/amh-nginx/gammascope.production.env -f ops/amh-nginx/docker-compose.amh.yml up -d --build +``` + +Restart: + +```bash +docker compose --env-file ops/amh-nginx/gammascope.production.env -f ops/amh-nginx/docker-compose.amh.yml restart +``` + +Stop: + +```bash +docker compose --env-file ops/amh-nginx/gammascope.production.env -f ops/amh-nginx/docker-compose.amh.yml down +``` + +Backup Postgres: + +```bash +docker compose --env-file ops/amh-nginx/gammascope.production.env -f ops/amh-nginx/docker-compose.amh.yml exec postgres \ + pg_dump -U gammascope gammascope > "gammascope-$(date +%Y%m%d-%H%M%S).sql" +``` + +## Troubleshooting + +If the site does not load, run `curl -I http://127.0.0.1:3000/` on the server. If local curl works, the problem is Nginx/AMH. If local curl fails, inspect `docker compose ... logs web`. + +If collector publish returns `403`, the computer's `GAMMASCOPE_ADMIN_TOKEN` does not match the server's `GAMMASCOPE_ADMIN_TOKEN`, or the server was not restarted after changing the env file. + +If collector publish cannot connect, check DNS, HTTPS, firewall, and the Nginx collector locations. The collector should publish to the public origin, not to `127.0.0.1`. + +If the browser live WebSocket is unavailable in private mode, that is expected unless the browser has an admin token. The web app's server-side API routes can still fetch live data using the server-side `GAMMASCOPE_ADMIN_TOKEN`, and the dashboard should fall back to polling. + +If AMH overwrites manual Nginx edits, move the custom locations into AMH's supported custom vhost/rules field or keep a copy of `ops/amh-nginx/gammascope.nginx.conf` and re-apply after AMH regenerates configs. diff --git a/ops/amh-nginx/docker-compose.amh.yml b/ops/amh-nginx/docker-compose.amh.yml new file mode 100644 index 0000000..fe655b9 --- /dev/null +++ b/ops/amh-nginx/docker-compose.amh.yml @@ -0,0 +1,95 @@ +name: gammascope + +x-api-environment: &api-environment + GAMMASCOPE_DATABASE_URL: postgresql://${GAMMASCOPE_POSTGRES_USER:-gammascope}:${GAMMASCOPE_POSTGRES_PASSWORD:?set GAMMASCOPE_POSTGRES_PASSWORD}@postgres:5432/${GAMMASCOPE_POSTGRES_DB:-gammascope} + GAMMASCOPE_REDIS_URL: redis://redis:6379/0 + GAMMASCOPE_REPLAY_CAPTURE_INTERVAL_SECONDS: ${GAMMASCOPE_REPLAY_CAPTURE_INTERVAL_SECONDS:-5} + GAMMASCOPE_REPLAY_RETENTION_DAYS: ${GAMMASCOPE_REPLAY_RETENTION_DAYS:-20} + GAMMASCOPE_SAVED_VIEW_RETENTION_DAYS: ${GAMMASCOPE_SAVED_VIEW_RETENTION_DAYS:-90} + GAMMASCOPE_PRIVATE_MODE_ENABLED: ${GAMMASCOPE_PRIVATE_MODE_ENABLED:-true} + GAMMASCOPE_ADMIN_TOKEN: ${GAMMASCOPE_ADMIN_TOKEN:?set GAMMASCOPE_ADMIN_TOKEN} + +services: + postgres: + image: postgres:16-alpine + restart: unless-stopped + environment: + POSTGRES_DB: ${GAMMASCOPE_POSTGRES_DB:-gammascope} + POSTGRES_USER: ${GAMMASCOPE_POSTGRES_USER:-gammascope} + POSTGRES_PASSWORD: ${GAMMASCOPE_POSTGRES_PASSWORD:?set GAMMASCOPE_POSTGRES_PASSWORD} + volumes: + - gammascope-postgres:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${GAMMASCOPE_POSTGRES_USER:-gammascope} -d ${GAMMASCOPE_POSTGRES_DB:-gammascope}"] + interval: 5s + timeout: 5s + retries: 10 + + redis: + image: redis:7-alpine + restart: unless-stopped + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 10 + + api: + image: gamma-scope-api:amh + build: + context: ../.. + dockerfile: apps/api/Dockerfile + restart: unless-stopped + environment: + <<: *api-environment + ports: + - "127.0.0.1:${GAMMASCOPE_API_HOST_PORT:-8000}:8000" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + healthcheck: + test: + [ + "CMD-SHELL", + "python -c \"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/api/spx/0dte/replay/sessions', timeout=2).read()\"" + ] + interval: 10s + timeout: 5s + retries: 12 + start_period: 20s + + web: + image: gamma-scope-web:amh + build: + context: ../.. + dockerfile: apps/web/Dockerfile + args: + NEXT_PUBLIC_GAMMASCOPE_WS_URL: ${GAMMASCOPE_PUBLIC_ORIGIN:?set GAMMASCOPE_PUBLIC_ORIGIN} + restart: unless-stopped + environment: + GAMMASCOPE_API_BASE_URL: http://api:8000 + GAMMASCOPE_ADMIN_TOKEN: ${GAMMASCOPE_ADMIN_TOKEN:?set GAMMASCOPE_ADMIN_TOKEN} + GAMMASCOPE_WEB_ADMIN_USERNAME: ${GAMMASCOPE_WEB_ADMIN_USERNAME:-admin} + GAMMASCOPE_WEB_ADMIN_PASSWORD: ${GAMMASCOPE_WEB_ADMIN_PASSWORD:?set GAMMASCOPE_WEB_ADMIN_PASSWORD} + GAMMASCOPE_WEB_ADMIN_SESSION_SECRET: ${GAMMASCOPE_WEB_ADMIN_SESSION_SECRET:?set GAMMASCOPE_WEB_ADMIN_SESSION_SECRET} + GAMMASCOPE_REPLAY_IMPORT_MAX_BYTES: ${GAMMASCOPE_REPLAY_IMPORT_MAX_BYTES:-104857600} + ports: + - "127.0.0.1:${GAMMASCOPE_WEB_HOST_PORT:-3000}:3000" + depends_on: + api: + condition: service_healthy + healthcheck: + test: + [ + "CMD-SHELL", + "node -e \"fetch('http://127.0.0.1:3000/').then((r)=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))\"" + ] + interval: 10s + timeout: 5s + retries: 12 + start_period: 20s + +volumes: + gammascope-postgres: diff --git a/ops/amh-nginx/gammascope.collector-client.env.example b/ops/amh-nginx/gammascope.collector-client.env.example new file mode 100644 index 0000000..2f7ca35 --- /dev/null +++ b/ops/amh-nginx/gammascope.collector-client.env.example @@ -0,0 +1,6 @@ +GAMMASCOPE_SERVER_API=https://gammascope.example.com +GAMMASCOPE_ADMIN_TOKEN=replace-with-the-same-admin-token-used-on-the-server +GAMMASCOPE_MOOMOO_HOST=127.0.0.1 +GAMMASCOPE_MOOMOO_PORT=11111 +GAMMASCOPE_RUT_SPOT=2050 +GAMMASCOPE_NDX_SPOT=18300 diff --git a/ops/amh-nginx/gammascope.nginx.conf b/ops/amh-nginx/gammascope.nginx.conf new file mode 100644 index 0000000..c998dde --- /dev/null +++ b/ops/amh-nginx/gammascope.nginx.conf @@ -0,0 +1,80 @@ +# Full Nginx vhost template for GammaScope behind AMH. +# Replace gammascope.example.com and certificate paths before enabling. + +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + +upstream gammascope_web { + server 127.0.0.1:3000; + keepalive 32; +} + +upstream gammascope_api { + server 127.0.0.1:8000; + keepalive 16; +} + +server { + listen 80; + listen [::]:80; + server_name gammascope.example.com; + + location /.well-known/acme-challenge/ { + root /home/wwwroot/gammascope; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name gammascope.example.com; + + ssl_certificate /etc/letsencrypt/live/gammascope.example.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/gammascope.example.com/privkey.pem; + + client_max_body_size 100m; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + + location = /api/spx/0dte/collector/events { + proxy_pass http://gammascope_api; + proxy_read_timeout 30s; + proxy_send_timeout 30s; + } + + location = /api/spx/0dte/collector/events/bulk { + proxy_pass http://gammascope_api; + proxy_read_timeout 60s; + proxy_send_timeout 60s; + } + + location ^~ /ws/ { + proxy_pass http://gammascope_api; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_read_timeout 1h; + proxy_send_timeout 1h; + proxy_buffering off; + } + + location / { + proxy_pass http://gammascope_web; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_read_timeout 60s; + proxy_send_timeout 60s; + proxy_buffering off; + } +} diff --git a/ops/amh-nginx/gammascope.production.env.example b/ops/amh-nginx/gammascope.production.env.example new file mode 100644 index 0000000..10000e6 --- /dev/null +++ b/ops/amh-nginx/gammascope.production.env.example @@ -0,0 +1,19 @@ +GAMMASCOPE_PUBLIC_ORIGIN=https://gammascope.example.com + +GAMMASCOPE_POSTGRES_DB=gammascope +GAMMASCOPE_POSTGRES_USER=gammascope +GAMMASCOPE_POSTGRES_PASSWORD=replace-with-a-long-random-database-password + +GAMMASCOPE_PRIVATE_MODE_ENABLED=true +GAMMASCOPE_ADMIN_TOKEN=replace-with-a-long-random-admin-token + +GAMMASCOPE_WEB_ADMIN_USERNAME=admin +GAMMASCOPE_WEB_ADMIN_PASSWORD=replace-with-a-long-random-web-password +GAMMASCOPE_WEB_ADMIN_SESSION_SECRET=replace-with-at-least-32-random-characters + +GAMMASCOPE_API_HOST_PORT=8000 +GAMMASCOPE_WEB_HOST_PORT=3000 +GAMMASCOPE_REPLAY_CAPTURE_INTERVAL_SECONDS=5 +GAMMASCOPE_REPLAY_RETENTION_DAYS=20 +GAMMASCOPE_SAVED_VIEW_RETENTION_DAYS=90 +GAMMASCOPE_REPLAY_IMPORT_MAX_BYTES=104857600 diff --git a/services/collector/gammascope_collector/publisher.py b/services/collector/gammascope_collector/publisher.py index 6eb6127..d82a05e 100644 --- a/services/collector/gammascope_collector/publisher.py +++ b/services/collector/gammascope_collector/publisher.py @@ -2,6 +2,7 @@ import argparse import json +import os import sys from collections.abc import Callable, Iterable, Sequence from dataclasses import asdict, dataclass @@ -13,6 +14,8 @@ COLLECTOR_EVENT_PATH = "/api/spx/0dte/collector/events" COLLECTOR_EVENTS_BULK_PATH = "/api/spx/0dte/collector/events/bulk" +ADMIN_TOKEN_ENV = "GAMMASCOPE_ADMIN_TOKEN" +ADMIN_TOKEN_HEADER = "X-GammaScope-Admin-Token" PostJson = Callable[[str, dict[str, object]], dict[str, Any]] PostJsonBatch = Callable[[str, list[dict[str, object]]], dict[str, Any]] @@ -44,14 +47,18 @@ def publish_events( events: Iterable[dict[str, object]], *, api_base: str, + admin_token: str | None = None, post_json: PostJson | None = None, ) -> PublishSummary: endpoint = collector_event_endpoint(api_base) - sender = post_json or _post_json + resolved_admin_token = _resolved_admin_token(admin_token) event_types: list[str] = [] for event in events: - response = sender(endpoint, event) + if post_json is None: + response = _post_json(endpoint, event, admin_token=resolved_admin_token) + else: + response = post_json(endpoint, event) if response.get("accepted") is not True: raise PublishError(f"Collector event rejected by {endpoint}: {response}") event_types.append(str(response.get("event_type", "unknown"))) @@ -63,12 +70,16 @@ def publish_events_bulk( events: Iterable[dict[str, object]], *, api_base: str, + admin_token: str | None = None, post_json: PostJsonBatch | None = None, ) -> PublishSummary: endpoint = collector_events_bulk_endpoint(api_base) batch = list(events) - sender = post_json or _post_json_batch - response = sender(endpoint, batch) + resolved_admin_token = _resolved_admin_token(admin_token) + if post_json is None: + response = _post_json_batch(endpoint, batch, admin_token=resolved_admin_token) + else: + response = post_json(endpoint, batch) if response.get("accepted") is not True: raise PublishError(f"Collector event batch rejected by {endpoint}: {response}") event_types = [str(event_type) for event_type in response.get("event_types", [])] @@ -89,12 +100,12 @@ def main(argv: Sequence[str] | None = None, *, post_json: PostJson | None = None print(json.dumps(summary.as_dict(), separators=(",", ":"), sort_keys=True)) -def _post_json(endpoint: str, event: dict[str, object]) -> dict[str, Any]: +def _post_json(endpoint: str, event: dict[str, object], *, admin_token: str | None = None) -> dict[str, Any]: body = json.dumps(event).encode("utf-8") request = Request( endpoint, data=body, - headers={"Content-Type": "application/json", "Accept": "application/json"}, + headers=_json_headers(admin_token), method="POST", ) try: @@ -107,12 +118,17 @@ def _post_json(endpoint: str, event: dict[str, object]) -> dict[str, Any]: raise PublishError(f"Could not reach collector ingestion endpoint {endpoint}: {exc.reason}") from exc -def _post_json_batch(endpoint: str, events: list[dict[str, object]]) -> dict[str, Any]: +def _post_json_batch( + endpoint: str, + events: list[dict[str, object]], + *, + admin_token: str | None = None, +) -> dict[str, Any]: body = json.dumps(events).encode("utf-8") request = Request( endpoint, data=body, - headers={"Content-Type": "application/json", "Accept": "application/json"}, + headers=_json_headers(admin_token), method="POST", ) try: @@ -129,6 +145,22 @@ def _parse_strikes(value: str) -> list[float]: return [float(part.strip()) for part in value.split(",") if part.strip()] +def _resolved_admin_token(admin_token: str | None) -> str | None: + token = admin_token if admin_token is not None else os.environ.get(ADMIN_TOKEN_ENV) + if token is None: + return None + stripped_token = token.strip() + return stripped_token or None + + +def _json_headers(admin_token: str | None = None) -> dict[str, str]: + headers = {"Content-Type": "application/json", "Accept": "application/json"} + resolved_admin_token = _resolved_admin_token(admin_token) + if resolved_admin_token is not None: + headers[ADMIN_TOKEN_HEADER] = resolved_admin_token + return headers + + def _normalize_argv(argv: Sequence[str] | None) -> Sequence[str] | None: if argv and argv[0] == "--": return argv[1:] diff --git a/services/collector/tests/test_publisher.py b/services/collector/tests/test_publisher.py index be53436..34d7c7d 100644 --- a/services/collector/tests/test_publisher.py +++ b/services/collector/tests/test_publisher.py @@ -3,6 +3,7 @@ import pytest +import gammascope_collector.publisher as publisher from gammascope_collector.events import health_event, underlying_tick_event from gammascope_collector.publisher import ( PublishError, @@ -35,6 +36,59 @@ def test_collector_events_bulk_endpoint_joins_api_base() -> None: ) +def test_publish_events_uses_admin_token_from_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GAMMASCOPE_ADMIN_TOKEN", "server-admin-token") + event = health_event( + collector_id="local-dev", + status="connected", + ibkr_account_mode="paper", + message="ok", + event_time=EVENT_TIME, + received_time=EVENT_TIME, + ) + captured_tokens: list[str | None] = [] + + def fake_post_json( + _endpoint: str, + _event: dict[str, object], + *, + admin_token: str | None = None, + ) -> dict[str, object]: + captured_tokens.append(admin_token) + return {"accepted": True, "event_type": "CollectorHealth"} + + monkeypatch.setattr(publisher, "_post_json", fake_post_json) + + publish_events([event], api_base="http://testserver") + + assert captured_tokens == ["server-admin-token"] + + +def test_post_json_adds_admin_token_header(monkeypatch: pytest.MonkeyPatch) -> None: + captured_requests: list[publisher.Request] = [] + + class FakeResponse: + def __enter__(self) -> object: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def read(self) -> bytes: + return b'{"accepted": true, "event_type": "CollectorHealth"}' + + def fake_urlopen(request: publisher.Request, timeout: float) -> FakeResponse: + captured_requests.append(request) + assert timeout == 5 + return FakeResponse() + + monkeypatch.setattr(publisher, "urlopen", fake_urlopen) + + publisher._post_json("http://testserver/collector", {"ok": True}, admin_token="server-admin-token") + + assert captured_requests[0].get_header("X-gammascope-admin-token") == "server-admin-token" + + def test_publish_events_posts_each_event_to_ingestion_endpoint() -> None: events = [ health_event( From acb764f7f70d37a97ec2ab0de245da0e0c7325ab Mon Sep 17 00:00:00 2001 From: Sakura <32687351+zifanzhou1024@users.noreply.github.com> Date: Sun, 3 May 2026 18:35:02 -0700 Subject: [PATCH 03/11] docs: use Debian Docker setup for AMH deploy --- docs/amh-nginx-server-setup.md | 64 ++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 19 deletions(-) diff --git a/docs/amh-nginx-server-setup.md b/docs/amh-nginx-server-setup.md index 369b229..31d1b25 100644 --- a/docs/amh-nginx-server-setup.md +++ b/docs/amh-nginx-server-setup.md @@ -32,8 +32,8 @@ The server does not need Moomoo OpenD. Keep OpenD on the computer that has your - AMH official installation docs say AMH 7.3 should be installed on a clean Debian, CentOS, or Ubuntu server and supports Nginx-based environments: https://amh.sh/install.htm - AMH official docs describe installing server/environment modules such as Nginx, LNMP/LNGX, and AMSSL from the panel: https://amh.sh/doc.htm - Nginx official reverse proxy docs use `proxy_pass` and `proxy_set_header` to forward application requests: https://docs.nginx.com/nginx/admin-guide/web-server/reverse-proxy/ -- Certbot official instructions recommend the snap-based Certbot install for Nginx on Ubuntu and note that port 80 HTTP should already work before issuing the certificate: https://certbot.eff.org/instructions?ws=nginx&os=ubuntufocal -- Docker official docs recommend installing Docker Engine from Docker's apt repository and using the Compose plugin on Linux: https://docs.docker.com/engine/install/ubuntu/ and https://docs.docker.com/compose/install/linux/ +- Certbot official instructions recommend the snap-based Certbot install for Nginx on Linux and note that port 80 HTTP should already work before issuing the certificate: https://certbot.eff.org/instructions?ws=nginx&os=debianbuster +- Docker official Debian docs recommend installing Docker Engine from Docker's apt repository using `/etc/apt/sources.list.d/docker.sources`, then installing the Compose plugin package: https://docs.docker.com/engine/install/debian/ I could open the shared ChatGPT URL, but the shared page did not expose the actual chat content without login in this environment. This guide is based on the repo and primary docs above. @@ -41,7 +41,7 @@ I could open the shared ChatGPT URL, but the shared page did not expose the actu You need: -- A VPS with a clean supported Linux image. Ubuntu 24.04 LTS is the most straightforward choice. +- A VPS with a clean supported Linux image. This guide is written for Debian 12/13 because the current server is Debian. - AMH installed with an Nginx-based environment such as LNGX or LNMP. - A domain or subdomain, for example `gammascope.example.com`. - DNS `A` record pointing that domain to the server public IP. @@ -71,26 +71,47 @@ AMH panel port, only from your IP Keep AMH's panel port restricted to your IP if the provider supports security group source IP rules. -## 2. Install Docker on the Server +## 2. Install Docker on the Debian Server -Follow Docker's official Ubuntu repository instructions. The short version for Ubuntu is: +Follow Docker's official Debian repository instructions. If a previous Ubuntu-style command created `/etc/apt/sources.list.d/docker.list`, remove it first because one malformed APT source blocks every `apt-get update`. ```bash -sudo apt-get update -sudo apt-get install -y ca-certificates curl -sudo install -m 0755 -d /etc/apt/keyrings -sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc -sudo chmod a+r /etc/apt/keyrings/docker.asc +set -eux -echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ - $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" \ - | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +rm -f /etc/apt/sources.list.d/docker.list +rm -f /etc/apt/sources.list.d/docker.sources -sudo apt-get update -sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -sudo docker run hello-world +apt-get update +apt-get install -y ca-certificates curl + +. /etc/os-release +if [ "$ID" != "debian" ]; then + echo "This block is for Debian only. Current OS: ID=$ID VERSION_CODENAME=$VERSION_CODENAME" + exit 1 +fi + +install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc +chmod a+r /etc/apt/keyrings/docker.asc + +cat > /etc/apt/sources.list.d/docker.sources < Date: Sun, 3 May 2026 18:50:50 -0700 Subject: [PATCH 04/11] docs: set AMH deployment domain --- docs/amh-nginx-server-setup.md | 28 +++++++++---------- .../gammascope.collector-client.env.example | 2 +- ops/amh-nginx/gammascope.nginx.conf | 10 +++---- .../gammascope.production.env.example | 2 +- 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/docs/amh-nginx-server-setup.md b/docs/amh-nginx-server-setup.md index 31d1b25..fbca57b 100644 --- a/docs/amh-nginx-server-setup.md +++ b/docs/amh-nginx-server-setup.md @@ -43,8 +43,7 @@ You need: - A VPS with a clean supported Linux image. This guide is written for Debian 12/13 because the current server is Debian. - AMH installed with an Nginx-based environment such as LNGX or LNMP. -- A domain or subdomain, for example `gammascope.example.com`. -- DNS `A` record pointing that domain to the server public IP. +- The domain `gamma.hiqjj.org`, with its DNS `A` record pointed at the server public IP. - Cloud firewall/security group opened for `80/tcp` and `443/tcp`. - SSH access to the server. - Docker Engine and Docker Compose plugin on the server. @@ -166,7 +165,7 @@ openssl rand -base64 48 | tr -d '\n' && echo Edit `ops/amh-nginx/gammascope.production.env`: ```text -GAMMASCOPE_PUBLIC_ORIGIN=https://gammascope.example.com +GAMMASCOPE_PUBLIC_ORIGIN=https://gamma.hiqjj.org GAMMASCOPE_POSTGRES_PASSWORD= GAMMASCOPE_ADMIN_TOKEN= GAMMASCOPE_WEB_ADMIN_PASSWORD= @@ -215,7 +214,7 @@ There are two workable paths. ### Option A: AMH Panel Vhost -Use AMH to create a site/vhost for `gammascope.example.com`, enable SSL with AMSSL, then add custom Nginx rules equivalent to these locations: +Use AMH to create a site/vhost for `gamma.hiqjj.org`, enable SSL with AMSSL, then add custom Nginx rules equivalent to these locations: ```nginx location = /api/spx/0dte/collector/events { @@ -271,11 +270,10 @@ Use this option when AMH owns certificate renewal and vhost generation. ### Option B: Full Nginx Template -Copy the repo template and edit the domain/certificate paths: +Copy the repo template. It already uses `gamma.hiqjj.org` and the matching Let's Encrypt certificate paths: ```bash sudo cp /opt/gammascope/ops/amh-nginx/gammascope.nginx.conf /etc/nginx/conf.d/gammascope.conf -sudo sed -i 's/gammascope.example.com/your-real-domain.example/g' /etc/nginx/conf.d/gammascope.conf ``` If your AMH Nginx is not using `/etc/nginx/conf.d`, locate its active config: @@ -307,7 +305,7 @@ sudo snap install core sudo snap refresh core sudo snap install --classic certbot sudo ln -sf /snap/bin/certbot /usr/local/bin/certbot -sudo certbot --nginx -d gammascope.example.com +sudo certbot --nginx -d gamma.hiqjj.org sudo certbot renew --dry-run ``` @@ -318,14 +316,14 @@ Certbot expects your domain to already resolve to the server and port `80` to be From your computer: ```bash -curl -I https://gammascope.example.com/ -curl -fsS https://gammascope.example.com/api/spx/0dte/replay/sessions | python3 -m json.tool +curl -I https://gamma.hiqjj.org/ +curl -fsS https://gamma.hiqjj.org/api/spx/0dte/replay/sessions | python3 -m json.tool ``` Collector ingestion should require the admin token: ```bash -curl -i -X POST https://gammascope.example.com/api/spx/0dte/collector/events/bulk \ +curl -i -X POST https://gamma.hiqjj.org/api/spx/0dte/collector/events/bulk \ -H 'Content-Type: application/json' \ --data '[]' ``` @@ -335,7 +333,7 @@ Expected without token: `403`. With the token: ```bash -curl -i -X POST https://gammascope.example.com/api/spx/0dte/collector/events/bulk \ +curl -i -X POST https://gamma.hiqjj.org/api/spx/0dte/collector/events/bulk \ -H "X-GammaScope-Admin-Token: " \ -H 'Content-Type: application/json' \ --data '[]' @@ -354,7 +352,7 @@ cp ops/amh-nginx/gammascope.collector-client.env.example ops/amh-nginx/gammascop Edit it: ```text -GAMMASCOPE_SERVER_API=https://gammascope.example.com +GAMMASCOPE_SERVER_API=https://gamma.hiqjj.org GAMMASCOPE_ADMIN_TOKEN= GAMMASCOPE_MOOMOO_HOST=127.0.0.1 GAMMASCOPE_MOOMOO_PORT=11111 @@ -414,14 +412,14 @@ docker compose --env-file ops/amh-nginx/gammascope.production.env -f ops/amh-ngi From your computer: ```bash -curl -fsS "https://gammascope.example.com/api/spx/0dte/heatmap/latest?metric=gex&symbol=SPX" | python3 -m json.tool +curl -fsS "https://gamma.hiqjj.org/api/spx/0dte/heatmap/latest?metric=gex&symbol=SPX" | python3 -m json.tool ``` Open: ```text -https://gammascope.example.com/ -https://gammascope.example.com/heatmap +https://gamma.hiqjj.org/ +https://gamma.hiqjj.org/heatmap ``` ## 11. Operating Commands diff --git a/ops/amh-nginx/gammascope.collector-client.env.example b/ops/amh-nginx/gammascope.collector-client.env.example index 2f7ca35..d0c2a4c 100644 --- a/ops/amh-nginx/gammascope.collector-client.env.example +++ b/ops/amh-nginx/gammascope.collector-client.env.example @@ -1,4 +1,4 @@ -GAMMASCOPE_SERVER_API=https://gammascope.example.com +GAMMASCOPE_SERVER_API=https://gamma.hiqjj.org GAMMASCOPE_ADMIN_TOKEN=replace-with-the-same-admin-token-used-on-the-server GAMMASCOPE_MOOMOO_HOST=127.0.0.1 GAMMASCOPE_MOOMOO_PORT=11111 diff --git a/ops/amh-nginx/gammascope.nginx.conf b/ops/amh-nginx/gammascope.nginx.conf index c998dde..da62fa5 100644 --- a/ops/amh-nginx/gammascope.nginx.conf +++ b/ops/amh-nginx/gammascope.nginx.conf @@ -1,5 +1,5 @@ # Full Nginx vhost template for GammaScope behind AMH. -# Replace gammascope.example.com and certificate paths before enabling. +# Certificate paths assume Certbot/Let's Encrypt for gamma.hiqjj.org. map $http_upgrade $connection_upgrade { default upgrade; @@ -19,7 +19,7 @@ upstream gammascope_api { server { listen 80; listen [::]:80; - server_name gammascope.example.com; + server_name gamma.hiqjj.org; location /.well-known/acme-challenge/ { root /home/wwwroot/gammascope; @@ -33,10 +33,10 @@ server { server { listen 443 ssl http2; listen [::]:443 ssl http2; - server_name gammascope.example.com; + server_name gamma.hiqjj.org; - ssl_certificate /etc/letsencrypt/live/gammascope.example.com/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/gammascope.example.com/privkey.pem; + ssl_certificate /etc/letsencrypt/live/gamma.hiqjj.org/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/gamma.hiqjj.org/privkey.pem; client_max_body_size 100m; diff --git a/ops/amh-nginx/gammascope.production.env.example b/ops/amh-nginx/gammascope.production.env.example index 10000e6..83d2f01 100644 --- a/ops/amh-nginx/gammascope.production.env.example +++ b/ops/amh-nginx/gammascope.production.env.example @@ -1,4 +1,4 @@ -GAMMASCOPE_PUBLIC_ORIGIN=https://gammascope.example.com +GAMMASCOPE_PUBLIC_ORIGIN=https://gamma.hiqjj.org GAMMASCOPE_POSTGRES_DB=gammascope GAMMASCOPE_POSTGRES_USER=gammascope From a5fa4633525f5c01d95b356c923cef987f910693 Mon Sep 17 00:00:00 2001 From: Sakura <32687351+zifanzhou1024@users.noreply.github.com> Date: Sun, 3 May 2026 19:13:59 -0700 Subject: [PATCH 05/11] docs: add AMH deployment runbook --- docs/amh-nginx-server-setup.md | 83 +++++-- ops/amh-nginx/README.md | 349 ++++++++++++++++++++++++++++ ops/amh-nginx/gammascope.nginx.conf | 21 ++ ops/amh-nginx/generate_secrets.py | 130 +++++++++++ tests/test_generate_secrets.py | 89 +++++++ 5 files changed, 654 insertions(+), 18 deletions(-) create mode 100644 ops/amh-nginx/README.md create mode 100755 ops/amh-nginx/generate_secrets.py create mode 100644 tests/test_generate_secrets.py diff --git a/docs/amh-nginx-server-setup.md b/docs/amh-nginx-server-setup.md index fbca57b..52245ec 100644 --- a/docs/amh-nginx-server-setup.md +++ b/docs/amh-nginx-server-setup.md @@ -25,6 +25,8 @@ The server does not need Moomoo OpenD. Keep OpenD on the computer that has your - `ops/amh-nginx/gammascope.nginx.conf`: full Nginx vhost template for AMH/manual Nginx. - `ops/amh-nginx/gammascope.production.env.example`: server environment template. - `ops/amh-nginx/gammascope.collector-client.env.example`: local collector environment template. +- `ops/amh-nginx/generate_secrets.py`: generates matching server and collector env files. +- `ops/amh-nginx/README.md`: condensed Debian deployment runbook. - `services/collector/gammascope_collector/publisher.py`: collector publishing now reads `GAMMASCOPE_ADMIN_TOKEN` and sends `X-GammaScope-Admin-Token`. ## Sources Checked @@ -35,7 +37,7 @@ The server does not need Moomoo OpenD. Keep OpenD on the computer that has your - Certbot official instructions recommend the snap-based Certbot install for Nginx on Linux and note that port 80 HTTP should already work before issuing the certificate: https://certbot.eff.org/instructions?ws=nginx&os=debianbuster - Docker official Debian docs recommend installing Docker Engine from Docker's apt repository using `/etc/apt/sources.list.d/docker.sources`, then installing the Compose plugin package: https://docs.docker.com/engine/install/debian/ -I could open the shared ChatGPT URL, but the shared page did not expose the actual chat content without login in this environment. This guide is based on the repo and primary docs above. +This guide also incorporates the live VPS setup notes from `gamma.hiqjj.org`: the server is Debian, Docker `hello-world` passed, the local Compose smoke tests passed, AMH/Nginx served the public app, and explicit `/_next/` proxying was needed to avoid static asset issues. ## Prerequisites @@ -147,32 +149,34 @@ rsync -az --delete \ ## 4. Configure Server Secrets -Create the production env file on the server: +Generate the production env file and a matching collector-client env file on the server: ```bash cd /opt/gammascope -cp ops/amh-nginx/gammascope.production.env.example ops/amh-nginx/gammascope.production.env +python3 ops/amh-nginx/generate_secrets.py \ + --server-output ops/amh-nginx/gammascope.production.env \ + --collector-output ops/amh-nginx/gammascope.collector-client.env ``` -Generate secrets: +Save the printed web admin password and collector admin token. The generated env files are ignored by git. -```bash -openssl rand -hex 24 -openssl rand -hex 32 -openssl rand -base64 48 | tr -d '\n' && echo -``` +`GAMMASCOPE_PUBLIC_ORIGIN` is compiled into the Next.js image. If you change the domain later, rebuild the web image. -Edit `ops/amh-nginx/gammascope.production.env`: +If you pasted generated secrets somewhere public, rotate them. On a fresh test install, stop and remove the database volume first because regenerating the env also changes `GAMMASCOPE_POSTGRES_PASSWORD`: -```text -GAMMASCOPE_PUBLIC_ORIGIN=https://gamma.hiqjj.org -GAMMASCOPE_POSTGRES_PASSWORD= -GAMMASCOPE_ADMIN_TOKEN= -GAMMASCOPE_WEB_ADMIN_PASSWORD= -GAMMASCOPE_WEB_ADMIN_SESSION_SECRET= +```bash +docker compose \ + --env-file ops/amh-nginx/gammascope.production.env \ + -f ops/amh-nginx/docker-compose.amh.yml \ + down -v + +python3 ops/amh-nginx/generate_secrets.py \ + --server-output ops/amh-nginx/gammascope.production.env \ + --collector-output ops/amh-nginx/gammascope.collector-client.env \ + --force ``` -`GAMMASCOPE_PUBLIC_ORIGIN` is compiled into the Next.js image. If you change the domain later, rebuild the web image. +Do not use `down -v` after you have real production data unless you have a database backup. For a live database, keep the existing Postgres password or rotate it manually inside Postgres before changing the env file. ## 5. Start Backend and Frontend Containers @@ -217,8 +221,38 @@ There are two workable paths. Use AMH to create a site/vhost for `gamma.hiqjj.org`, enable SSL with AMSSL, then add custom Nginx rules equivalent to these locations: ```nginx +location ^~ /_next/ { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 3600; + proxy_send_timeout 3600; +} + +location ^~ /images/ { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location = /favicon.ico { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + location = /api/spx/0dte/collector/events { proxy_pass http://127.0.0.1:8000; + proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -229,6 +263,7 @@ location = /api/spx/0dte/collector/events { location = /api/spx/0dte/collector/events/bulk { proxy_pass http://127.0.0.1:8000; + proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -266,6 +301,8 @@ location / { } ``` +Do not add a broad `/api/ -> 127.0.0.1:8000` rule in AMH. Next.js owns routes such as `/api/admin/login`, and broad API proxying will break the web admin flow. + Use this option when AMH owns certificate renewal and vhost generation. ### Option B: Full Nginx Template @@ -288,7 +325,7 @@ Validate and reload: ```bash sudo nginx -t -sudo nginx -s reload +sudo systemctl reload nginx || sudo systemctl restart nginx ``` ## 7. Configure HTTPS @@ -320,6 +357,16 @@ curl -I https://gamma.hiqjj.org/ curl -fsS https://gamma.hiqjj.org/api/spx/0dte/replay/sessions | python3 -m json.tool ``` +Verify that AMH is not intercepting Next.js assets: + +```bash +ASSET_PATH="$(curl -fsS https://gamma.hiqjj.org/ | grep -oE '/_next/[^"]+' | head -1)" +echo "$ASSET_PATH" +curl -I "https://gamma.hiqjj.org$ASSET_PATH" +``` + +Expected: `HTTP/2 200` and a Next static asset content type such as CSS or JavaScript. + Collector ingestion should require the admin token: ```bash diff --git a/ops/amh-nginx/README.md b/ops/amh-nginx/README.md new file mode 100644 index 0000000..902a633 --- /dev/null +++ b/ops/amh-nginx/README.md @@ -0,0 +1,349 @@ +# GammaScope AMH/Nginx Deployment + +This folder contains the server-side deployment assets for `gamma.hiqjj.org`. + +The target layout is: + +- AMH/Nginx is the public HTTPS entrypoint on ports `80` and `443`. +- Docker Compose runs Postgres, Redis, FastAPI, and Next.js on the Debian server. +- FastAPI is bound only to `127.0.0.1:8000`. +- Next.js is bound only to `127.0.0.1:3000`. +- Your own computer runs Moomoo OpenD and the GammaScope collector, then publishes data to `https://gamma.hiqjj.org`. + +## Files + +- `docker-compose.amh.yml`: production Compose stack. +- `gammascope.nginx.conf`: full Nginx vhost template for `gamma.hiqjj.org`. +- `gammascope.production.env.example`: server env template. +- `gammascope.collector-client.env.example`: local collector env template. +- `generate_secrets.py`: generates matching server and collector env files. + +## 1. Debian Docker Setup + +Run this on the VPS as `root`. It removes the broken old `docker.list` file if one exists and installs Docker using Docker's current Debian `.sources` repository format. + +```bash +set -eux + +rm -f /etc/apt/sources.list.d/docker.list +rm -f /etc/apt/sources.list.d/docker.sources + +apt-get update +apt-get install -y ca-certificates curl git openssl python3 + +. /etc/os-release +test "$ID" = "debian" + +install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc +chmod a+r /etc/apt/keyrings/docker.asc + +cat > /etc/apt/sources.list.d/docker.sources < http://127.0.0.1:3000 +/images/ -> http://127.0.0.1:3000 +/favicon.ico -> http://127.0.0.1:3000 +/ -> http://127.0.0.1:3000 +/ws/ -> http://127.0.0.1:8000 +/api/spx/0dte/collector/events -> http://127.0.0.1:8000 +/api/spx/0dte/collector/events/bulk -> http://127.0.0.1:8000 +``` + +Do not add a broad `/api/ -> 127.0.0.1:8000` rule. Next.js owns routes such as `/api/admin/login`, and broad API proxying will break the web admin flow. + +For AMH URL rules, paste location blocks, not a full `server { ... }` block: + +```nginx +location ^~ /_next/ { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 3600; + proxy_send_timeout 3600; +} + +location ^~ /images/ { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location = /favicon.ico { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location = /api/spx/0dte/collector/events { + proxy_pass http://127.0.0.1:8000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location = /api/spx/0dte/collector/events/bulk { + proxy_pass http://127.0.0.1:8000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location ^~ /ws/ { + proxy_pass http://127.0.0.1:8000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 3600; + proxy_send_timeout 3600; + proxy_buffering off; +} + +location / { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 3600; + proxy_send_timeout 3600; + proxy_buffering off; +} +``` + +If you use the full template directly, copy it after the TLS certificate exists: + +```bash +cp /opt/gammascope/ops/amh-nginx/gammascope.nginx.conf /etc/nginx/conf.d/gammascope.conf +nginx -t +systemctl reload nginx || systemctl restart nginx +``` + +The template expects certificates at: + +```text +/etc/letsencrypt/live/gamma.hiqjj.org/fullchain.pem +/etc/letsencrypt/live/gamma.hiqjj.org/privkey.pem +``` + +If AMH manages SSL elsewhere, update the two `ssl_certificate` paths in the copied Nginx config. + +## 6. Public Smoke Checks + +Run from your computer: + +```bash +curl -I https://gamma.hiqjj.org/ +curl -fsS https://gamma.hiqjj.org/api/spx/0dte/replay/sessions | python3 -m json.tool +``` + +Verify that AMH is not intercepting Next.js assets: + +```bash +ASSET_PATH="$(curl -fsS https://gamma.hiqjj.org/ | grep -oE '/_next/[^"]+' | head -1)" +echo "$ASSET_PATH" +curl -I "https://gamma.hiqjj.org$ASSET_PATH" +``` + +Expected: `HTTP/2 200` and a Next static asset content type such as CSS or JavaScript. + +Collector ingestion should reject missing tokens: + +```bash +curl -i -X POST https://gamma.hiqjj.org/api/spx/0dte/collector/events/bulk \ + -H 'Content-Type: application/json' \ + --data '[]' +``` + +Expected: `403`. + +Collector ingestion should accept the generated token: + +```bash +ADMIN_TOKEN="" + +curl -i -X POST https://gamma.hiqjj.org/api/spx/0dte/collector/events/bulk \ + -H "X-GammaScope-Admin-Token: $ADMIN_TOKEN" \ + -H 'Content-Type: application/json' \ + --data '[]' +``` + +Expected: `200`. + +## 7. Configure Your Local Collector + +Copy the generated collector env from the server to your local repo checkout: + +```bash +scp root@gamma.hiqjj.org:/opt/gammascope/ops/amh-nginx/gammascope.collector-client.env \ + /Users/sakura/WebstormProjects/gamma-scope/.worktrees/amh-nginx-server-setup/ops/amh-nginx/gammascope.collector-client.env +``` + +Start Moomoo OpenD locally, then run from your computer: + +```bash +cd /Users/sakura/WebstormProjects/gamma-scope/.worktrees/amh-nginx-server-setup + +set -a +. ops/amh-nginx/gammascope.collector-client.env +set +a + +pnpm collector:moomoo-snapshot -- \ + --host "$GAMMASCOPE_MOOMOO_HOST" \ + --port "$GAMMASCOPE_MOOMOO_PORT" \ + --api "$GAMMASCOPE_SERVER_API" \ + --spot RUT="$GAMMASCOPE_RUT_SPOT" \ + --spot NDX="$GAMMASCOPE_NDX_SPOT" \ + --publish +``` + +Open: + +```text +https://gamma.hiqjj.org/ +https://gamma.hiqjj.org/heatmap +``` + +## 8. Operations + +Update and rebuild: + +```bash +cd /opt/gammascope +git pull +docker compose --env-file ops/amh-nginx/gammascope.production.env -f ops/amh-nginx/docker-compose.amh.yml up -d --build +``` + +View logs: + +```bash +docker compose --env-file ops/amh-nginx/gammascope.production.env -f ops/amh-nginx/docker-compose.amh.yml logs -f api +docker compose --env-file ops/amh-nginx/gammascope.production.env -f ops/amh-nginx/docker-compose.amh.yml logs -f web +``` + +Backup Postgres: + +```bash +docker compose --env-file ops/amh-nginx/gammascope.production.env -f ops/amh-nginx/docker-compose.amh.yml exec postgres \ + pg_dump -U gammascope gammascope > "gammascope-$(date +%Y%m%d-%H%M%S).sql" +``` diff --git a/ops/amh-nginx/gammascope.nginx.conf b/ops/amh-nginx/gammascope.nginx.conf index da62fa5..9a2f4df 100644 --- a/ops/amh-nginx/gammascope.nginx.conf +++ b/ops/amh-nginx/gammascope.nginx.conf @@ -46,6 +46,27 @@ server { proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Host $host; + location ^~ /_next/ { + proxy_pass http://gammascope_web; + proxy_http_version 1.1; + proxy_read_timeout 60s; + proxy_send_timeout 60s; + } + + location ^~ /images/ { + proxy_pass http://gammascope_web; + proxy_http_version 1.1; + proxy_read_timeout 60s; + proxy_send_timeout 60s; + } + + location = /favicon.ico { + proxy_pass http://gammascope_web; + proxy_http_version 1.1; + proxy_read_timeout 60s; + proxy_send_timeout 60s; + } + location = /api/spx/0dte/collector/events { proxy_pass http://gammascope_api; proxy_read_timeout 30s; diff --git a/ops/amh-nginx/generate_secrets.py b/ops/amh-nginx/generate_secrets.py new file mode 100755 index 0000000..6ac612d --- /dev/null +++ b/ops/amh-nginx/generate_secrets.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import secrets +import sys +from dataclasses import dataclass +from pathlib import Path + + +DEFAULT_DOMAIN = "gamma.hiqjj.org" + + +@dataclass(frozen=True) +class SecretValues: + postgres_password: str + admin_token: str + web_admin_password: str + web_admin_session_secret: str + + +def generate_secret_values() -> SecretValues: + return SecretValues( + postgres_password=secrets.token_hex(24), + admin_token=secrets.token_hex(32), + web_admin_password=secrets.token_hex(24), + web_admin_session_secret=secrets.token_urlsafe(48), + ) + + +def render_server_env(values: SecretValues, *, domain: str = DEFAULT_DOMAIN) -> str: + origin = _https_origin(domain) + return "\n".join( + [ + f"GAMMASCOPE_PUBLIC_ORIGIN={origin}", + "", + "GAMMASCOPE_POSTGRES_DB=gammascope", + "GAMMASCOPE_POSTGRES_USER=gammascope", + f"GAMMASCOPE_POSTGRES_PASSWORD={values.postgres_password}", + "", + "GAMMASCOPE_PRIVATE_MODE_ENABLED=true", + f"GAMMASCOPE_ADMIN_TOKEN={values.admin_token}", + "", + "GAMMASCOPE_WEB_ADMIN_USERNAME=admin", + f"GAMMASCOPE_WEB_ADMIN_PASSWORD={values.web_admin_password}", + f"GAMMASCOPE_WEB_ADMIN_SESSION_SECRET={values.web_admin_session_secret}", + "", + "GAMMASCOPE_API_HOST_PORT=8000", + "GAMMASCOPE_WEB_HOST_PORT=3000", + "GAMMASCOPE_REPLAY_CAPTURE_INTERVAL_SECONDS=5", + "GAMMASCOPE_REPLAY_RETENTION_DAYS=20", + "GAMMASCOPE_SAVED_VIEW_RETENTION_DAYS=90", + "GAMMASCOPE_REPLAY_IMPORT_MAX_BYTES=104857600", + "", + ] + ) + + +def render_collector_env(values: SecretValues, *, domain: str = DEFAULT_DOMAIN) -> str: + origin = _https_origin(domain) + return "\n".join( + [ + f"GAMMASCOPE_SERVER_API={origin}", + f"GAMMASCOPE_ADMIN_TOKEN={values.admin_token}", + "GAMMASCOPE_MOOMOO_HOST=127.0.0.1", + "GAMMASCOPE_MOOMOO_PORT=11111", + "GAMMASCOPE_RUT_SPOT=2050", + "GAMMASCOPE_NDX_SPOT=18300", + "", + ] + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Generate GammaScope AMH deployment secrets.") + parser.add_argument("--domain", default=DEFAULT_DOMAIN, help=f"Public HTTPS domain. Default: {DEFAULT_DOMAIN}") + parser.add_argument("--server-output", type=Path, help="Path to write gammascope.production.env.") + parser.add_argument("--collector-output", type=Path, help="Path to write gammascope.collector-client.env.") + parser.add_argument("--force", action="store_true", help="Overwrite existing output files.") + parser.add_argument("--print", dest="print_env", action="store_true", help="Print generated env files to stdout.") + args = parser.parse_args(argv) + + values = generate_secret_values() + server_env = render_server_env(values, domain=args.domain) + collector_env = render_collector_env(values, domain=args.domain) + + try: + if args.server_output is not None: + _write_output(args.server_output, server_env, force=args.force) + print(f"wrote server env: {args.server_output}") + if args.collector_output is not None: + _write_output(args.collector_output, collector_env, force=args.force) + print(f"wrote collector env: {args.collector_output}") + except FileExistsError as exc: + print(str(exc), file=sys.stderr) + return 2 + + if args.print_env or (args.server_output is None and args.collector_output is None): + print("# gammascope.production.env") + print(server_env, end="") + print("# gammascope.collector-client.env") + print(collector_env, end="") + + if args.server_output is not None or args.collector_output is not None: + print("web admin username: admin") + print(f"web admin password: {values.web_admin_password}") + print(f"collector admin token: {values.admin_token}") + + return 0 + + +def _https_origin(domain: str) -> str: + normalized = domain.strip().rstrip("/") + if not normalized: + raise ValueError("domain must be non-empty") + if normalized.startswith(("http://", "https://")): + return normalized + return f"https://{normalized}" + + +def _write_output(path: Path, content: str, *, force: bool) -> None: + if path.exists() and not force: + raise FileExistsError(f"{path} already exists; pass --force to overwrite it") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + path.chmod(0o600) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_generate_secrets.py b/tests/test_generate_secrets.py new file mode 100644 index 0000000..b1ed549 --- /dev/null +++ b/tests/test_generate_secrets.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import importlib.util +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "ops" / "amh-nginx" / "generate_secrets.py" + + +def test_render_env_files_share_domain_and_admin_token() -> None: + module = _load_generate_secrets_module() + values = module.SecretValues( + postgres_password="postgres-secret", + admin_token="admin-secret", + web_admin_password="web-secret", + web_admin_session_secret="session-secret", + ) + + server_env = module.render_server_env(values, domain="gamma.hiqjj.org") + collector_env = module.render_collector_env(values, domain="gamma.hiqjj.org") + + assert "GAMMASCOPE_PUBLIC_ORIGIN=https://gamma.hiqjj.org" in server_env + assert "GAMMASCOPE_SERVER_API=https://gamma.hiqjj.org" in collector_env + assert "GAMMASCOPE_ADMIN_TOKEN=admin-secret" in server_env + assert "GAMMASCOPE_ADMIN_TOKEN=admin-secret" in collector_env + assert "GAMMASCOPE_WEB_ADMIN_PASSWORD=web-secret" in server_env + assert "GAMMASCOPE_WEB_ADMIN_SESSION_SECRET=session-secret" in server_env + + +def test_cli_writes_env_files_and_refuses_overwrite(tmp_path: Path) -> None: + server_env = tmp_path / "gammascope.production.env" + collector_env = tmp_path / "gammascope.collector-client.env" + + result = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--server-output", + str(server_env), + "--collector-output", + str(collector_env), + ], + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert "wrote server env" in result.stdout + assert "wrote collector env" in result.stdout + + server_text = server_env.read_text(encoding="utf-8") + collector_text = collector_env.read_text(encoding="utf-8") + admin_token = _env_value(server_text, "GAMMASCOPE_ADMIN_TOKEN") + + assert _env_value(server_text, "GAMMASCOPE_PUBLIC_ORIGIN") == "https://gamma.hiqjj.org" + assert _env_value(collector_text, "GAMMASCOPE_SERVER_API") == "https://gamma.hiqjj.org" + assert _env_value(collector_text, "GAMMASCOPE_ADMIN_TOKEN") == admin_token + + overwrite_result = subprocess.run( + [sys.executable, str(SCRIPT), "--server-output", str(server_env)], + check=False, + capture_output=True, + text=True, + ) + + assert overwrite_result.returncode == 2 + assert "already exists" in overwrite_result.stderr + + +def _load_generate_secrets_module(): + spec = importlib.util.spec_from_file_location("generate_secrets", SCRIPT) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _env_value(env_text: str, name: str) -> str: + prefix = f"{name}=" + for line in env_text.splitlines(): + if line.startswith(prefix): + return line.removeprefix(prefix) + raise AssertionError(f"{name} not found") From 08e831d4b25b31007a723abff25c1784b69a880d Mon Sep 17 00:00:00 2001 From: Sakura <32687351+zifanzhou1024@users.noreply.github.com> Date: Sun, 3 May 2026 20:24:07 -0700 Subject: [PATCH 06/11] fix: use macOS CA bundle for collector publishes --- .../gammascope_collector/publisher.py | 29 +++++++++++- services/collector/tests/test_publisher.py | 46 +++++++++++++++++++ 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/services/collector/gammascope_collector/publisher.py b/services/collector/gammascope_collector/publisher.py index d82a05e..a4e2a51 100644 --- a/services/collector/gammascope_collector/publisher.py +++ b/services/collector/gammascope_collector/publisher.py @@ -3,9 +3,11 @@ import argparse import json import os +import ssl import sys from collections.abc import Callable, Iterable, Sequence from dataclasses import asdict, dataclass +from pathlib import Path from typing import Any from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen @@ -16,6 +18,8 @@ COLLECTOR_EVENTS_BULK_PATH = "/api/spx/0dte/collector/events/bulk" ADMIN_TOKEN_ENV = "GAMMASCOPE_ADMIN_TOKEN" ADMIN_TOKEN_HEADER = "X-GammaScope-Admin-Token" +SSL_CERT_FILE_ENV = "SSL_CERT_FILE" +MACOS_SYSTEM_CERT_FILE = Path("/etc/ssl/cert.pem") PostJson = Callable[[str, dict[str, object]], dict[str, Any]] PostJsonBatch = Callable[[str, list[dict[str, object]]], dict[str, Any]] @@ -109,7 +113,7 @@ def _post_json(endpoint: str, event: dict[str, object], *, admin_token: str | No method="POST", ) try: - with urlopen(request, timeout=5) as response: + with _urlopen_request(request, timeout=5) as response: return json.loads(response.read().decode("utf-8")) except HTTPError as exc: detail = exc.read().decode("utf-8", errors="replace") @@ -132,7 +136,7 @@ def _post_json_batch( method="POST", ) try: - with urlopen(request, timeout=10) as response: + with _urlopen_request(request, timeout=10) as response: return json.loads(response.read().decode("utf-8")) except HTTPError as exc: detail = exc.read().decode("utf-8", errors="replace") @@ -161,6 +165,27 @@ def _json_headers(admin_token: str | None = None) -> dict[str, str]: return headers +def _urlopen_request(request: Request, *, timeout: float): + ssl_context = _ssl_context_for_request(request) + if ssl_context is None: + return urlopen(request, timeout=timeout) + return urlopen(request, timeout=timeout, context=ssl_context) + + +def _ssl_context_for_request(request: Request) -> ssl.SSLContext | None: + if not request.full_url.lower().startswith("https://"): + return None + if os.environ.get(SSL_CERT_FILE_ENV): + return None + + default_cafile = ssl.get_default_verify_paths().cafile + if default_cafile and Path(default_cafile).exists(): + return None + if MACOS_SYSTEM_CERT_FILE.exists(): + return ssl.create_default_context(cafile=str(MACOS_SYSTEM_CERT_FILE)) + return None + + def _normalize_argv(argv: Sequence[str] | None) -> Sequence[str] | None: if argv and argv[0] == "--": return argv[1:] diff --git a/services/collector/tests/test_publisher.py b/services/collector/tests/test_publisher.py index 34d7c7d..ec251dc 100644 --- a/services/collector/tests/test_publisher.py +++ b/services/collector/tests/test_publisher.py @@ -1,5 +1,7 @@ from datetime import UTC, datetime import json +from pathlib import Path +from types import SimpleNamespace import pytest @@ -89,6 +91,50 @@ def fake_urlopen(request: publisher.Request, timeout: float) -> FakeResponse: assert captured_requests[0].get_header("X-gammascope-admin-token") == "server-admin-token" +def test_post_json_uses_macos_system_ca_when_python_ca_path_is_missing( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + cert_file = tmp_path / "cert.pem" + cert_file.write_text("test cert", encoding="utf-8") + expected_context = object() + captured_contexts: list[object | None] = [] + + class FakeResponse: + def __enter__(self) -> object: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def read(self) -> bytes: + return b'{"accepted": true, "event_type": "CollectorHealth"}' + + def fake_urlopen(request: publisher.Request, *, timeout: float, context: object | None = None) -> FakeResponse: + captured_contexts.append(context) + assert request.full_url == "https://testserver/collector" + assert timeout == 5 + return FakeResponse() + + monkeypatch.delenv("SSL_CERT_FILE", raising=False) + monkeypatch.setattr(publisher, "MACOS_SYSTEM_CERT_FILE", cert_file) + monkeypatch.setattr( + publisher.ssl, + "get_default_verify_paths", + lambda: SimpleNamespace(cafile="/missing/python/cert.pem"), + ) + monkeypatch.setattr( + publisher.ssl, + "create_default_context", + lambda *, cafile: expected_context, + ) + monkeypatch.setattr(publisher, "urlopen", fake_urlopen) + + publisher._post_json("https://testserver/collector", {"ok": True}, admin_token="server-admin-token") + + assert captured_contexts == [expected_context] + + def test_publish_events_posts_each_event_to_ingestion_endpoint() -> None: events = [ health_event( From 8777157042e49dfd0018fcaa520245b3be831cbb Mon Sep 17 00:00:00 2001 From: Sakura <32687351+zifanzhou1024@users.noreply.github.com> Date: Sun, 3 May 2026 20:36:36 -0700 Subject: [PATCH 07/11] fix: route realtime browser updates through web --- .../app/api/spx/0dte/snapshot/latest/route.ts | 51 +++++++- apps/web/lib/snapshotStream.ts | 11 +- apps/web/tests/snapshotRoute.test.ts | 120 ++++++++++++++---- apps/web/tests/snapshotStream.test.ts | 13 ++ docs/amh-nginx-server-setup.md | 83 +++++++++++- ops/amh-nginx/README.md | 92 +++++++++++++- ops/amh-nginx/gammascope.nginx.conf | 63 +++++++++ 7 files changed, 397 insertions(+), 36 deletions(-) diff --git a/apps/web/app/api/spx/0dte/snapshot/latest/route.ts b/apps/web/app/api/spx/0dte/snapshot/latest/route.ts index 0d596ba..038c4c5 100644 --- a/apps/web/app/api/spx/0dte/snapshot/latest/route.ts +++ b/apps/web/app/api/spx/0dte/snapshot/latest/route.ts @@ -1,10 +1,51 @@ import { NextResponse } from "next/server"; -import { loadDashboardSnapshot } from "../../../../../../lib/serverSnapshotSource"; +import { verifyAdminRequest } from "../../../../../../lib/adminSession"; -export async function GET(request: Request) { - const response = NextResponse.json(await loadDashboardSnapshot({ - requestHeaders: request.headers - })); +const DEFAULT_API_BASE_URL = "http://127.0.0.1:8000"; +const SNAPSHOT_PATH = "/api/spx/0dte/snapshot/latest"; +const ADMIN_TOKEN_HEADER = "X-GammaScope-Admin-Token"; + +function snapshotUrl(apiBaseUrl: string): string { + return `${apiBaseUrl.replace(/\/+$/, "")}${SNAPSHOT_PATH}`; +} + +function noStoreJson(payload: unknown, init?: ResponseInit) { + const response = NextResponse.json(payload, init); response.headers.set("Cache-Control", "no-store"); return response; } + +function upstreamHeaders(request: Request): HeadersInit { + const headers: Record = { + Accept: "application/json" + }; + const adminToken = process.env.GAMMASCOPE_ADMIN_TOKEN?.trim(); + + if (adminToken && verifyAdminRequest(request, { csrf: false }).ok) { + headers[ADMIN_TOKEN_HEADER] = adminToken; + } + + return headers; +} + +export async function GET(request: Request): Promise { + const apiBaseUrl = process.env.GAMMASCOPE_API_BASE_URL ?? DEFAULT_API_BASE_URL; + + try { + const upstreamResponse = await fetch(snapshotUrl(apiBaseUrl), { + cache: "no-store", + headers: upstreamHeaders(request) + }); + + const response = new Response(await upstreamResponse.text(), { + status: upstreamResponse.status, + headers: { + "Content-Type": upstreamResponse.headers.get("Content-Type") ?? "application/json" + } + }); + response.headers.set("Cache-Control", "no-store"); + return response; + } catch { + return noStoreJson({ error: "Snapshot API unavailable" }, { status: 502 }); + } +} diff --git a/apps/web/lib/snapshotStream.ts b/apps/web/lib/snapshotStream.ts index 1bf104c..86ba144 100644 --- a/apps/web/lib/snapshotStream.ts +++ b/apps/web/lib/snapshotStream.ts @@ -30,7 +30,16 @@ export function snapshotWebSocketUrl(apiBaseUrl = DEFAULT_API_BASE_URL): string } export function clientSnapshotWebSocketUrl(): string { - return process.env.NEXT_PUBLIC_GAMMASCOPE_WS_URL || snapshotWebSocketUrl(); + const configuredUrl = process.env.NEXT_PUBLIC_GAMMASCOPE_WS_URL; + if (!configuredUrl) { + return snapshotWebSocketUrl(); + } + + const protocol = new URL(configuredUrl).protocol; + if (protocol === "ws:" || protocol === "wss:") { + return configuredUrl; + } + return snapshotWebSocketUrl(configuredUrl); } export function startSnapshotStream({ diff --git a/apps/web/tests/snapshotRoute.test.ts b/apps/web/tests/snapshotRoute.test.ts index 60a90c4..4a81c24 100644 --- a/apps/web/tests/snapshotRoute.test.ts +++ b/apps/web/tests/snapshotRoute.test.ts @@ -1,46 +1,110 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import type { AnalyticsSnapshot } from "../lib/contracts"; -import { seedSnapshot } from "../lib/seedSnapshot"; -const loadDashboardSnapshot = vi.fn(); +const ADMIN_ENV = { + GAMMASCOPE_WEB_ADMIN_USERNAME: "admin", + GAMMASCOPE_WEB_ADMIN_PASSWORD: "correct-horse-battery-staple", + GAMMASCOPE_WEB_ADMIN_SESSION_SECRET: "test-session-secret-with-enough-entropy", + GAMMASCOPE_ADMIN_TOKEN: "upstream-admin-token" +} as const; -vi.mock("../lib/serverSnapshotSource", () => ({ - loadDashboardSnapshot -})); +function textResponse(body: string, init: ResponseInit = {}): Response { + return new Response(body, { + status: init.status ?? 200, + headers: { + "Content-Type": "application/json", + ...init.headers + } + }); +} + +function setAdminEnv() { + vi.stubEnv("GAMMASCOPE_WEB_ADMIN_USERNAME", ADMIN_ENV.GAMMASCOPE_WEB_ADMIN_USERNAME); + vi.stubEnv("GAMMASCOPE_WEB_ADMIN_PASSWORD", ADMIN_ENV.GAMMASCOPE_WEB_ADMIN_PASSWORD); + vi.stubEnv("GAMMASCOPE_WEB_ADMIN_SESSION_SECRET", ADMIN_ENV.GAMMASCOPE_WEB_ADMIN_SESSION_SECRET); + vi.stubEnv("GAMMASCOPE_ADMIN_TOKEN", ADMIN_ENV.GAMMASCOPE_ADMIN_TOKEN); +} describe("GET /api/spx/0dte/snapshot/latest", () => { afterEach(() => { - loadDashboardSnapshot.mockReset(); + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); vi.resetModules(); }); - it("returns the latest dashboard snapshot without caching", async () => { - const snapshot = { - ...seedSnapshot, - session_id: "route-test-session", - mode: "live", - spot: 5212.75, - rows: [ - { - ...seedSnapshot.rows[0]!, - gamma_diff: 0 - } - ] - } satisfies AnalyticsSnapshot; - loadDashboardSnapshot.mockResolvedValue(snapshot); + it("proxies to FastAPI and preserves upstream body, status, and content type", async () => { + const fetcher = vi.fn(async () => textResponse(JSON.stringify({ ok: true }), { + status: 202, + headers: { + "Content-Type": "application/vnd.gammascope.snapshot+json" + } + })); + vi.stubGlobal("fetch", fetcher); + vi.stubEnv("GAMMASCOPE_API_BASE_URL", "http://fastapi.test/"); const { GET } = await import("../app/api/spx/0dte/snapshot/latest/route"); - const request = new Request("http://localhost/api/spx/0dte/snapshot/latest", { + const response = await GET(new Request("http://localhost/api/spx/0dte/snapshot/latest")); + + expect(response.status).toBe(202); + await expect(response.text()).resolves.toBe(JSON.stringify({ ok: true })); + expect(response.headers.get("Content-Type")).toBe("application/vnd.gammascope.snapshot+json"); + expect(response.headers.get("Cache-Control")).toBe("no-store"); + expect(fetcher).toHaveBeenCalledWith("http://fastapi.test/api/spx/0dte/snapshot/latest", { + cache: "no-store", headers: { - cookie: "gammascope_admin=signed-session" + Accept: "application/json" } }); - const response = await GET(request); + }); - await expect(response.json()).resolves.toEqual(snapshot); - expect(response.headers.get("Cache-Control")).toBe("no-store"); - expect(loadDashboardSnapshot).toHaveBeenCalledWith({ - requestHeaders: request.headers + it("forwards the upstream admin token when the web admin session is valid", async () => { + setAdminEnv(); + const { ADMIN_COOKIE_NAME, createAdminSessionValue } = await import("../lib/adminSession"); + const sessionValue = createAdminSessionValue(); + const fetcher = vi.fn(async () => textResponse("{}")); + vi.stubGlobal("fetch", fetcher); + + const { GET } = await import("../app/api/spx/0dte/snapshot/latest/route"); + await GET(new Request("http://localhost/api/spx/0dte/snapshot/latest", { + headers: { + Cookie: `${ADMIN_COOKIE_NAME}=${encodeURIComponent(sessionValue)}` + } + })); + + expect(fetcher).toHaveBeenCalledWith("http://127.0.0.1:8000/api/spx/0dte/snapshot/latest", { + cache: "no-store", + headers: { + Accept: "application/json", + "X-GammaScope-Admin-Token": ADMIN_ENV.GAMMASCOPE_ADMIN_TOKEN + } }); }); + + it("does not forward the upstream admin token when the request is unauthenticated", async () => { + setAdminEnv(); + const fetcher = vi.fn(async () => textResponse("{}")); + vi.stubGlobal("fetch", fetcher); + + const { GET } = await import("../app/api/spx/0dte/snapshot/latest/route"); + await GET(new Request("http://localhost/api/spx/0dte/snapshot/latest")); + + expect(fetcher).toHaveBeenCalledWith("http://127.0.0.1:8000/api/spx/0dte/snapshot/latest", { + cache: "no-store", + headers: { + Accept: "application/json" + } + }); + }); + + it("returns no-store 502 JSON when the upstream fetch fails", async () => { + vi.stubGlobal("fetch", vi.fn(async () => { + throw new Error("offline"); + })); + + const { GET } = await import("../app/api/spx/0dte/snapshot/latest/route"); + const response = await GET(new Request("http://localhost/api/spx/0dte/snapshot/latest")); + + expect(response.status).toBe(502); + await expect(response.json()).resolves.toEqual({ error: "Snapshot API unavailable" }); + expect(response.headers.get("Cache-Control")).toBe("no-store"); + }); }); diff --git a/apps/web/tests/snapshotStream.test.ts b/apps/web/tests/snapshotStream.test.ts index a119fcc..e7058ea 100644 --- a/apps/web/tests/snapshotStream.test.ts +++ b/apps/web/tests/snapshotStream.test.ts @@ -80,6 +80,19 @@ describe("clientSnapshotWebSocketUrl", () => { process.env.NEXT_PUBLIC_GAMMASCOPE_WS_URL = original; } }); + + it("converts a public HTTPS origin env value to the snapshot websocket URL", () => { + const original = process.env.NEXT_PUBLIC_GAMMASCOPE_WS_URL; + process.env.NEXT_PUBLIC_GAMMASCOPE_WS_URL = "https://gamma.hiqjj.org"; + + expect(clientSnapshotWebSocketUrl()).toBe("wss://gamma.hiqjj.org/ws/spx/0dte"); + + if (original === undefined) { + delete process.env.NEXT_PUBLIC_GAMMASCOPE_WS_URL; + } else { + process.env.NEXT_PUBLIC_GAMMASCOPE_WS_URL = original; + } + }); }); describe("startSnapshotStream", () => { diff --git a/docs/amh-nginx-server-setup.md b/docs/amh-nginx-server-setup.md index 52245ec..890bc39 100644 --- a/docs/amh-nginx-server-setup.md +++ b/docs/amh-nginx-server-setup.md @@ -250,6 +250,87 @@ location = /favicon.ico { proxy_set_header X-Forwarded-Proto $scheme; } +location ^~ /api/admin/ { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location ^~ /api/replay/imports { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location = /api/views { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location = /api/spx/0dte/snapshot/latest { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location = /api/spx/0dte/status { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location = /api/spx/0dte/heatmap/latest { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location ^~ /api/spx/0dte/experimental/ { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location ^~ /api/spx/0dte/replay/ { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location = /api/spx/0dte/scenario { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + location = /api/spx/0dte/collector/events { proxy_pass http://127.0.0.1:8000; proxy_http_version 1.1; @@ -301,7 +382,7 @@ location / { } ``` -Do not add a broad `/api/ -> 127.0.0.1:8000` rule in AMH. Next.js owns routes such as `/api/admin/login`, and broad API proxying will break the web admin flow. +Do not add a broad `/api/ -> 127.0.0.1:8000` rule in AMH. Next.js owns routes such as `/api/admin/login` and the authenticated realtime proxy at `/api/spx/0dte/snapshot/latest`; broad API proxying will make private-mode browser pages keep seeing seed data. Use this option when AMH owns certificate renewal and vhost generation. diff --git a/ops/amh-nginx/README.md b/ops/amh-nginx/README.md index 902a633..96fce8c 100644 --- a/ops/amh-nginx/README.md +++ b/ops/amh-nginx/README.md @@ -146,13 +146,22 @@ Set the reverse proxy rules to: /_next/ -> http://127.0.0.1:3000 /images/ -> http://127.0.0.1:3000 /favicon.ico -> http://127.0.0.1:3000 +/api/admin/ -> http://127.0.0.1:3000 +/api/replay/imports -> http://127.0.0.1:3000 +/api/views -> http://127.0.0.1:3000 +/api/spx/0dte/snapshot/latest -> http://127.0.0.1:3000 +/api/spx/0dte/status -> http://127.0.0.1:3000 +/api/spx/0dte/heatmap/latest -> http://127.0.0.1:3000 +/api/spx/0dte/experimental/ -> http://127.0.0.1:3000 +/api/spx/0dte/replay/ -> http://127.0.0.1:3000 +/api/spx/0dte/scenario -> http://127.0.0.1:3000 / -> http://127.0.0.1:3000 /ws/ -> http://127.0.0.1:8000 /api/spx/0dte/collector/events -> http://127.0.0.1:8000 /api/spx/0dte/collector/events/bulk -> http://127.0.0.1:8000 ``` -Do not add a broad `/api/ -> 127.0.0.1:8000` rule. Next.js owns routes such as `/api/admin/login`, and broad API proxying will break the web admin flow. +Do not add a broad `/api/ -> 127.0.0.1:8000` rule. Next.js owns routes such as `/api/admin/login` and the authenticated realtime proxy at `/api/spx/0dte/snapshot/latest`; broad API proxying will make private-mode browser pages keep seeing seed data. For AMH URL rules, paste location blocks, not a full `server { ... }` block: @@ -186,6 +195,87 @@ location = /favicon.ico { proxy_set_header X-Forwarded-Proto $scheme; } +location ^~ /api/admin/ { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location ^~ /api/replay/imports { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location = /api/views { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location = /api/spx/0dte/snapshot/latest { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location = /api/spx/0dte/status { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location = /api/spx/0dte/heatmap/latest { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location ^~ /api/spx/0dte/experimental/ { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location ^~ /api/spx/0dte/replay/ { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location = /api/spx/0dte/scenario { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + location = /api/spx/0dte/collector/events { proxy_pass http://127.0.0.1:8000; proxy_http_version 1.1; diff --git a/ops/amh-nginx/gammascope.nginx.conf b/ops/amh-nginx/gammascope.nginx.conf index 9a2f4df..b8c82b1 100644 --- a/ops/amh-nginx/gammascope.nginx.conf +++ b/ops/amh-nginx/gammascope.nginx.conf @@ -67,6 +67,69 @@ server { proxy_send_timeout 60s; } + location ^~ /api/admin/ { + proxy_pass http://gammascope_web; + proxy_http_version 1.1; + proxy_read_timeout 60s; + proxy_send_timeout 60s; + } + + location ^~ /api/replay/imports { + proxy_pass http://gammascope_web; + proxy_http_version 1.1; + proxy_read_timeout 60s; + proxy_send_timeout 60s; + } + + location = /api/views { + proxy_pass http://gammascope_web; + proxy_http_version 1.1; + proxy_read_timeout 60s; + proxy_send_timeout 60s; + } + + location = /api/spx/0dte/snapshot/latest { + proxy_pass http://gammascope_web; + proxy_http_version 1.1; + proxy_read_timeout 60s; + proxy_send_timeout 60s; + } + + location = /api/spx/0dte/status { + proxy_pass http://gammascope_web; + proxy_http_version 1.1; + proxy_read_timeout 60s; + proxy_send_timeout 60s; + } + + location = /api/spx/0dte/heatmap/latest { + proxy_pass http://gammascope_web; + proxy_http_version 1.1; + proxy_read_timeout 60s; + proxy_send_timeout 60s; + } + + location ^~ /api/spx/0dte/experimental/ { + proxy_pass http://gammascope_web; + proxy_http_version 1.1; + proxy_read_timeout 60s; + proxy_send_timeout 60s; + } + + location ^~ /api/spx/0dte/replay/ { + proxy_pass http://gammascope_web; + proxy_http_version 1.1; + proxy_read_timeout 60s; + proxy_send_timeout 60s; + } + + location = /api/spx/0dte/scenario { + proxy_pass http://gammascope_web; + proxy_http_version 1.1; + proxy_read_timeout 60s; + proxy_send_timeout 60s; + } + location = /api/spx/0dte/collector/events { proxy_pass http://gammascope_api; proxy_read_timeout 30s; From 65d85c39716115ab3f7d7b246bb841e894b78e55 Mon Sep 17 00:00:00 2001 From: Sakura <32687351+zifanzhou1024@users.noreply.github.com> Date: Sun, 3 May 2026 21:18:39 -0700 Subject: [PATCH 08/11] fix: make live dashboard public in private mode --- README.md | 11 +++------ apps/api/gammascope_api/auth.py | 11 +++------ apps/api/gammascope_api/routes/stream.py | 5 ---- apps/api/tests/test_heatmap_route.py | 10 ++++---- apps/api/tests/test_private_mode.py | 31 ++++++++++++------------ docs/amh-nginx-server-setup.md | 8 +++--- ops/amh-nginx/README.md | 2 ++ 7 files changed, 35 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 6d5d5d0..c1ce49b 100644 --- a/README.md +++ b/README.md @@ -93,21 +93,18 @@ Set private mode when the API may be reachable by non-admin users: `GAMMASCOPE_PRIVATE_MODE=true` is also accepted. Truthy values are `1`, `true`, `yes`, `on`, and `enabled`. -In private mode, public replay remains open: +In private mode, public viewing remains open. Live snapshots, live status, scenarios, live WebSocket updates, replay, heatmap, and experimental analytics do not require an admin token: curl -s http://127.0.0.1:8000/api/spx/0dte/replay/sessions | python -m json.tool curl -s "http://127.0.0.1:8000/api/spx/0dte/replay/snapshot?session_id=seed-spx-2026-04-23" | python -m json.tool + curl -s http://127.0.0.1:8000/api/spx/0dte/snapshot/latest | python -m json.tool -Live collector state requires the admin token: +Collector ingestion, raw collector state, replay imports, and maintenance/admin operations require the admin token: curl -s -H "X-GammaScope-Admin-Token: local-admin-token" \ http://127.0.0.1:8000/api/spx/0dte/collector/state | python -m json.tool -The live WebSocket accepts the same header, or `admin_token` as a query parameter for simple local clients: - - ws://127.0.0.1:8000/ws/spx/0dte?admin_token=local-admin-token - -Without a valid admin token, private-mode latest snapshot, status, and scenario requests use seeded replay/fallback data instead of live collector state. Saved-view public requests list only `owner_scope: "public_demo"`; creating or listing admin scoped views requires the admin token. If `GAMMASCOPE_ADMIN_TOKEN` is unset or blank, private admin operations return `403`. +Saved-view public requests list only `owner_scope: "public_demo"`; creating or listing admin scoped views requires the admin token. If `GAMMASCOPE_ADMIN_TOKEN` is unset or blank, private admin operations return `403`. ### Local IBKR Health Probe diff --git a/apps/api/gammascope_api/auth.py b/apps/api/gammascope_api/auth.py index 34f45ca..d33e39e 100644 --- a/apps/api/gammascope_api/auth.py +++ b/apps/api/gammascope_api/auth.py @@ -3,14 +3,13 @@ import os import secrets -from fastapi import HTTPException, WebSocket +from fastapi import HTTPException ADMIN_TOKEN_ENV = "GAMMASCOPE_ADMIN_TOKEN" PRIVATE_MODE_ENABLED_ENV = "GAMMASCOPE_PRIVATE_MODE_ENABLED" PRIVATE_MODE_LEGACY_ENV = "GAMMASCOPE_PRIVATE_MODE" ADMIN_TOKEN_HEADER = "X-GammaScope-Admin-Token" -ADMIN_TOKEN_QUERY_PARAM = "admin_token" _TRUTHY_VALUES = {"1", "true", "yes", "on", "enabled"} @@ -40,12 +39,8 @@ def require_private_mode_admin_token(token: str | None) -> None: require_admin_token(token) -def can_read_live_state(token: str | None) -> bool: - return not private_mode_enabled() or is_valid_admin_token(token) - - -def websocket_admin_token(websocket: WebSocket) -> str | None: - return websocket.headers.get(ADMIN_TOKEN_HEADER) or websocket.query_params.get(ADMIN_TOKEN_QUERY_PARAM) +def can_read_live_state(_token: str | None) -> bool: + return True def _truthy_env(name: str) -> bool: diff --git a/apps/api/gammascope_api/routes/stream.py b/apps/api/gammascope_api/routes/stream.py index 01ce202..22d9c33 100644 --- a/apps/api/gammascope_api/routes/stream.py +++ b/apps/api/gammascope_api/routes/stream.py @@ -3,7 +3,6 @@ from fastapi import APIRouter, HTTPException, Query, WebSocket, WebSocketDisconnect -from gammascope_api.auth import is_valid_admin_token, private_mode_enabled, websocket_admin_token from gammascope_api.fixtures import load_json_fixture from gammascope_api.ingestion.live_snapshot_service import get_live_snapshot_service from gammascope_api.routes.replay import replay_stream_snapshots, seed_replay_snapshots @@ -19,10 +18,6 @@ @router.websocket("/ws/spx/0dte") async def stream_spx_0dte(websocket: WebSocket) -> None: - if private_mode_enabled() and not is_valid_admin_token(websocket_admin_token(websocket)): - await websocket.close(code=1008) - return - await websocket.accept() try: while True: diff --git a/apps/api/tests/test_heatmap_route.py b/apps/api/tests/test_heatmap_route.py index 541c98c..9cf498c 100644 --- a/apps/api/tests/test_heatmap_route.py +++ b/apps/api/tests/test_heatmap_route.py @@ -147,7 +147,7 @@ def test_latest_heatmap_route_fallback_does_not_write_configured_repository() -> assert repository.snapshot_upserts == 0 -def test_latest_heatmap_route_private_fallback_does_not_write_configured_repository(monkeypatch) -> None: +def test_latest_heatmap_route_private_mode_returns_public_live_heatmap(monkeypatch) -> None: monkeypatch.setenv("GAMMASCOPE_PRIVATE_MODE_ENABLED", "true") monkeypatch.setenv("GAMMASCOPE_ADMIN_TOKEN", "local-admin-token") repository = _RecordingHeatmapRepository() @@ -162,10 +162,10 @@ def test_latest_heatmap_route_private_fallback_does_not_write_configured_reposit response = client.get("/api/spx/0dte/heatmap/latest") assert response.status_code == 200 - assert response.json()["sessionId"] != "moomoo-spx-0dte-live" - assert response.json()["persistenceStatus"] == "skipped" - assert repository.baseline_upserts == 0 - assert repository.snapshot_upserts == 0 + assert response.json()["sessionId"] == "moomoo-spx-0dte-live" + assert response.json()["persistenceStatus"] == "persisted" + assert repository.baseline_upserts == 1 + assert repository.snapshot_upserts == 1 class _RecordingHeatmapRepository(InMemoryHeatmapRepository): diff --git a/apps/api/tests/test_private_mode.py b/apps/api/tests/test_private_mode.py index 8d4de30..4a4b28d 100644 --- a/apps/api/tests/test_private_mode.py +++ b/apps/api/tests/test_private_mode.py @@ -1,6 +1,5 @@ import pytest from fastapi.testclient import TestClient -from starlette.websockets import WebSocketDisconnect from gammascope_api.ingestion.collector_state import collector_state from gammascope_api.ingestion.latest_state_cache import ( @@ -120,7 +119,7 @@ def test_collector_ingest_validation_errors_keep_body_locations( assert all("url" not in error for error in response.json()["detail"]) -def test_private_mode_latest_snapshot_hides_live_state_without_admin_token(monkeypatch: pytest.MonkeyPatch) -> None: +def test_private_mode_latest_snapshot_is_public_without_admin_token(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("GAMMASCOPE_PRIVATE_MODE_ENABLED", "true") monkeypatch.setenv("GAMMASCOPE_ADMIN_TOKEN", "local-admin-token") @@ -138,14 +137,14 @@ def test_private_mode_latest_snapshot_hides_live_state_without_admin_token(monke ) assert public_response.status_code == 200 - assert public_response.json()["mode"] == "replay" - assert public_response.json()["session_id"] == "seed-spx-2026-04-23" + assert public_response.json()["mode"] == "live" + assert public_response.json()["session_id"] == "private-live-session" assert admin_response.status_code == 200 assert admin_response.json()["mode"] == "live" assert admin_response.json()["session_id"] == "private-live-session" -def test_private_mode_status_and_scenario_hide_live_state_without_admin_token(monkeypatch: pytest.MonkeyPatch) -> None: +def test_private_mode_status_and_scenario_are_public_without_admin_token(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("GAMMASCOPE_PRIVATE_MODE_ENABLED", "true") monkeypatch.setenv("GAMMASCOPE_ADMIN_TOKEN", "local-admin-token") @@ -180,9 +179,9 @@ def test_private_mode_status_and_scenario_hide_live_state_without_admin_token(mo ) assert public_status_response.status_code == 200 - assert public_status_response.json()["message"] != "Mock live cycle" + assert public_status_response.json()["message"] == "Mock live cycle" assert public_scenario_response.status_code == 200 - assert public_scenario_response.json()["session_id"] == "seed-spx-2026-04-23" + assert public_scenario_response.json()["session_id"] == "private-scenario-session" assert admin_scenario_response.status_code == 200 assert admin_scenario_response.json()["session_id"] == "private-scenario-session" @@ -206,20 +205,22 @@ def test_private_mode_keeps_replay_rest_open(monkeypatch: pytest.MonkeyPatch) -> assert snapshot_response.json()["mode"] == "replay" -def test_private_mode_live_websocket_requires_token_and_accepts_query_token(monkeypatch: pytest.MonkeyPatch) -> None: +def test_private_mode_live_websocket_is_public_without_admin_token(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("GAMMASCOPE_PRIVATE_MODE_ENABLED", "true") monkeypatch.setenv("GAMMASCOPE_ADMIN_TOKEN", "local-admin-token") - with pytest.raises(WebSocketDisconnect) as disconnect: - with client.websocket_connect("/ws/spx/0dte") as websocket: - websocket.receive_json() + for event in _live_events("private-websocket-session"): + assert client.post( + "/api/spx/0dte/collector/events", + json=event, + headers={"X-GammaScope-Admin-Token": "local-admin-token"}, + ).status_code == 200 - with client.websocket_connect("/ws/spx/0dte?admin_token=local-admin-token") as websocket: + with client.websocket_connect("/ws/spx/0dte") as websocket: payload = websocket.receive_json() - assert disconnect.value.code == 1008 - assert payload["mode"] == "replay" - assert payload["session_id"] == "seed-spx-2026-04-23" + assert payload["mode"] == "live" + assert payload["session_id"] == "private-websocket-session" def test_private_mode_keeps_replay_websocket_public(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/docs/amh-nginx-server-setup.md b/docs/amh-nginx-server-setup.md index 890bc39..a1bb045 100644 --- a/docs/amh-nginx-server-setup.md +++ b/docs/amh-nginx-server-setup.md @@ -8,9 +8,9 @@ This guide moves GammaScope from a local-only setup to a server layout where AMH flowchart LR Mac["Your computer\nMoomoo OpenD + GammaScope collector"] -->|HTTPS bulk collector events + admin token| Nginx["AMH / Nginx\n80 and 443"] Browser["Browser"] -->|HTTPS| Nginx - Nginx -->|collector endpoints and ws| API["FastAPI container\n127.0.0.1:8000"] + Nginx -->|collector endpoints and public ws| API["FastAPI container\n127.0.0.1:8000"] Nginx -->|web app and Next API routes| Web["Next.js container\n127.0.0.1:3000"] - Web -->|server-side API proxy + admin token| API + Web -->|server-side public API proxy| API API --> Postgres["Postgres volume"] API --> Redis["Redis container"] ``` @@ -550,6 +550,8 @@ https://gamma.hiqjj.org/ https://gamma.hiqjj.org/heatmap ``` +Live dashboard viewing is public. The web admin login is only needed for replay import/upload flows; collector ingestion and maintenance commands still require the generated admin token. + ## 11. Operating Commands Rebuild after code changes: @@ -587,6 +589,6 @@ If collector publish returns `403`, the computer's `GAMMASCOPE_ADMIN_TOKEN` does If collector publish cannot connect, check DNS, HTTPS, firewall, and the Nginx collector locations. The collector should publish to the public origin, not to `127.0.0.1`. -If the browser live WebSocket is unavailable in private mode, that is expected unless the browser has an admin token. The web app's server-side API routes can still fetch live data using the server-side `GAMMASCOPE_ADMIN_TOKEN`, and the dashboard should fall back to polling. +If the browser live WebSocket is unavailable, check the AMH `/ws/` reverse proxy rule. Live dashboard viewing is public, so a missing admin login should not block realtime data. If AMH overwrites manual Nginx edits, move the custom locations into AMH's supported custom vhost/rules field or keep a copy of `ops/amh-nginx/gammascope.nginx.conf` and re-apply after AMH regenerates configs. diff --git a/ops/amh-nginx/README.md b/ops/amh-nginx/README.md index 96fce8c..f995384 100644 --- a/ops/amh-nginx/README.md +++ b/ops/amh-nginx/README.md @@ -414,6 +414,8 @@ https://gamma.hiqjj.org/ https://gamma.hiqjj.org/heatmap ``` +Live dashboard viewing is public. The web admin login is only needed for replay import/upload flows; collector ingestion still uses the generated admin token. + ## 8. Operations Update and rebuild: From 7dad4dde016a703041e02d800ac5b11a75b60955 Mon Sep 17 00:00:00 2001 From: Sakura <32687351+zifanzhou1024@users.noreply.github.com> Date: Sun, 3 May 2026 21:47:48 -0700 Subject: [PATCH 09/11] docs: consolidate production deployment runbook --- docs/deployment.md | 811 ++++++++++++++++++++++++++++++---- tests/deployment-doc.test.mjs | 26 ++ 2 files changed, 740 insertions(+), 97 deletions(-) create mode 100644 tests/deployment-doc.test.mjs diff --git a/docs/deployment.md b/docs/deployment.md index a996e4b..d98c2d6 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1,124 +1,635 @@ -# GammaScope Deployment Guide +# GammaScope Production Deployment -This guide covers the current deployable GammaScope stack for the live Moomoo-backed 0DTE heatmap and dashboard. +This is the canonical deployment runbook for the working `gamma.hiqjj.org` setup. -## Components +It captures the server layout, AMH/Nginx routing, Docker Compose stack, local Moomoo collector, operational commands, and smoke tests used to get the current deployment working. It intentionally does not contain passwords, tokens, SSH passwords, or generated secrets. -Run these services together: +## Current Production Shape -- Postgres for replay snapshots, heatmap snapshots, 5-minute heatmap buckets, OI baselines, and saved views. -- FastAPI app from `apps/api`, exposed to the web app and collector. -- Next.js web app from `apps/web`. -- Moomoo OpenD on the machine that can access the licensed Moomoo data session. -- Moomoo snapshot collector from `services/collector`. +Use these values for the current deployment unless you are intentionally creating a new environment: -Redis is not required for this deployment. The heatmap history and replay path use Postgres. +```text +Public domain: gamma.hiqjj.org +Server SSH target: root@149.56.14.95 or root@gamma.hiqjj.org +Server app path: /opt/gammascope +GitHub repo: https://github.com/zifanzhou1024/gamma-scope.git +Deployment branch: codex/amh-nginx-server-setup +Server OS: Debian +Public reverse proxy: AMH Nginx +API container port: 127.0.0.1:8000 +Web container port: 127.0.0.1:3000 +Collector machine: local Mac with Moomoo OpenD on 127.0.0.1:11111 +``` + +Access policy: -## Required Environment +- Public visitors can view the live dashboard, live heatmap, replay, experimental pages, and live WebSocket data without logging in. +- The web admin login is for replay import/upload and future admin-only actions. +- Collector ingestion, raw collector state, replay import mutation, and maintenance endpoints still require `GAMMASCOPE_ADMIN_TOKEN`. +- Never commit `gammascope.production.env`, `gammascope.collector-client.env`, database dumps, replay parquet files, or raw licensed market data. -Set these for the API: +## Architecture + +```mermaid +flowchart LR + Mac["Local Mac\nMoomoo OpenD + GammaScope collector"] -->|"HTTPS bulk collector events\nX-GammaScope-Admin-Token"| Nginx["AMH / Nginx\n80 and 443"] + Browser["Public browser"] -->|HTTPS| Nginx + Nginx -->|"web app + Next API routes"| Web["Next.js container\n127.0.0.1:3000"] + Nginx -->|"collector endpoints + /ws/"| API["FastAPI container\n127.0.0.1:8000"] + Web -->|"GAMMASCOPE_API_BASE_URL=http://api:8000"| API + API --> Postgres["Postgres volume"] + API --> Redis["Redis container"] +``` + +The server does not need Moomoo OpenD. Keep OpenD on the computer that has the licensed Moomoo data session, then publish snapshots to the public server API. + +## Files That Matter + +```text +docs/deployment.md This runbook +docs/amh-nginx-server-setup.md Longer AMH setup notes +ops/amh-nginx/README.md Condensed AMH runbook with pasteable location blocks +ops/amh-nginx/docker-compose.amh.yml Server Compose stack +ops/amh-nginx/gammascope.nginx.conf Full Nginx vhost template +ops/amh-nginx/generate_secrets.py Env/secret generator +ops/amh-nginx/gammascope.production.env.example Server env template +ops/amh-nginx/gammascope.collector-client.env.example Local collector env template +``` + +Generated files are ignored by Git: + +```text +ops/amh-nginx/gammascope.production.env +ops/amh-nginx/gammascope.collector-client.env +``` + +## 1. DNS, Firewall, and AMH + +DNS should point the subdomain at the VPS: + +```text +Type: A +Name: gamma +Value: 149.56.14.95 +TTL: Auto or 300 +``` + +Verify DNS: ```bash -GAMMASCOPE_DATABASE_URL=postgresql://gammascope:gammascope@127.0.0.1:5432/gammascope +dig +short gamma.hiqjj.org +``` + +Expected: + +```text +149.56.14.95 +``` + +Firewall rules: + +```text +22/tcp SSH, preferably restricted to your IP +80/tcp HTTP certificate challenge and redirect +443/tcp HTTPS app +AMH panel only from trusted IPs +``` + +Do not expose Postgres, Redis, `8000`, or `3000` publicly. The Compose file binds API and web to localhost. + +In AMH, create a site/vhost for: + +```text +gamma.hiqjj.org +``` + +Enable SSL in AMH/AMSSL for that vhost. The site root can remain AMH's default generated web root because Nginx proxies requests to Docker. + +## 2. Install Docker on Debian + +Run as `root` on the VPS. This block also fixes the earlier failure mode where an Ubuntu Docker repo line was accidentally added to a Debian system. + +```bash +set -eux + +rm -f /etc/apt/sources.list.d/docker.list +rm -f /etc/apt/sources.list.d/docker.sources + +apt-get update +apt-get install -y ca-certificates curl git openssl python3 + +. /etc/os-release +test "$ID" = "debian" + +install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc +chmod a+r /etc/apt/keyrings/docker.asc + +cat > /etc/apt/sources.list.d/docker.sources < +GAMMASCOPE_PRIVATE_MODE_ENABLED=true +GAMMASCOPE_ADMIN_TOKEN= +GAMMASCOPE_WEB_ADMIN_USERNAME=admin +GAMMASCOPE_WEB_ADMIN_PASSWORD= +GAMMASCOPE_WEB_ADMIN_SESSION_SECRET= +GAMMASCOPE_API_HOST_PORT=8000 +GAMMASCOPE_WEB_HOST_PORT=3000 GAMMASCOPE_REPLAY_CAPTURE_INTERVAL_SECONDS=5 GAMMASCOPE_REPLAY_RETENTION_DAYS=20 GAMMASCOPE_SAVED_VIEW_RETENTION_DAYS=90 +GAMMASCOPE_REPLAY_IMPORT_MAX_BYTES=104857600 +``` + +The generated collector env contains: + +```text +GAMMASCOPE_SERVER_API=https://gamma.hiqjj.org +GAMMASCOPE_ADMIN_TOKEN= +GAMMASCOPE_MOOMOO_HOST=127.0.0.1 +GAMMASCOPE_MOOMOO_PORT=11111 +GAMMASCOPE_RUT_SPOT=2050 +GAMMASCOPE_NDX_SPOT=18300 ``` -Set private mode when the API is reachable outside a trusted local machine: +If you are rotating secrets on a fresh test install, remove the old database volume first: ```bash -GAMMASCOPE_PRIVATE_MODE_ENABLED=true -GAMMASCOPE_ADMIN_TOKEN= +cd /opt/gammascope + +docker compose \ + --env-file ops/amh-nginx/gammascope.production.env \ + -f ops/amh-nginx/docker-compose.amh.yml \ + down -v + +python3 ops/amh-nginx/generate_secrets.py \ + --domain gamma.hiqjj.org \ + --server-output ops/amh-nginx/gammascope.production.env \ + --collector-output ops/amh-nginx/gammascope.collector-client.env \ + --force +``` + +Do not run `down -v` on production data unless you have a verified database backup. + +## 5. Start the Server Containers + +Run on the VPS: + +```bash +cd /opt/gammascope + +docker compose \ + --env-file ops/amh-nginx/gammascope.production.env \ + -f ops/amh-nginx/docker-compose.amh.yml \ + up -d --build +``` + +Check status: + +```bash +docker compose \ + --env-file ops/amh-nginx/gammascope.production.env \ + -f ops/amh-nginx/docker-compose.amh.yml \ + ps +``` + +Expected services: + +```text +gammascope-api-1 healthy, 127.0.0.1:8000->8000 +gammascope-web-1 running/healthy, 127.0.0.1:3000->3000 +gammascope-postgres-1 healthy +gammascope-redis-1 healthy +``` + +Local server smoke tests: + +```bash +curl -I http://127.0.0.1:3000/ +curl -fsS http://127.0.0.1:8000/api/spx/0dte/replay/sessions | python3 -m json.tool +``` + +## 6. Configure AMH/Nginx + +The key routing rule is: browser-facing app and Next API routes go to `127.0.0.1:3000`; collector ingestion and live WebSocket go to `127.0.0.1:8000`. + +Route table: + +```text +/_next/ -> http://127.0.0.1:3000 +/images/ -> http://127.0.0.1:3000 +/favicon.ico -> http://127.0.0.1:3000 +/api/admin/ -> http://127.0.0.1:3000 +/api/replay/imports -> http://127.0.0.1:3000 +/api/views -> http://127.0.0.1:3000 +/api/spx/0dte/snapshot/latest -> http://127.0.0.1:3000 +/api/spx/0dte/status -> http://127.0.0.1:3000 +/api/spx/0dte/heatmap/latest -> http://127.0.0.1:3000 +/api/spx/0dte/experimental/ -> http://127.0.0.1:3000 +/api/spx/0dte/replay/ -> http://127.0.0.1:3000 +/api/spx/0dte/scenario -> http://127.0.0.1:3000 +/ -> http://127.0.0.1:3000 +/ws/ -> http://127.0.0.1:8000 +/api/spx/0dte/collector/events -> http://127.0.0.1:8000 +/api/spx/0dte/collector/events/bulk -> http://127.0.0.1:8000 +``` + +Do not add a broad `/api/ -> 127.0.0.1:8000` rule. It will bypass Next-owned routes such as admin login, replay import proxying, and browser API route handling. + +For AMH URL rules, paste only `location ... { ... }` blocks, not a full `server { ... }` block. The pasteable block set is in: + +```text +/opt/gammascope/ops/amh-nginx/README.md +``` + +The full Nginx vhost template is: + +```text +/opt/gammascope/ops/amh-nginx/gammascope.nginx.conf +``` + +If using the full template directly, copy it only after the TLS certificate exists: + +```bash +cp /opt/gammascope/ops/amh-nginx/gammascope.nginx.conf /etc/nginx/conf.d/gammascope.conf +``` + +If AMH manages SSL somewhere other than Let's Encrypt's default path, update these lines in the copied config: + +```text +ssl_certificate /etc/letsencrypt/live/gamma.hiqjj.org/fullchain.pem; +ssl_certificate_key /etc/letsencrypt/live/gamma.hiqjj.org/privkey.pem; +``` + +### Reloading AMH Nginx + +On the current server, `nginx.service` may be inactive because AMH runs its own Nginx binary under `/usr/local/nginx-1.24/sbin/nginx`. If `systemctl reload nginx` fails, use the AMH binary: + +```bash +AMH_NGINX=/usr/local/nginx-1.24/sbin/nginx + +$AMH_NGINX -t +$AMH_NGINX -s reload || { + master_pid="$(pgrep -o -x nginx)" + kill -HUP "$master_pid" +} ``` -Set these for the web app: +Useful check: ```bash -GAMMASCOPE_API_BASE_URL=http://127.0.0.1:8000 -NEXT_PUBLIC_GAMMASCOPE_WS_URL=ws://127.0.0.1:8000/ws/spx/0dte +ps -ef | grep '[n]ginx' ``` -When private mode is enabled, also pass the same `GAMMASCOPE_ADMIN_TOKEN` to the web process so authenticated admin requests can proxy live data. +## 7. Public Server Smoke Tests + +Run from your computer after AMH/Nginx is configured: -## Install +```bash +curl -I https://gamma.hiqjj.org/ +curl -fsS https://gamma.hiqjj.org/api/spx/0dte/replay/sessions | python3 -m json.tool +``` + +Expected: + +```text +HTTP/2 200 +``` -Install repo dependencies and the Python packages used by the API and collector: +Verify Next static assets are proxied to the web container: ```bash +ASSET_PATH="$(curl -fsS https://gamma.hiqjj.org/ | grep -oE '/_next/[^"]+' | head -1)" +echo "$ASSET_PATH" +curl -I "https://gamma.hiqjj.org$ASSET_PATH" +``` + +Expected: `HTTP/2 200` and a CSS or JavaScript content type. If assets do not load, ensure the `location ^~ /_next/` rule proxies to `127.0.0.1:3000`. + +Verify collector ingestion is protected: + +```bash +curl -i -X POST https://gamma.hiqjj.org/api/spx/0dte/collector/events/bulk \ + -H 'Content-Type: application/json' \ + --data '[]' +``` + +Expected without token: `403`. + +With the generated collector token loaded locally: + +```bash +set -a +. ops/amh-nginx/gammascope.collector-client.env +set +a + +curl -i -X POST https://gamma.hiqjj.org/api/spx/0dte/collector/events/bulk \ + -H "X-GammaScope-Admin-Token: $GAMMASCOPE_ADMIN_TOKEN" \ + -H 'Content-Type: application/json' \ + --data '[]' +``` + +Expected with an empty batch: `200` and `accepted_count: 0`. + +## 8. Configure the Local Collector Machine + +Run this on the Mac that runs Moomoo OpenD. + +Install repo dependencies if needed: + +```bash +cd /Users/sakura/WebstormProjects/gamma-scope/.worktrees/amh-nginx-server-setup + pnpm install python3 -m venv .venv .venv/bin/python -m pip install -e "apps/api[dev]" .venv/bin/python -m pip install --upgrade moomoo-api pandas ``` -For local Postgres, start the compose service: +Copy the collector env from the server: ```bash -docker compose up -d postgres +cd /Users/sakura/WebstormProjects/gamma-scope/.worktrees/amh-nginx-server-setup + +mkdir -p ops/amh-nginx +scp root@gamma.hiqjj.org:/opt/gammascope/ops/amh-nginx/gammascope.collector-client.env \ + ops/amh-nginx/gammascope.collector-client.env +chmod 600 ops/amh-nginx/gammascope.collector-client.env ``` -For production, point `GAMMASCOPE_DATABASE_URL` at a persistent Postgres instance and keep the database on private networking when possible. +If SSH aliasing is broken, bypass `~/.ssh/config` and use the IP: + +```bash +scp -F /dev/null root@149.56.14.95:/opt/gammascope/ops/amh-nginx/gammascope.collector-client.env \ + ops/amh-nginx/gammascope.collector-client.env +``` -## Start Order +Start Moomoo OpenD locally and confirm it listens on `127.0.0.1:11111`: -Start Postgres first, then API, then web, then Moomoo OpenD and the collector. +```bash +python3 - <<'PY' +import socket +s = socket.socket() +s.settimeout(2) +try: + s.connect(("127.0.0.1", 11111)) + print("moomoo-opend-port=reachable") +finally: + s.close() +PY +``` -Run the API: +Load the collector env: ```bash -PYTHONPATH=apps/api \ -GAMMASCOPE_DATABASE_URL=postgresql://gammascope:gammascope@127.0.0.1:5432/gammascope \ -.venv/bin/python -m uvicorn gammascope_api.main:app \ - --app-dir apps/api \ - --host 0.0.0.0 \ - --port 8000 +set -a +. ops/amh-nginx/gammascope.collector-client.env +set +a + +printf 'api=%s host=%s port=%s\n' \ + "$GAMMASCOPE_SERVER_API" \ + "$GAMMASCOPE_MOOMOO_HOST" \ + "$GAMMASCOPE_MOOMOO_PORT" ``` -Run the web app in development mode: +Run one bounded publish: ```bash -GAMMASCOPE_API_BASE_URL=http://127.0.0.1:8000 \ -NEXT_PUBLIC_GAMMASCOPE_WS_URL=ws://127.0.0.1:8000/ws/spx/0dte \ -pnpm dev:web +pnpm collector:moomoo-snapshot \ + --host "$GAMMASCOPE_MOOMOO_HOST" \ + --port "$GAMMASCOPE_MOOMOO_PORT" \ + --api "$GAMMASCOPE_SERVER_API" \ + --spot RUT="$GAMMASCOPE_RUT_SPOT" \ + --spot NDX="$GAMMASCOPE_NDX_SPOT" \ + --max-loops 1 \ + --publish ``` -For a long-running non-dev Next.js process, build and start Next directly: +Expected: JSON output with `status: "connected"` and `publish.accepted_count` greater than zero. + +If Python reports `SSL: CERTIFICATE_VERIFY_FAILED` on macOS, update to the latest branch. The collector publisher falls back to `/etc/ssl/cert.pem`. If a local environment still fails, run the collector with: ```bash -GAMMASCOPE_API_BASE_URL=http://127.0.0.1:8000 \ -NEXT_PUBLIC_GAMMASCOPE_WS_URL=ws://127.0.0.1:8000/ws/spx/0dte \ -pnpm --filter @gammascope/web exec next build +SSL_CERT_FILE=/etc/ssl/cert.pem pnpm collector:moomoo-snapshot ... +``` + +## 9. Run the Collector Continuously -GAMMASCOPE_API_BASE_URL=http://127.0.0.1:8000 \ -NEXT_PUBLIC_GAMMASCOPE_WS_URL=ws://127.0.0.1:8000/ws/spx/0dte \ -pnpm --filter @gammascope/web exec next start --hostname 0.0.0.0 --port 3000 +The one-loop command is only a smoke test. The live dashboard needs a continuous collector process. + +Use `screen` on the Mac: + +```bash +cd /Users/sakura/WebstormProjects/gamma-scope/.worktrees/amh-nginx-server-setup +mkdir -p .gammascope + +screen -S gammascope-collector -X quit >/dev/null 2>&1 || true + +screen -dmS gammascope-collector zsh -lc ' + cd /Users/sakura/WebstormProjects/gamma-scope/.worktrees/amh-nginx-server-setup || exit 1 + set -a + . ops/amh-nginx/gammascope.collector-client.env + set +a + pnpm collector:moomoo-snapshot \ + --host "$GAMMASCOPE_MOOMOO_HOST" \ + --port "$GAMMASCOPE_MOOMOO_PORT" \ + --api "$GAMMASCOPE_SERVER_API" \ + --spot RUT="$GAMMASCOPE_RUT_SPOT" \ + --spot NDX="$GAMMASCOPE_NDX_SPOT" \ + --publish 2>&1 | tee -a .gammascope/moomoo-collector.screen.log +' + +screen -ls +tail -f .gammascope/moomoo-collector.screen.log ``` -Use a process manager such as `systemd`, `supervisord`, `pm2`, or `screen` to keep the API, web app, and collector alive. +Attach: -## Moomoo Collector +```bash +screen -r gammascope-collector +``` -Start Moomoo OpenD locally and confirm it listens on: +Detach without stopping: ```text -host=127.0.0.1 -port=11111 +Ctrl-a d ``` -Run the Moomoo collector against the API: +Stop: ```bash -pnpm collector:moomoo-snapshot -- \ - --api http://127.0.0.1:8000 \ - --spot RUT=2050 \ - --spot NDX=18300 \ - --publish +screen -S gammascope-collector -X quit +``` + +The collector publishes every 2 seconds during active windows and slows to about 60 seconds outside active market/pre-open windows. During off-hours, wait at least one full minute before deciding it is not updating. + +## 10. Verify Realtime Data + +Public latest snapshot should be live without logging in: + +```bash +curl -fsS https://gamma.hiqjj.org/api/admin/session | python3 -m json.tool + +curl -fsS https://gamma.hiqjj.org/api/spx/0dte/snapshot/latest \ + | python3 -c 'import json,sys; p=json.load(sys.stdin); print({k:p.get(k) for k in ["session_id","mode","symbol","expiry","spot","snapshot_time","source_status","freshness_ms"]}); print("rows", len(p.get("rows", [])))' +``` + +Expected: + +```text +authenticated=false +mode=live +session_id=moomoo-spx-0dte-live +rows > 0 ``` -The collector currently discovers SPX, SPY, QQQ, IWM, RUT, and NDX. The heatmap API processes SPX, SPY, QQQ, IWM, and NDX. SPX is exposed as `SPXW` in heatmap payloads; the other symbols use their own trading class. +Public heatmap should be live: + +```bash +curl -fsS 'https://gamma.hiqjj.org/api/spx/0dte/heatmap/latest?metric=gex&symbol=SPX' \ + | python3 -c 'import json,sys; p=json.load(sys.stdin); print({k:p.get(k) for k in ["sessionId","symbol","isLive","persistenceStatus"]}); print("rows", len(p.get("rows", [])))' +``` -The API captures every ready Moomoo heatmap session from each bulk publish. This is important for multi-panel heatmap deployments: replay persistence should contain these live session IDs after successful collector publishes: +Expected: + +```text +sessionId=moomoo-spx-0dte-live +isLive=True +rows > 0 +``` + +Public live WebSocket should work without admin auth. Node 22 has `WebSocket` built in: + +```bash +node - <<'NODE' +const ws = new WebSocket('wss://gamma.hiqjj.org/ws/spx/0dte'); +const timeout = setTimeout(() => { + console.error('websocket timeout'); + try { ws.close(); } catch {} + process.exit(1); +}, 10000); + +ws.addEventListener('message', (event) => { + clearTimeout(timeout); + const payload = JSON.parse(event.data); + console.log(JSON.stringify({ + session_id: payload.session_id, + mode: payload.mode, + symbol: payload.symbol, + rows: Array.isArray(payload.rows) ? payload.rows.length : null + })); + ws.close(); + process.exit(0); +}); + +ws.addEventListener('error', () => { + clearTimeout(timeout); + console.error('websocket error'); + process.exit(1); +}); +NODE +``` + +Expected: + +```json +{"session_id":"moomoo-spx-0dte-live","mode":"live","symbol":"SPX","rows":122} +``` + +Confirm persisted live replay sessions: + +```bash +ssh root@gamma.hiqjj.org +cd /opt/gammascope + +docker compose --env-file ops/amh-nginx/gammascope.production.env -f ops/amh-nginx/docker-compose.amh.yml exec postgres \ + psql -U gammascope -d gammascope -c " + select session_id, symbol, snapshot_count, end_time + from replay_sessions + where session_id like 'moomoo-%-0dte-live' + order by session_id; + " +``` + +Expected session IDs include: ```text moomoo-spx-0dte-live @@ -128,75 +639,181 @@ moomoo-iwm-0dte-live moomoo-ndx-0dte-live ``` -Open interest from Moomoo is treated as the daily OI baseline once captured at or after 09:25 New York time. Before that baseline locks, heatmap payloads remain marked provisional. +## 11. Operating Commands + +Update and rebuild server: -## Smoke Checks +```bash +ssh root@gamma.hiqjj.org +cd /opt/gammascope + +git fetch origin +git switch codex/amh-nginx-server-setup +git pull --ff-only + +docker compose \ + --env-file ops/amh-nginx/gammascope.production.env \ + -f ops/amh-nginx/docker-compose.amh.yml \ + up -d --build +``` -Check API health: +View logs: ```bash -curl -s http://127.0.0.1:8000/api/spx/0dte/status | python -m json.tool +docker compose --env-file ops/amh-nginx/gammascope.production.env -f ops/amh-nginx/docker-compose.amh.yml logs -f api +docker compose --env-file ops/amh-nginx/gammascope.production.env -f ops/amh-nginx/docker-compose.amh.yml logs -f web ``` -Check every heatmap symbol: +Restart app containers: ```bash -for symbol in SPX SPY QQQ IWM NDX; do - curl -fsS \ - "http://127.0.0.1:8000/api/spx/0dte/heatmap/latest?metric=gex&symbol=${symbol}" \ - | python -c 'import json,sys; p=json.load(sys.stdin); print(p["symbol"], p["sessionId"], len(p["rows"]), p["lastSyncedAt"])' -done +docker compose --env-file ops/amh-nginx/gammascope.production.env -f ops/amh-nginx/docker-compose.amh.yml restart api web ``` -Expected result: every symbol prints its own symbol, a `moomoo-*-0dte-live` session ID, and a non-zero row count. +Stop app stack without deleting data: -Confirm replay sessions are persisted: +```bash +docker compose --env-file ops/amh-nginx/gammascope.production.env -f ops/amh-nginx/docker-compose.amh.yml down +``` + +Backup Postgres: ```bash -psql "$GAMMASCOPE_DATABASE_URL" -c " - select session_id, symbol, snapshot_count, end_time - from replay_sessions - where session_id like 'moomoo-%-0dte-live' - order by session_id; -" +cd /opt/gammascope +docker compose --env-file ops/amh-nginx/gammascope.production.env -f ops/amh-nginx/docker-compose.amh.yml exec postgres \ + pg_dump -U gammascope gammascope > "gammascope-$(date +%Y%m%d-%H%M%S).sql" +chmod 600 gammascope-*.sql ``` -Open the web UI: +Dry-run retention cleanup: -```text -http://localhost:3000/ -http://localhost:3000/heatmap +```bash +curl -fsS -X POST \ + "http://127.0.0.1:8000/api/admin/retention/cleanup?dry_run=true" \ + | python3 -m json.tool +``` + +Execute cleanup with admin token: + +```bash +set -a +. /opt/gammascope/ops/amh-nginx/gammascope.production.env +set +a + +curl -fsS -X POST \ + -H "X-GammaScope-Admin-Token: $GAMMASCOPE_ADMIN_TOKEN" \ + "http://127.0.0.1:8000/api/admin/retention/cleanup?dry_run=false" \ + | python3 -m json.tool ``` -## Operations +## 12. Troubleshooting -Keep the API and collector on the same trusted network. Collector ingestion can mutate live state and should be protected by private mode if exposed. +### Website loads but CSS/images/assets are missing -Do not commit `.gammascope/`, replay parquet files, database dumps, Moomoo credentials, API tokens, or raw licensed market data. +AMH is probably intercepting Next.js static assets. Ensure this rule exists and uses `^~`: -Run cleanup in dry-run mode first: +```nginx +location ^~ /_next/ { + proxy_pass http://127.0.0.1:3000; +} +``` + +Then hard-refresh the browser. + +### Website shows replay instead of live + +Check whether the public API is live: ```bash -curl -s -X POST \ - "http://127.0.0.1:8000/api/admin/retention/cleanup?dry_run=true" \ - | python -m json.tool +curl -fsS https://gamma.hiqjj.org/api/spx/0dte/snapshot/latest \ + | python3 -c 'import json,sys; p=json.load(sys.stdin); print(p["mode"], p["session_id"], p.get("freshness_ms"))' +``` + +If it returns `replay`, either the server has not been updated to the public-live branch or the API container did not rebuild. Run the update/rebuild commands in section 11. + +If it returns `live` but the browser does not, hard-refresh and confirm the browser is on `https://gamma.hiqjj.org/`, not `http://localhost:3000/`. + +### Live data stops updating + +On the Mac: + +```bash +screen -ls +tail -f /Users/sakura/WebstormProjects/gamma-scope/.worktrees/amh-nginx-server-setup/.gammascope/moomoo-collector.screen.log +``` + +If the screen session is gone, restart it with section 9. If OpenD is not reachable, restart Moomoo OpenD. + +During off-hours, updates are expected to slow to about 60 seconds. + +### Collector publish returns `403` + +The local collector token does not match the server token, or the env file was not loaded. + +On the server: + +```bash +cd /opt/gammascope +grep '^GAMMASCOPE_ADMIN_TOKEN=' ops/amh-nginx/gammascope.production.env ``` -When private mode is enabled, destructive cleanup requires the admin token: +On the Mac: ```bash -curl -s -X POST \ - -H "X-GammaScope-Admin-Token: ${GAMMASCOPE_ADMIN_TOKEN}" \ - "http://127.0.0.1:8000/api/admin/retention/cleanup?dry_run=false" \ - | python -m json.tool +cd /Users/sakura/WebstormProjects/gamma-scope/.worktrees/amh-nginx-server-setup +grep '^GAMMASCOPE_ADMIN_TOKEN=' ops/amh-nginx/gammascope.collector-client.env +``` + +The values must match. Do not paste them anywhere public. + +### Collector publish has certificate verification errors on macOS + +Use the latest branch. If the issue persists: + +```bash +SSL_CERT_FILE=/etc/ssl/cert.pem pnpm collector:moomoo-snapshot \ + --host "$GAMMASCOPE_MOOMOO_HOST" \ + --port "$GAMMASCOPE_MOOMOO_PORT" \ + --api "$GAMMASCOPE_SERVER_API" \ + --spot RUT="$GAMMASCOPE_RUT_SPOT" \ + --spot NDX="$GAMMASCOPE_NDX_SPOT" \ + --max-loops 1 \ + --publish +``` + +### `systemctl reload nginx` fails + +AMH may not use the Debian `nginx.service`. Use the AMH Nginx binary: + +```bash +/usr/local/nginx-1.24/sbin/nginx -t +/usr/local/nginx-1.24/sbin/nginx -s reload +``` + +Fallback: + +```bash +kill -HUP "$(pgrep -o -x nginx)" ``` -## Troubleshooting +### `curl -I https://gamma.hiqjj.org/ws/spx/0dte` returns `404` + +That is not a valid WebSocket test because `curl -I` sends `HEAD`, not a WebSocket upgrade. Use the Node WebSocket test in section 10. + +### Only seeded replay sessions exist + +The server stack is running but no live collector data has been published. Run the one-loop collector smoke test in section 8, then start the continuous collector in section 9. -If only SPX shows data, check that the collector is publishing all Moomoo compatibility events and that the API process has the multi-session replay capture code. The `/api/spx/0dte/collector/state` response should show multiple underlying ticks and option ticks after a successful collector publish. +### Broad `/api/` proxy rule exists in AMH -If SPY, QQQ, IWM, or NDX returns `404`, query `replay_sessions` for the corresponding `moomoo-*-0dte-live` session and run a fresh collector publish. The heatmap route falls back to persisted Moomoo replay snapshots when in-memory collector state is empty. +Remove it. It can bypass Next.js routes and break admin login, replay import proxying, and public browser API behavior. Use the route table in section 6. -If the web page shows unavailable panels but direct API requests work, confirm `GAMMASCOPE_API_BASE_URL` is set for the Next.js process and that the Next route proxy can reach the FastAPI host. +## 13. Security Notes -If the collector reports Moomoo snapshot request failures, confirm OpenD is running, logged in, and allowed to serve the subscribed quote data for the selected symbols. +- Keep the AMH panel restricted to trusted IPs. +- Keep `GAMMASCOPE_ADMIN_TOKEN` private; it can publish collector data. +- Keep the web admin password private; it controls replay import/upload. +- Keep server env files mode `0600`. +- Do not expose `127.0.0.1:8000` or `127.0.0.1:3000` directly. +- Back up Postgres before any `down -v`, secret rotation, or destructive cleanup. +- Treat Moomoo data as licensed market data; do not commit raw snapshots or replay parquet files. diff --git a/tests/deployment-doc.test.mjs b/tests/deployment-doc.test.mjs new file mode 100644 index 0000000..d3080e0 --- /dev/null +++ b/tests/deployment-doc.test.mjs @@ -0,0 +1,26 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; + +const deploymentDoc = readFileSync(new URL("../docs/deployment.md", import.meta.url), "utf8"); + +test("deployment guide documents the working AMH production target", () => { + assert.match(deploymentDoc, /gamma\.hiqjj\.org/); + assert.match(deploymentDoc, /149\.56\.14\.95/); + assert.match(deploymentDoc, /codex\/amh-nginx-server-setup/); + assert.match(deploymentDoc, /\/opt\/gammascope/); + assert.match(deploymentDoc, /\/usr\/local\/nginx-1\.24\/sbin\/nginx/); +}); + +test("deployment guide preserves the public live viewing policy", () => { + assert.match(deploymentDoc, /Public visitors can view the live dashboard/); + assert.match(deploymentDoc, /web admin login is for replay import\/upload/); + assert.match(deploymentDoc, /Collector ingestion, raw collector state, replay import mutation, and maintenance endpoints still require/); +}); + +test("deployment guide includes realtime collector operations and smoke tests", () => { + assert.match(deploymentDoc, /screen -dmS gammascope-collector/); + assert.match(deploymentDoc, /wss:\/\/gamma\.hiqjj\.org\/ws\/spx\/0dte/); + assert.match(deploymentDoc, /moomoo-spx-0dte-live/); + assert.match(deploymentDoc, /Do not add a broad `\/api\/ -> 127\.0\.0\.1:8000` rule/); +}); From 87e2f14fb2b8ecef673e9c65d2821661f3557262 Mon Sep 17 00:00:00 2001 From: Sakura <32687351+zifanzhou1024@users.noreply.github.com> Date: Sun, 3 May 2026 21:58:24 -0700 Subject: [PATCH 10/11] docs: add quick deployment path --- docs/deployment.md | 30 ++++ ops/amh-nginx/bootstrap_gamma_server.sh | 168 ++++++++++++++++++++ ops/amh-nginx/start_moomoo_collector_mac.sh | 92 +++++++++++ tests/deployment-doc.test.mjs | 38 +++++ 4 files changed, 328 insertions(+) create mode 100755 ops/amh-nginx/bootstrap_gamma_server.sh create mode 100755 ops/amh-nginx/start_moomoo_collector_mac.sh diff --git a/docs/deployment.md b/docs/deployment.md index d98c2d6..81be0b4 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -4,6 +4,34 @@ This is the canonical deployment runbook for the working `gamma.hiqjj.org` setup It captures the server layout, AMH/Nginx routing, Docker Compose stack, local Moomoo collector, operational commands, and smoke tests used to get the current deployment working. It intentionally does not contain passwords, tokens, SSH passwords, or generated secrets. +## Quick Start + +Use this when redeploying the same `gamma.hiqjj.org` shape. These commands do not contain credentials. On the first server run, the bootstrap script generates secrets and prints the web admin password plus collector token once; save those privately. + +Server, from your Mac: + +```bash +ssh root@149.56.14.95 'curl -fsSL https://raw.githubusercontent.com/zifanzhou1024/gamma-scope/codex/amh-nginx-server-setup/ops/amh-nginx/bootstrap_gamma_server.sh | bash' +``` + +Local Moomoo collector, from your Mac repo checkout after Moomoo OpenD is running: + +```bash +cd /Users/sakura/WebstormProjects/gamma-scope/.worktrees/amh-nginx-server-setup && mkdir -p ops/amh-nginx && scp root@gamma.hiqjj.org:/opt/gammascope/ops/amh-nginx/gammascope.collector-client.env ops/amh-nginx/gammascope.collector-client.env && chmod 600 ops/amh-nginx/gammascope.collector-client.env && bash ops/amh-nginx/start_moomoo_collector_mac.sh +``` + +If this is a brand-new AMH vhost, paste the `gamma.hiqjj.org` URL rule blocks from `/opt/gammascope/ops/amh-nginx/README.md` into AMH's `gamma.conf` once, then reload: + +```bash +/usr/local/nginx-1.24/sbin/nginx -t && /usr/local/nginx-1.24/sbin/nginx -s reload +``` + +Fast public checks: + +```bash +curl -I https://gamma.hiqjj.org/ && curl -fsS https://gamma.hiqjj.org/api/spx/0dte/snapshot/latest +``` + ## Current Production Shape Use these values for the current deployment unless you are intentionally creating a new environment: @@ -52,6 +80,8 @@ ops/amh-nginx/README.md Condensed AMH runbook with paste ops/amh-nginx/docker-compose.amh.yml Server Compose stack ops/amh-nginx/gammascope.nginx.conf Full Nginx vhost template ops/amh-nginx/generate_secrets.py Env/secret generator +ops/amh-nginx/bootstrap_gamma_server.sh One-command server bootstrap +ops/amh-nginx/start_moomoo_collector_mac.sh One-command local collector starter ops/amh-nginx/gammascope.production.env.example Server env template ops/amh-nginx/gammascope.collector-client.env.example Local collector env template ``` diff --git a/ops/amh-nginx/bootstrap_gamma_server.sh b/ops/amh-nginx/bootstrap_gamma_server.sh new file mode 100755 index 0000000..75ddba7 --- /dev/null +++ b/ops/amh-nginx/bootstrap_gamma_server.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +DOMAIN="${GAMMASCOPE_DOMAIN:-gamma.hiqjj.org}" +REPO_URL="${GAMMASCOPE_REPO_URL:-https://github.com/zifanzhou1024/gamma-scope.git}" +BRANCH="${GAMMASCOPE_BRANCH:-codex/amh-nginx-server-setup}" +APP_DIR="${GAMMASCOPE_APP_DIR:-/opt/gammascope}" + +SERVER_ENV="ops/amh-nginx/gammascope.production.env" +COLLECTOR_ENV="ops/amh-nginx/gammascope.collector-client.env" +COMPOSE_FILE="ops/amh-nginx/docker-compose.amh.yml" + +log() { + printf '\n==> %s\n' "$*" +} + +die() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +require_root() { + if [ "$(id -u)" -ne 0 ]; then + die "run this script as root on the Debian VPS" + fi +} + +require_debian() { + . /etc/os-release + if [ "${ID:-}" != "debian" ]; then + die "this bootstrap is written for Debian; detected ID=${ID:-unknown}" + fi +} + +install_base_packages() { + log "Installing base packages" + apt-get update + apt-get install -y ca-certificates curl git openssl python3 +} + +install_docker_if_needed() { + if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then + log "Docker is already installed" + systemctl enable --now docker >/dev/null 2>&1 || true + return + fi + + log "Installing Docker from the Debian repository" + rm -f /etc/apt/sources.list.d/docker.list + rm -f /etc/apt/sources.list.d/docker.sources + + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc + chmod a+r /etc/apt/keyrings/docker.asc + + . /etc/os-release + cat > /etc/apt/sources.list.d/docker.sources </dev/null +} + +checkout_repo() { + log "Checking out $REPO_URL branch $BRANCH into $APP_DIR" + + if [ -d "$APP_DIR/.git" ]; then + cd "$APP_DIR" + git fetch origin + git switch "$BRANCH" + git pull --ff-only + return + fi + + if [ -e "$APP_DIR" ] && [ -n "$(find "$APP_DIR" -mindepth 1 -maxdepth 1 2>/dev/null)" ]; then + die "$APP_DIR exists but is not a Git checkout" + fi + + mkdir -p "$APP_DIR" + git clone --branch "$BRANCH" "$REPO_URL" "$APP_DIR" + cd "$APP_DIR" +} + +generate_env_files_if_needed() { + cd "$APP_DIR" + + if [ -f "$SERVER_ENV" ] && [ -f "$COLLECTOR_ENV" ]; then + log "Found $SERVER_ENV and $COLLECTOR_ENV; keeping existing secrets" + chmod 600 "$SERVER_ENV" "$COLLECTOR_ENV" + return + fi + + if [ -f "$SERVER_ENV" ] || [ -f "$COLLECTOR_ENV" ]; then + die "only one env file exists; refusing to generate mismatched secrets. Restore the missing env file or move both files aside." + fi + + log "Generating new server and collector env files" + python3 ops/amh-nginx/generate_secrets.py \ + --domain "$DOMAIN" \ + --server-output "$SERVER_ENV" \ + --collector-output "$COLLECTOR_ENV" +} + +start_compose_stack() { + cd "$APP_DIR" + + log "Building and starting GammaScope containers" + docker compose \ + --env-file "$SERVER_ENV" \ + -f "$COMPOSE_FILE" \ + up -d --build + + docker compose \ + --env-file "$SERVER_ENV" \ + -f "$COMPOSE_FILE" \ + ps +} + +run_smoke_tests() { + log "Running local server smoke tests" + curl -fsSI http://127.0.0.1:3000/ >/dev/null + curl -fsS http://127.0.0.1:8000/api/spx/0dte/replay/sessions >/dev/null +} + +print_next_steps() { + cat <&2 + exit 1 +} + +need_command() { + command -v "$1" >/dev/null 2>&1 || die "missing command: $1" +} + +need_command pnpm +need_command python3 +need_command screen + +cd "$ROOT_DIR" + +[ -f package.json ] || die "run from the GammaScope repo root or set GAMMASCOPE_LOCAL_REPO" +[ -f "$ENV_FILE" ] || die "missing collector env file: $ENV_FILE" + +if [ ! -x .venv/bin/python ]; then + python3 -m venv .venv +fi + +if [ ! -d node_modules ]; then + pnpm install +fi + +.venv/bin/python -m pip install -e "apps/api[dev]" moomoo-api pandas + +set -a +. "$ENV_FILE" +set +a + +: "${GAMMASCOPE_SERVER_API:?missing GAMMASCOPE_SERVER_API}" +: "${GAMMASCOPE_ADMIN_TOKEN:?missing GAMMASCOPE_ADMIN_TOKEN}" +: "${GAMMASCOPE_MOOMOO_HOST:?missing GAMMASCOPE_MOOMOO_HOST}" +: "${GAMMASCOPE_MOOMOO_PORT:?missing GAMMASCOPE_MOOMOO_PORT}" +: "${GAMMASCOPE_RUT_SPOT:?missing GAMMASCOPE_RUT_SPOT}" +: "${GAMMASCOPE_NDX_SPOT:?missing GAMMASCOPE_NDX_SPOT}" + +python3 - <<'PY' +import os +import socket + +host = os.environ["GAMMASCOPE_MOOMOO_HOST"] +port = int(os.environ["GAMMASCOPE_MOOMOO_PORT"]) + +sock = socket.socket() +sock.settimeout(2) +try: + sock.connect((host, port)) +except OSError as exc: + raise SystemExit(f"Moomoo OpenD is not reachable at {host}:{port}: {exc}") +finally: + sock.close() + +print(f"moomoo-opend={host}:{port} reachable") +PY + +mkdir -p "$(dirname "$LOG_FILE")" + +screen -S "$SESSION_NAME" -X quit >/dev/null 2>&1 || true + +screen -dmS "$SESSION_NAME" bash -lc " + cd $(printf '%q' "$ROOT_DIR") || exit 1 + set -a + . $(printf '%q' "$ENV_FILE") + set +a + SSL_CERT_FILE=\"\${SSL_CERT_FILE:-/etc/ssl/cert.pem}\" pnpm collector:moomoo-snapshot \ + --host \"\$GAMMASCOPE_MOOMOO_HOST\" \ + --port \"\$GAMMASCOPE_MOOMOO_PORT\" \ + --api \"\$GAMMASCOPE_SERVER_API\" \ + --spot RUT=\"\$GAMMASCOPE_RUT_SPOT\" \ + --spot NDX=\"\$GAMMASCOPE_NDX_SPOT\" \ + --publish 2>&1 | tee -a $(printf '%q' "$LOG_FILE") +" + +printf 'collector screen session started: %s\n' "$SESSION_NAME" +printf 'log file: %s\n' "$LOG_FILE" +printf 'attach with: screen -r %s\n' "$SESSION_NAME" diff --git a/tests/deployment-doc.test.mjs b/tests/deployment-doc.test.mjs index d3080e0..30c46cb 100644 --- a/tests/deployment-doc.test.mjs +++ b/tests/deployment-doc.test.mjs @@ -3,6 +3,14 @@ import { readFileSync } from "node:fs"; import test from "node:test"; const deploymentDoc = readFileSync(new URL("../docs/deployment.md", import.meta.url), "utf8"); +const bootstrapScript = readFileSync( + new URL("../ops/amh-nginx/bootstrap_gamma_server.sh", import.meta.url), + "utf8", +); +const collectorScript = readFileSync( + new URL("../ops/amh-nginx/start_moomoo_collector_mac.sh", import.meta.url), + "utf8", +); test("deployment guide documents the working AMH production target", () => { assert.match(deploymentDoc, /gamma\.hiqjj\.org/); @@ -24,3 +32,33 @@ test("deployment guide includes realtime collector operations and smoke tests", assert.match(deploymentDoc, /moomoo-spx-0dte-live/); assert.match(deploymentDoc, /Do not add a broad `\/api\/ -> 127\.0\.0\.1:8000` rule/); }); + +test("deployment guide starts with a no-secret quick-start path", () => { + assert.ok( + deploymentDoc.indexOf("## Quick Start") < deploymentDoc.indexOf("## Current Production Shape"), + "quick start should appear before the detailed runbook", + ); + assert.match(deploymentDoc, /bootstrap_gamma_server\.sh/); + assert.match(deploymentDoc, /raw\.githubusercontent\.com\/zifanzhou1024\/gamma-scope/); + assert.match(deploymentDoc, /start_moomoo_collector_mac\.sh/); + assert.match(bootstrapScript, /generate_secrets\.py/); + assert.match(bootstrapScript, /docker compose/); + assert.match(bootstrapScript, /keeping existing secrets/); + assert.match(collectorScript, /screen -dmS "\$SESSION_NAME"/); + assert.match(collectorScript, /GAMMASCOPE_ADMIN_TOKEN/); +}); + +test("deployment quick start does not include concrete credentials", () => { + const combined = `${deploymentDoc}\n${bootstrapScript}\n${collectorScript}`; + const credentialPatterns = [ + /web admin password:\s+[A-Za-z0-9+/=_-]{12,}/i, + /collector admin token:\s+[A-Za-z0-9+/=_-]{12,}/i, + /root@149\.56\.14\.95's password:\s*\S+/i, + /password of\s+\S+/i, + /GAMMASCOPE_(?:ADMIN_TOKEN|WEB_ADMIN_PASSWORD|POSTGRES_PASSWORD|WEB_ADMIN_SESSION_SECRET)=[A-Za-z0-9+/=_-]{16,}/, + ]; + + for (const pattern of credentialPatterns) { + assert.doesNotMatch(combined, pattern); + } +}); From 1656377cc601b26ea2d574478810214b7adcfd51 Mon Sep 17 00:00:00 2001 From: Sakura <32687351+zifanzhou1024@users.noreply.github.com> Date: Sun, 3 May 2026 22:08:41 -0700 Subject: [PATCH 11/11] docs: point deployment quick start at main --- docs/amh-nginx-server-setup.md | 2 +- docs/deployment.md | 10 +++++----- ops/amh-nginx/README.md | 2 +- ops/amh-nginx/bootstrap_gamma_server.sh | 2 +- tests/deployment-doc.test.mjs | 6 ++++-- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/amh-nginx-server-setup.md b/docs/amh-nginx-server-setup.md index a1bb045..85115a5 100644 --- a/docs/amh-nginx-server-setup.md +++ b/docs/amh-nginx-server-setup.md @@ -132,7 +132,7 @@ If this branch has been pushed to GitHub: ```bash git clone . git fetch origin -git switch codex/amh-nginx-server-setup +git switch main ``` If the branch has not been pushed yet, send it from your computer: diff --git a/docs/deployment.md b/docs/deployment.md index 81be0b4..14cceff 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -11,7 +11,7 @@ Use this when redeploying the same `gamma.hiqjj.org` shape. These commands do no Server, from your Mac: ```bash -ssh root@149.56.14.95 'curl -fsSL https://raw.githubusercontent.com/zifanzhou1024/gamma-scope/codex/amh-nginx-server-setup/ops/amh-nginx/bootstrap_gamma_server.sh | bash' +ssh root@149.56.14.95 'curl -fsSL https://raw.githubusercontent.com/zifanzhou1024/gamma-scope/main/ops/amh-nginx/bootstrap_gamma_server.sh | bash' ``` Local Moomoo collector, from your Mac repo checkout after Moomoo OpenD is running: @@ -41,7 +41,7 @@ Public domain: gamma.hiqjj.org Server SSH target: root@149.56.14.95 or root@gamma.hiqjj.org Server app path: /opt/gammascope GitHub repo: https://github.com/zifanzhou1024/gamma-scope.git -Deployment branch: codex/amh-nginx-server-setup +Deployment branch: main Server OS: Debian Public reverse proxy: AMH Nginx API container port: 127.0.0.1:8000 @@ -187,7 +187,7 @@ cd /opt/gammascope git clone https://github.com/zifanzhou1024/gamma-scope.git . git fetch origin -git switch codex/amh-nginx-server-setup +git switch main ``` Existing install: @@ -195,7 +195,7 @@ Existing install: ```bash cd /opt/gammascope git fetch origin -git switch codex/amh-nginx-server-setup +git switch main git pull --ff-only ``` @@ -678,7 +678,7 @@ ssh root@gamma.hiqjj.org cd /opt/gammascope git fetch origin -git switch codex/amh-nginx-server-setup +git switch main git pull --ff-only docker compose \ diff --git a/ops/amh-nginx/README.md b/ops/amh-nginx/README.md index f995384..c426a06 100644 --- a/ops/amh-nginx/README.md +++ b/ops/amh-nginx/README.md @@ -63,7 +63,7 @@ mkdir -p /opt/gammascope cd /opt/gammascope git clone https://github.com/zifanzhou1024/gamma-scope.git . -git switch codex/amh-nginx-server-setup +git switch main git pull ``` diff --git a/ops/amh-nginx/bootstrap_gamma_server.sh b/ops/amh-nginx/bootstrap_gamma_server.sh index 75ddba7..fd818dc 100755 --- a/ops/amh-nginx/bootstrap_gamma_server.sh +++ b/ops/amh-nginx/bootstrap_gamma_server.sh @@ -3,7 +3,7 @@ set -Eeuo pipefail DOMAIN="${GAMMASCOPE_DOMAIN:-gamma.hiqjj.org}" REPO_URL="${GAMMASCOPE_REPO_URL:-https://github.com/zifanzhou1024/gamma-scope.git}" -BRANCH="${GAMMASCOPE_BRANCH:-codex/amh-nginx-server-setup}" +BRANCH="${GAMMASCOPE_BRANCH:-main}" APP_DIR="${GAMMASCOPE_APP_DIR:-/opt/gammascope}" SERVER_ENV="ops/amh-nginx/gammascope.production.env" diff --git a/tests/deployment-doc.test.mjs b/tests/deployment-doc.test.mjs index 30c46cb..a715ac2 100644 --- a/tests/deployment-doc.test.mjs +++ b/tests/deployment-doc.test.mjs @@ -15,7 +15,7 @@ const collectorScript = readFileSync( test("deployment guide documents the working AMH production target", () => { assert.match(deploymentDoc, /gamma\.hiqjj\.org/); assert.match(deploymentDoc, /149\.56\.14\.95/); - assert.match(deploymentDoc, /codex\/amh-nginx-server-setup/); + assert.match(deploymentDoc, /Deployment branch:\s+main/); assert.match(deploymentDoc, /\/opt\/gammascope/); assert.match(deploymentDoc, /\/usr\/local\/nginx-1\.24\/sbin\/nginx/); }); @@ -39,9 +39,11 @@ test("deployment guide starts with a no-secret quick-start path", () => { "quick start should appear before the detailed runbook", ); assert.match(deploymentDoc, /bootstrap_gamma_server\.sh/); - assert.match(deploymentDoc, /raw\.githubusercontent\.com\/zifanzhou1024\/gamma-scope/); + assert.match(deploymentDoc, /raw\.githubusercontent\.com\/zifanzhou1024\/gamma-scope\/main\/ops\/amh-nginx\/bootstrap_gamma_server\.sh/); assert.match(deploymentDoc, /start_moomoo_collector_mac\.sh/); + assert.doesNotMatch(deploymentDoc, /raw\.githubusercontent\.com\/zifanzhou1024\/gamma-scope\/codex\/amh-nginx-server-setup/); assert.match(bootstrapScript, /generate_secrets\.py/); + assert.match(bootstrapScript, /BRANCH="\$\{GAMMASCOPE_BRANCH:-main\}"/); assert.match(bootstrapScript, /docker compose/); assert.match(bootstrapScript, /keeping existing secrets/); assert.match(collectorScript, /screen -dmS "\$SESSION_NAME"/);