diff --git a/.env.example b/.env.example index 602ff64d..c220582a 100644 --- a/.env.example +++ b/.env.example @@ -12,18 +12,7 @@ NETWORK= NETUID= LOG_ID_PREFIX= -# Pyth data backend. `hermes` (default) keeps the legacy -# benchmarks.pyth.network + hermes.pyth.network endpoints. `pro` switches -# the validator to https://pyth.dourolabs.app/v1/fixed_rate@200ms/history -# (public; `fixed_rate@200ms` is the one channel that satisfies every feed's -# min_channel, including stocks/metals/oil which 404 on `real_time`) -# and the miner's get_asset_price to Pyth Lazer -# https://pyth-lazer.dourolabs.app/v1/latest_price (requires PYTH_API_KEY). -PYTH_BACKEND=hermes -# Miner-only. Bearer token from pythdata.app -> Pyth Terminal, used by -# the miner's get_asset_price live-price fetch via Pyth Lazer when -# PYTH_BACKEND=pro. Validators do NOT need this: on `pro` they read the -# public history endpoint above, which takes no auth. +# Pyth API key for Lazer (Pro) backend. If set, the miner will use Pyth Lazer instead of Hermes for assets with a Lazer feed. PYTH_API_KEY= # Bigtable storage backend for miner predictions. Only required when the diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c63f028f..36697e06 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,8 +11,6 @@ on: jobs: ci: runs-on: ubuntu-latest - env: - PYTH_BACKEND: pro steps: - name: Checkout code uses: actions/checkout@v5 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b5f12967..87ef7d88 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,9 @@ repos: - repo: https://github.com/psf/black - rev: 25.1.0 + # Keep in sync with the black version resolved in uv.lock (and used by + # `uv run black` / CI). black formatting changes between releases, so a + # stale rev here reformats differently than CI and fights the pipeline. + rev: 26.5.1 hooks: - id: black language_version: python3.11 diff --git a/README.md b/README.md index 84e7df9f..db5abdf2 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ ## 🔭 1. Overview -> **TL;DR** — Miners submit ensembles of simulated price paths for a basket of crypto, equity, and commodity assets across two timeframes (`24h` and `1h`). Validators score each ensemble with CRPS on price changes over multiple time increments, take a rolling weighted average within a per-timeframe window (10 days for `24h`, 3 days for `1h`), and allocate emissions via softmax — split 50/50 between the two timeframes. Lower CRPS → more emissions. +> **TL;DR** — Miners submit ensembles of simulated price paths for a basket of crypto, equity, and commodity assets across two timeframes (`24h` and `1h`). Validators score each ensemble with CRPS on price changes over multiple time increments, take a rolling weighted average within a per-timeframe window (10 days for `24h`, 5 days for `1h`), and allocate emissions via softmax — equally split between 3 competitions: Crypto 1h, Crypto 24h, Commodities/Equities 24h. Lower CRPS → more emissions. ### 1.1. Introduction @@ -91,7 +91,7 @@ sequenceDiagram note left of Validator: asset='BTC', start_time='2025-02-10T14:59:00', time_increment=300, etc. Miner-->>Validator: Prediction results - note right of Miner: result: price prediction
[[{"time": "2025-02-10T14:59:00+00:00", "price": 97352.60591372}, ...], ...] + note right of Miner: result: price prediction
[, , [97352.605, ...], [97361.753, ...], ...] Validator->>Validator: Run "prediction results" validation function @@ -107,12 +107,13 @@ Miners are tasked with providing probabilistic forecasts of an asset's future pr The asset set has grown over time: -| Date | Change | -| ---------- | --------------------------------------------------------------------------------------------------- | -| Launch | BTC only on the `24h` competition. 100 simulated paths, 5-minute increments. | -| 2025-11-13 | Bumped to 1000 paths; added ETH, SOL, XAU to the `24h` competition. Synth begins moving toward HFT. | -| 2026-01 | Added tokenized equities SPYX, NVDAX, TSLAX, AAPLX, GOOGLX to the `24h` competition. | -| 2026-03 | Added XRP, HYPE, WTIOIL to the `24h` competition; added HYPE to the `1h` competition. | +| Date | Change | +| ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Launch | BTC only on the `24h` competition. 100 simulated paths, 5-minute increments. | +| 2025-11-13 | Bumped to 1000 paths; added ETH, SOL, XAU to the `24h` competition. Synth begins moving toward HFT. | +| 2026-01 | Added tokenized equities SPYX, NVDAX, TSLAX, AAPLX, GOOGLX to the `24h` competition. | +| 2026-03 | Added XRP, HYPE, WTIOIL to the `24h` competition; added HYPE to the `1h` competition. | +| 2026-06 | Changed the split of the competitions from 2 (1h/24h) to 3 (Crypto 1h, Crypto 24h, Commodities/Equities 24h). Added XRP to the 1h time length, removed XAU from the 1h time length, added SPCX to the 24h time length | Whereas other subnets ask miners to predict single values for future prices, we’re interested in the miners correctly quantifying uncertainty. We want their price paths to represent their view of the probability distribution of the future price, and we want their paths to encapsulate realistic price dynamics, such as volatility clustering and skewed fat tailed price change distributions. As the network matures, modelling the correlations between asset prices will be essential. @@ -121,36 +122,25 @@ If the miners do a good job, the Synth Subnet will become the world-leading sour The checking prompts sent to the miners will have the format: (start_time, asset, time_increment, time_horizon, num_simulations) -The two competitions differ on the parameters below: - -| Parameter | `24h` competition | `1h` HFT competition | -| ------------------------------ | ------------------------------------------------------------------------ | ------------------------ | -| Emissions share | 50% | 50% | -| Cycle period (all assets) | ~60 min | ~10 min | -| Start time ($t_0$) | +1 min from request | +1 min from request | -| Time increment ($\Delta t$) | 5 min | 1 min | -| Time horizon ($T$) | 24 h | 1 h | -| Simulations ($N_{\text{sim}}$) | 1000 | 1000 | -| Assets | BTC, ETH, XAU, SOL, SPYX, NVDAX, TSLAX, AAPLX, GOOGLX, XRP, HYPE, WTIOIL | BTC, ETH, SOL, XAU, HYPE | -| Rolling-average window | 10 days | 3 days | -| Softmax temperature ($\beta$) | 0.1 | 0.2 (sharper allocation) | - -**Asset Weights** - -| Asset | Weight | Competitions | -| ------ | ------------------ | ------------ | -| BTC | 1.0 | `24h`, `1h` | -| ETH | 0.7064366394033871 | `24h`, `1h` | -| XAU | 1.7370922597118699 | `24h`, `1h` | -| SOL | 0.6310037175639559 | `24h`, `1h` | -| SPYX | 3.437935601155441 | `24h` | -| NVDAX | 1.6028217601617174 | `24h` | -| TSLAX | 1.6068755936957768 | `24h` | -| AAPLX | 2.0916380815843123 | `24h` | -| GOOGLX | 1.6827392777257926 | `24h` | -| XRP | 0.5658394110809131 | `24h` | -| HYPE | 0.4784547133706857 | `24h`, `1h` | -| WTIOIL | 0.8475062847978935 | `24h` | +The three competitions differ on the parameters below: + +| Parameter | Crypto 1h | Crypto 24h | Commodities/Equities 24h | +| ------------------------------ | ------------------------ | ------------------------ | ---------------------------------------------------- | +| Emissions share | 1/3 | 1/3 | 1/3 | +| Cycle period (all assets) | ~15 min | ~15 min | ~32 min | +| Start time ($t_0$) | +1 min from request | +1 min from request | +1 min from request | +| Time increment ($\Delta t$) | 1 min | 5 min | 5 min | +| Time horizon ($T$) | 1 h | 24 h | 24 h | +| Simulations ($N_{\text{sim}}$) | 1000 | 1000 | 1000 | +| Assets | BTC, ETH, SOL, XRP, HYPE | BTC, ETH, SOL, XRP, HYPE | XAU, SPYX, NVDAX, GOOGLX, TSLAX, AAPLX, WTIOIL, SPCX | +| Rolling-average window | 5 days | 10 days | 10 days | +| Softmax temperature ($\beta$) | 0.3 (sharper allocation) | 0.15 | 0.15 | + +The validator requests are sent to miner following this schedule: + +| Cycle | Low frequency (LF) | High frequency (HF) | +| ------ | ------------------------------------------------------------------------------ | ------------------------ | +| Assets | BTC, ETH, SOL, XRP, HYPE, XAU, SPYX, NVDAX, GOOGLX, TSLAX, AAPLX, WTIOIL, SPCX | BTC, ETH, SOL, XRP, HYPE | Validators cycle through the assets, sending out prediction requests at regular intervals. The miner has until the start time to return ($N_{\text{sim}}$) paths, each containing price predictions at times given by: @@ -166,6 +156,24 @@ We recommend the miner sends a request to the Pyth Oracle to acquire the price o If they fail to return predictions by the start_time or the predictions are in the wrong format, the submission is marked invalid and assigned the 90th-percentile score during the per-prompt CRPS transformation (see [§1.4](#14-calculation-of-leaderboard-score)). +The assets and their weights for the rolling average are as follows: + +| Asset | Weight | +| ------ | ------------------ | +| BTC | 1.0 | +| ETH | 0.7064366394033871 | +| XAU | 1.7370922597118699 | +| SOL | 0.6310037175639559 | +| SPYX | 3.437935601155441 | +| NVDAX | 1.6028217601617174 | +| TSLAX | 1.6068755936957768 | +| AAPLX | 2.0916380815843123 | +| GOOGLX | 1.6827392777257926 | +| XRP | 0.5658394110809131 | +| HYPE | 0.4784547133706857 | +| WTIOIL | 0.8475062847978935 | +| SPCX | 1.6068755936957768 | + [Back to top ^][table-of-contents] ### 1.3. Validator's Scoring Methodology @@ -195,7 +203,7 @@ The CRPS values are calculated on the price change in basis points for each inte To comprehensively assess the miners' forecasts, the CRPS is applied to sets of price changes in basis points over different time increments. The exact intervals depend on the prompt type: -- **`24h` prompts**: 5 minutes, 30 minutes, 3 hours, 24 hours. +- **`24h` LF prompts**: 5 minutes, 30 minutes, 3 hours, 24 hours. - **`1h` HFT prompts**: 1, 2, 5, 15, 30, and 60 minutes, plus a "gaps from start" series measured at every 5-minute offset between 5 and 60 minutes (i.e. price change from $t_0$ to $t_0 + 5\text{min}$, $t_0 + 10\text{min}$, …, $t_0 + 60\text{min}$). For each time increment: @@ -218,24 +226,16 @@ sequenceDiagram participant Bittensor loop Continuously - par 24h competition + loop 3 Competitions Validator->>Storage: Get matured predictions (≥24h old) Validator->>PricesProvider: Get real prices Validator->>Validator: CRPS sum, cap worst 10% / invalid at 90th pct, subtract best Validator->>Storage: Save per-request scores - Validator->>Storage: Get scores in 10-day window - Validator->>Validator: Weighted rolling average → softmax (β=0.1) → w_24h(i) - Validator->>Storage: Save w_24h - and 1h competition - Validator->>Storage: Get matured predictions (≥1h old) - Validator->>PricesProvider: Get real prices - Validator->>Validator: CRPS sum, cap worst 10% / invalid at 90th pct, subtract best - Validator->>Storage: Save per-request scores - Validator->>Storage: Get scores in 3-day window - Validator->>Validator: Weighted rolling average → softmax (β=0.2) → w_1h(i) - Validator->>Storage: Save w_1h + Validator->>Storage: Get scores in the rolling window + Validator->>Validator: Weighted rolling average → softmax → weighted_score + Validator->>Storage: Save weighted_score end - Note over Validator: Combine: w(i) = 0.5·w_24h(i) + 0.5·w_1h(i) + Note over Validator: Combine: w(i) = 1/3·w_1h_crypto(i) + 1/3·w_24h_crypto(i) + 1/3·w_24h_com_equ(i) Validator->>Bittensor: Send combined weights w(i) end ``` @@ -252,7 +252,7 @@ After calculating the sum of the CRPS values, the validator transforms the resul #### Rolling Average (Leaderboard Score) -The validator is required to store the historic request scores (as calculated in the previous step) for each miner. After each new request is scored, the validator recalculates the ‘leaderboard score’ for each miner, using a rolling average over their past **per-request** scores within a per-timeframe window (10 days for the `24h` competition, 3 days for the `1h` HFT competition), weighted by asset-specific weights. The `1h` competition runs ~6× more cycles per day than `24h` (every ~10 min vs every ~60 min), so it accumulates per-miner samples much faster — hence the shorter window. +The validator is required to store the historic request scores (as calculated in the previous step) for each miner. After each new request is scored, the validator recalculates the ‘leaderboard score’ for each miner, using a rolling average over their past **per-request** scores within a per-timeframe window, weighted by asset-specific weights. The `1h` prompts runs ~6× more cycles per day than `24h`, so it accumulates per-miner samples much faster — hence the shorter window. This approach emphasizes recent performance while still accounting for historical scores. The leaderboard score for miner $i$ at time $t$ is calculated as: @@ -265,7 +265,7 @@ where: - $S_{i,j}$ is the score of miner $i$ at request $j$. - $w_{k,j}$ is the weight given to asset $k$ scored at request $j$. -- The sum runs over all requests $j$ such that $t - t_j \leq T$, where $T$ is the per-timeframe rolling window size (10 days for `24h`, 3 days for `1h`). +- The sum runs over all requests $j$ such that $t - t_j \leq T$, where $T$ is the per-timeframe rolling window size. Thus, highest-ranking miners are those with the lowest calculated scores. @@ -280,10 +280,10 @@ $$ where: - $L_i(t)$ is in basis points (inherited from the CRPS units in §1.3). -- $\beta$ is a per-competition softmax temperature: `0.1` for the `24h` competition and `0.2` for the `1h` HFT competition (so the `1h` competition allocates emissions more sharply across miners). +- $\beta$ is a per-competition softmax temperature (see §1.2). - $E(t)$ is the total emission at time $t$. -The two competitions are scored independently. Their softmax weights are then each scaled by 50% (the emissions split shown in §1.2) and summed for miners that placed on both. +The three competitions are scored independently. Their softmax weights are then each scaled by one third (the emissions split shown in §1.2) and summed for miners that placed on multiple competitions. [Back to top ^][table-of-contents] @@ -296,10 +296,11 @@ The fastest way to validate your environment and the reference miner locally: ```shell git clone https://github.com/synthdataco/synth-subnet.git cd synth-subnet -curl -LsSf https://astral.sh/uv/install.sh | sh # install uv, if you don't have it -source $HOME/.local/bin/env # put uv on PATH in the current shell +curl -LsSf https://astral.sh/uv/install.sh | sh # install uv, if you don't have it +source $HOME/.local/bin/env # put uv on PATH in the current shell +export PYTHONPATH=. uv sync && source .venv/bin/activate -python synth/miner/run.py # prints "CORRECT" if the dummy model's output format is valid +python synth/miner/run.py # prints "CORRECT" if the dummy model's output format is valid ``` From there, follow the [miner tutorial](./docs/miner_tutorial.md) to plug in your own model, register a Bittensor wallet, and launch the miner under PM2. Validators should jump straight to the [validator guide](./docs/validator_guide.md). diff --git a/docs/validator_guide.md b/docs/validator_guide.md index ce5b9477..ed2a9bb1 100644 --- a/docs/validator_guide.md +++ b/docs/validator_guide.md @@ -9,10 +9,6 @@ - [5. Options](#5-options) - [5.1. Common Options](#51-common-options) - [`--axon.port INTEGER`](#--axonport-integer) - - [`--sma.low.days INTEGER`](#--smalowdays-integer) - - [`--sma.high.days INTEGER`](#--smahighdays-integer) - - [`--softmax.low.beta FLOAT`](#--softmaxlowbeta-float) - - [`--softmax.high.beta FLOAT`](#--softmaxhighbeta-float) - [`--cycle_interval_minutes.low INTEGER`](#--cycle_interval_minuteslow-integer) - [`--cycle_interval_minutes.high INTEGER`](#--cycle_interval_minuteshigh-integer) - [`--logging.debug`](#--loggingdebug) @@ -287,138 +283,6 @@ pm2 start validator.test.config.js -- --axon.port 8091 [Back to top ^][table-of-contents] -#### `--sma.low.days INTEGER` - -The window for the simple moving average (SMA) of the validator scores for the 24h prompt. - -Default: `10` - -Example: - -```js -// validator.config.js -module.exports = { - apps: [ - { - name: "validator", - interpreter: "python3", - script: "./neurons/validator.py", - args: "--sma.low.days 10", - env: { - PYTHONPATH: ".", - }, - }, - ], -}; -``` - -Alternatively, you can add the args directly to the command: - -```shell -pm2 start validator.config.js -- --sma.low.days 10 -``` - -[Back to top ^][table-of-contents] - -#### `--sma.high.days INTEGER` - -The window for the simple moving average (SMA) of the validator scores for the 1h prompt. - -Default: `3` - -Example: - -```js -// validator.config.js -module.exports = { - apps: [ - { - name: "validator", - interpreter: "python3", - script: "./neurons/validator.py", - args: "--sma.high.days 3", - env: { - PYTHONPATH: ".", - }, - }, - ], -}; -``` - -Alternatively, you can add the args directly to the command: - -```shell -pm2 start validator.config.js -- --sma.high.days 3 -``` - -[Back to top ^][table-of-contents] - -#### `--softmax.low.beta FLOAT` - -Negative beta to give higher weight to lower scores for the 24h prompt - -Default: `-0.1` - -Example: - -```js -// validator.config.js -module.exports = { - apps: [ - { - name: "validator", - interpreter: "python3", - script: "./neurons/validator.py", - args: "--softmax.low.beta -0.1", - env: { - PYTHONPATH: ".", - }, - }, - ], -}; -``` - -Alternatively, you can add the args directly to the command: - -```shell -pm2 start validator.config.js -- --softmax.low.beta -0.1 -``` - -[Back to top ^][table-of-contents] - -#### `--softmax.high.beta FLOAT` - -Negative beta to give higher weight to lower scores for the 1h prompt - -Default: `-0.2` - -Example: - -```js -// validator.config.js -module.exports = { - apps: [ - { - name: "validator", - interpreter: "python3", - script: "./neurons/validator.py", - args: "--softmax.high.beta -0.2", - env: { - PYTHONPATH: ".", - }, - }, - ], -}; -``` - -Alternatively, you can add the args directly to the command: - -```shell -pm2 start validator.config.js -- --softmax.high.beta -0.2 -``` - -[Back to top ^][table-of-contents] - #### `--cycle_interval_minutes.low INTEGER` Cycle interval in minutes for the 24h prompt. diff --git a/entrypoint-validator.sh b/entrypoint-validator.sh index eede9aaf..6ad34b94 100644 --- a/entrypoint-validator.sh +++ b/entrypoint-validator.sh @@ -6,14 +6,6 @@ mainnet_netuid=50 netuid=${NETUID:-$mainnet_netuid} vpermit_tao_limit=999999 -default_softmax_beta_low=-0.4 -softmax_beta_low="${SOFTMAX_BETA_LOW:-$default_softmax_beta_low}" -default_softmax_beta_high=-0.4 -softmax_beta_high="${SOFTMAX_BETA_HIGH:-$default_softmax_beta_high}" -default_sma_days_low=10 -sma_days_low="${SMA_DAYS_LOW:-$default_sma_days_low}" -default_sma_days_high=5 -sma_days_high="${SMA_DAYS_HIGH:-$default_sma_days_high}" default_validator_coldkey_name=validator validator_coldkey_name="${VALIDATOR_COLDKEY_NAME:-$default_validator_coldkey_name}" @@ -31,10 +23,6 @@ python3.11 ./neurons/validator.py \ --netuid $netuid \ --logging.debug \ --neuron.axon_off true \ - --sma.low.days $sma_days_low \ - --sma.high.days $sma_days_high \ - --softmax.low.beta $softmax_beta_low \ - --softmax.high.beta $softmax_beta_high \ --neuron.vpermit_tao_limit $vpermit_tao_limit \ --gcp.log_id_prefix $log_id_prefix \ --neuron.nprocs 8 \ @@ -48,10 +36,6 @@ python3.11 ./neurons/validator.py \ --netuid $netuid \ --logging.debug \ --neuron.axon_off true \ - --sma.low.days $sma_days_low \ - --sma.high.days $sma_days_high \ - --softmax.low.beta $softmax_beta_low \ - --softmax.high.beta $softmax_beta_high \ --neuron.vpermit_tao_limit $vpermit_tao_limit \ --gcp.log_id_prefix $log_id_prefix \ --neuron.nprocs 8 \ @@ -65,10 +49,6 @@ python3.11 ./neurons/validator.py \ --netuid $netuid \ --logging.debug \ --neuron.axon_off true \ - --sma.low.days $sma_days_low \ - --sma.high.days $sma_days_high \ - --softmax.low.beta $softmax_beta_low \ - --softmax.high.beta $softmax_beta_high \ --neuron.vpermit_tao_limit $vpermit_tao_limit \ --gcp.log_id_prefix $log_id_prefix \ --neuron.nprocs 8 \ diff --git a/miner.config.js b/miner.config.js index afef477d..fb2486ae 100644 --- a/miner.config.js +++ b/miner.config.js @@ -4,7 +4,7 @@ module.exports = { name: "miner", interpreter: "python3", script: "./neurons/miner.py", - args: "--netuid 50 --logging.debug --logging.trace --wallet.name miner --wallet.hotkey default --axon.port 8091 --blacklist.force_validator_permit true --blacklist.validator_min_stake 1000", + args: "--netuid 50 --logging.debug --logging.trace --wallet.name miner --wallet.hotkey default --axon.port 8091 --blacklist.force_validator_permit true --blacklist.validator_min_stake 65000", env: { PYTHONPATH: ".", }, diff --git a/neurons/validator.py b/neurons/validator.py index 57f7a38c..22b131fd 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -42,6 +42,7 @@ from synth.validator.miner_data_handler import MinerDataHandler from synth.validator.price_data_provider import PriceDataProvider from synth.validator.storage_backend import STORAGE_BACKEND_BIGTABLE +from synth.validator import competition_config from synth.validator.prompt_config import ( PromptConfig, LOW_FREQUENCY, @@ -89,10 +90,6 @@ def __init__(self, config=None, cycle_name: str = CYCLE_LOW_FREQUENCY): self.price_data_provider = PriceDataProvider() self.miner_uids: list[int] = [] - LOW_FREQUENCY.softmax_beta = self.config.softmax.low.beta - HIGH_FREQUENCY.softmax_beta = self.config.softmax.high.beta - LOW_FREQUENCY.window_days = self.config.sma.low.days - HIGH_FREQUENCY.window_days = self.config.sma.high.days LOW_FREQUENCY.data_retention_days = self.config.retention.low.days HIGH_FREQUENCY.data_retention_days = self.config.retention.high.days LOW_FREQUENCY.cycle_interval_minutes = ( @@ -246,36 +243,25 @@ def forward_score(self): # with predictions and calculate the rewards, # we store the rewards in the miner_scores table # ========================================== # - bt.logging.info( - f"forward score {LOW_FREQUENCY.label} frequency", "forward_score" - ) current_time = get_current_time() scored_time: datetime = round_time_to_minutes(current_time) - success_low = calculate_scores( - self.miner_data_handler, - self.price_data_provider, - scored_time, - LOW_FREQUENCY, - self.config.neuron.nprocs, - ) + success_count = 0 + for comp in competition_config.ALL_COMPETITIONS: + bt.logging.info(f"forward score {comp.label}", "forward_score") - scored_time: datetime = round_time_to_minutes(current_time) - current_time = get_current_time() - bt.logging.info( - f"forward score {HIGH_FREQUENCY.label} frequency", "forward_score" - ) - success_high = calculate_scores( - self.miner_data_handler, - self.price_data_provider, - scored_time, - HIGH_FREQUENCY, - self.config.neuron.nprocs, - ) + if calculate_scores( + self.miner_data_handler, + self.price_data_provider, + scored_time, + comp, + self.config.neuron.nprocs, + ): + success_count += 1 self.cleanup_history() - if not success_low and not success_high: + if success_count == 0: return # ================= Step 4 ================= # diff --git a/synth/miner/price_simulation.py b/synth/miner/price_simulation.py index e813fc81..0ab9ff4a 100644 --- a/synth/miner/price_simulation.py +++ b/synth/miner/price_simulation.py @@ -12,12 +12,8 @@ ) # Hermes Pyth API documentation: https://hermes.pyth.network/docs/ -# Pyth Lazer (the Pro replacement for latest_price) docs: -# https://docs.pyth.network/price-feeds/pro/api/rest#post-v1latest_price -# Selected by env: PYTH_BACKEND=pro (default `hermes`). When `pro`, requests -# go to Lazer with a Bearer PYTH_API_KEY. WTIOIL has no working Lazer feed -# today so we route it through Hyperliquid (same coin `xyz:CL` the validator -# uses for WTIOIL history). +# Pyth Lazer https://docs.pyth.network/price-feeds/pro/api/rest#post-v1latest_price +# If env: PYTH_API_KEY= is set, the miner will use Pyth Lazer instead of Hermes for assets with a Lazer feed. TOKEN_MAP = { "BTC": "e62df6c8b4a85fe1a67db44dc12de5db330f7ac66b72dc658afedf0f4a415b43", @@ -35,10 +31,7 @@ } # Lazer u32 feed IDs sourced from https://pyth.dourolabs.app/v1/symbols by -# matching each asset's hermes_id. Re-discover with -# verify/pyth-lazer-listing.py. WTIOIL has no active Lazer feed (the closest -# replacement USOILSPOT is inactive); it is intentionally absent and routed -# to Hyperliquid in the `pro` branch instead. +# matching each asset's hermes_id. LAZER_FEED_ID_MAP: dict[str, int] = { "BTC": 1, "ETH": 2, @@ -53,11 +46,10 @@ "TSLAX": 1847, } -# Hyperliquid `coin` codes for assets that have no usable Pyth feed. Mirrors -# `PriceDataProvider.HYPERLIQUID_SYMBOL_MAP` on the validator side so the -# miner's spot price comes from the same source the validator scores against. +# Hyperliquid `coin` codes for assets that have no usable Pyth feed. HYPERLIQUID_ASSET_MAP = { "WTIOIL": "xyz:CL", + "SPCX": "xyz:SPCX", } # `fixed_rate@200ms` meets every feed's min_channel (crypto majors accept @@ -88,12 +80,7 @@ def _fetch_price_hermes(asset: str) -> float | None: return live_price -def _fetch_price_lazer(asset: str) -> float | None: - api_key = os.environ.get("PYTH_API_KEY") - if not api_key: - print("PYTH_API_KEY not set; required when PYTH_BACKEND=pro") - return None - +def _fetch_price_lazer(asset: str, api_key: str) -> float | None: payload = { "channel": LAZER_CHANNEL, "priceFeedIds": [LAZER_FEED_ID_MAP[asset]], @@ -155,12 +142,11 @@ def _fetch_price_hyperliquid(asset: str) -> float | None: reraise=True, ) def get_asset_price(asset="BTC") -> float | None: - backend = os.environ.get("PYTH_BACKEND", "hermes").lower() - if backend == "pro": - if asset in HYPERLIQUID_ASSET_MAP: - return _fetch_price_hyperliquid(asset) - if asset in LAZER_FEED_ID_MAP: - return _fetch_price_lazer(asset) + if asset in HYPERLIQUID_ASSET_MAP: + return _fetch_price_hyperliquid(asset) + api_key = os.environ.get("PYTH_API_KEY") + if asset in LAZER_FEED_ID_MAP and api_key: + return _fetch_price_lazer(asset, api_key) return _fetch_price_hermes(asset) diff --git a/synth/utils/config.py b/synth/utils/config.py index 51e8ad03..09cabcfc 100644 --- a/synth/utils/config.py +++ b/synth/utils/config.py @@ -254,20 +254,6 @@ def add_validator_args(_, parser: argparse.ArgumentParser): default="full", ) - parser.add_argument( - "--sma.low.days", - type=int, - help="Window for the simple moving average (SMA) of the validator scores for the low frequency prompt.", - default=10, - ) - - parser.add_argument( - "--sma.high.days", - type=int, - help="Window for the simple moving average (SMA) of the validator scores for the high frequency prompt.", - default=3, - ) - parser.add_argument( "--retention.low.days", type=int, @@ -282,20 +268,6 @@ def add_validator_args(_, parser: argparse.ArgumentParser): default=4, ) - parser.add_argument( - "--softmax.low.beta", - type=float, - help="Negative beta to give higher weight to lower scores for the low frequency prompt.", - default=-0.1, - ) - - parser.add_argument( - "--softmax.high.beta", - type=float, - help="Negative beta to give higher weight to lower scores for the high frequency prompt.", - default=-0.2, - ) - parser.add_argument( "--cycle_interval_minutes.low", type=int, diff --git a/synth/utils/logging.py b/synth/utils/logging.py index 6a4d616c..0cf23c5c 100644 --- a/synth/utils/logging.py +++ b/synth/utils/logging.py @@ -22,6 +22,7 @@ "sqlalchemy.orm.strategies.LazyLoader", "sqlalchemy.orm.relationships.RelationshipProperty", "sqlalchemy.orm.path_registry", + "sqlalchemy.pool.impl.QueuePool", "async_substrate_interface", "urllib3.connectionpool", ) diff --git a/synth/utils/sequential_scheduler.py b/synth/utils/sequential_scheduler.py index be565f82..60165372 100644 --- a/synth/utils/sequential_scheduler.py +++ b/synth/utils/sequential_scheduler.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta import time @@ -39,13 +39,6 @@ def run_cycle( prompt_config = self.prompt_config asset_list = prompt_config.asset_list - if get_current_time() <= datetime( - 2026, 4, 10, 14, 0, 0, tzinfo=timezone.utc - ): - if prompt_config.label == "low": - asset_list = prompt_config.asset_list[:9] - elif prompt_config.label == "high": - asset_list = prompt_config.asset_list[:4] delay = self.select_delay( cycle_start_time, diff --git a/synth/validator/competition_config.py b/synth/validator/competition_config.py new file mode 100644 index 00000000..5ba72e80 --- /dev/null +++ b/synth/validator/competition_config.py @@ -0,0 +1,105 @@ +from dataclasses import dataclass + +SMOOTHED_SCORE_COEFFICIENT = 1 / 3 + + +@dataclass +class CompetitionConfig: + asset_list: list[str] + label: str + time_length: int + time_increment: int + scoring_intervals: dict[str, int] # Define scoring intervals in seconds. + window_days: int + softmax_beta: float + + +COM_EQU_24H = CompetitionConfig( + asset_list=[ + "XAU", + "SPYX", + "NVDAX", + "GOOGLX", + "TSLAX", + "AAPLX", + "WTIOIL", + "SPCX", + ], + label="Commodities/Equities 24h", + time_length=86400, + time_increment=300, + scoring_intervals={ + "5min": 300, # 5 minutes + "30min": 1800, # 30 minutes + "3hour": 10800, # 3 hours + "24hour_abs": 86400, # 24 hours + }, + window_days=10, + softmax_beta=-0.15, +) + +CRYPTO_24H = CompetitionConfig( + asset_list=[ + "BTC", + "ETH", + "SOL", + "XRP", + "HYPE", + ], + label="Crypto 24h", + time_length=86400, + time_increment=300, + scoring_intervals={ + "5min": 300, # 5 minutes + "30min": 1800, # 30 minutes + "3hour": 10800, # 3 hours + "24hour_abs": 86400, # 24 hours + }, + window_days=10, + softmax_beta=-0.15, +) + +CRYPTO_1H = CompetitionConfig( + asset_list=[ + "BTC", + "ETH", + "SOL", + "XRP", + "HYPE", + ], + label="Crypto 1h", + time_length=3600, + time_increment=60, + scoring_intervals={ + "1min": 60, + "2min": 120, + "5min": 300, + "15min": 900, + "30min": 1800, + "60min_abs": 3600, + "0_5min_gaps": 300, + "0_10min_gaps": 600, + "0_15min_gaps": 900, + "0_20min_gaps": 1200, + "0_25min_gaps": 1500, + "0_30min_gaps": 1800, + "0_35min_gaps": 2100, + "0_40min_gaps": 2400, + "0_45min_gaps": 2700, + "0_50min_gaps": 3000, + "0_55min_gaps": 3300, + "0_60min_gaps": 3600, + }, + window_days=5, + softmax_beta=-0.3, +) + + +# Canonical list of competitions. Both the scoring cycle (neurons/validator.py) +# and the moving-average update (forward.py) iterate this, so a competition +# added/renamed/reordered here propagates to both — don't re-list elsewhere. +ALL_COMPETITIONS = [ + COM_EQU_24H, + CRYPTO_24H, + CRYPTO_1H, +] diff --git a/synth/validator/forward.py b/synth/validator/forward.py index 2535ac68..063beb53 100644 --- a/synth/validator/forward.py +++ b/synth/validator/forward.py @@ -37,7 +37,7 @@ ) from synth.utils.logging import print_execution_time from synth.utils.uids import check_uid_availability -from synth.validator import prompt_config +from synth.validator import competition_config from synth.validator.miner_data_handler import MinerDataHandler from synth.validator.moving_average import ( combine_moving_averages, @@ -92,14 +92,13 @@ def calculate_moving_average_and_update_rewards( miner_data_handler: MinerDataHandler, scored_time: datetime, ) -> list[dict]: - prompts = [prompt_config.LOW_FREQUENCY, prompt_config.HIGH_FREQUENCY] - moving_averages_data: dict[str, list[dict]] = {} - for prompt in prompts: + for comp in competition_config.ALL_COMPETITIONS: miner_scores_df = miner_data_handler.get_miner_scores( scored_time, - prompt.window_days, - prompt.time_length, + comp.window_days, + comp.time_length, + comp.asset_list, ) if miner_scores_df.empty: @@ -111,16 +110,16 @@ def calculate_moving_average_and_update_rewards( miner_data_handler, df, scored_time, - prompt, + comp, ) - print_rewards_df(moving_averages, prompt.label) + print_rewards_df(moving_averages, comp.label) if moving_averages is None or len(moving_averages) == 0: continue miner_data_handler.update_miner_rewards(moving_averages) - moving_averages_data[prompt.label] = moving_averages + moving_averages_data[comp.label] = moving_averages return combine_moving_averages(moving_averages_data) @@ -130,12 +129,12 @@ def calculate_scores( miner_data_handler: MinerDataHandler, price_data_provider: PriceDataProvider, scored_time: datetime, - prompt: prompt_config.PromptConfig, + comp: competition_config.CompetitionConfig, nprocs: int = 2, ) -> bool: # get latest prediction request from validator validator_requests = miner_data_handler.get_validator_requests_to_score( - scored_time, prompt.window_days, prompt.time_length + scored_time, comp.window_days, comp.time_length, comp.asset_list ) if validator_requests is None or len(validator_requests) == 0: @@ -152,6 +151,7 @@ def calculate_scores( miner_data_handler=miner_data_handler, price_data_provider=price_data_provider, validator_request=validator_request, + comp=comp, nprocs=nprocs, ) diff --git a/synth/validator/miner_data_handler.py b/synth/validator/miner_data_handler.py index c14f36c6..441ce93b 100644 --- a/synth/validator/miner_data_handler.py +++ b/synth/validator/miner_data_handler.py @@ -491,7 +491,8 @@ def get_validator_requests_to_score( self, scored_time: datetime, window_days: int, - time_length: int | None = None, + time_length: int, + asset_list: list[str], ) -> typing.Optional[list[ValidatorRequest]]: """ Retrieve the list of IDs of the latest validator requests that (start_time + time_length) < scored_time @@ -499,9 +500,6 @@ def get_validator_requests_to_score( This is to ensure that we only get requests that are within the window_days. and exclude records that are already scored """ - if time_length is None: - time_length = prompt_config.LOW_FREQUENCY.time_length - try: with self.engine.connect() as connection: subq = ( @@ -578,6 +576,7 @@ def get_validator_requests_to_score( # Skip thinned-empty requests. exists(alive_subq), ValidatorRequest.time_length == time_length, + ValidatorRequest.asset.in_(asset_list), ) ) .order_by(ValidatorRequest.start_time.asc()) @@ -658,11 +657,10 @@ def get_miner_scores( self, scored_time: datetime, window_days: int, - time_length: int | None = None, + time_length: int, + asset_list: list[str], ): min_scored_time = scored_time - timedelta(days=window_days) - if time_length is None: - time_length = prompt_config.LOW_FREQUENCY.time_length try: with self.engine.connect() as connection: @@ -681,6 +679,7 @@ def get_miner_scores( JOIN validator_requests vr ON vr.id = mp.validator_requests_id WHERE ms.scored_time > :min_scored_time AND vr.time_length = :time_length + AND vr.asset = ANY(:asset_list) """) result = connection.execute( @@ -688,6 +687,7 @@ def get_miner_scores( { "min_scored_time": min_scored_time, "time_length": time_length, + "asset_list": asset_list, }, ) diff --git a/synth/validator/moving_average.py b/synth/validator/moving_average.py index 99d9f706..869c1a99 100644 --- a/synth/validator/moving_average.py +++ b/synth/validator/moving_average.py @@ -10,7 +10,8 @@ from synth.utils.logging import print_execution_time from synth.validator.miner_data_handler import MinerDataHandler -from synth.validator.prompt_config import PromptConfig +from synth.validator.competition_config import CompetitionConfig +from synth.validator.competition_config import SMOOTHED_SCORE_COEFFICIENT from synth.validator.reward import compute_softmax # Per-asset weighting coefficients for score normalization across assets. @@ -28,6 +29,7 @@ "XRP": 0.5658394110809131, "HYPE": 0.4784547133706857, "WTIOIL": 0.8475062847978935, + "SPCX": 1.6068755936957768, } @@ -108,7 +110,7 @@ def compute_smoothed_score( miner_data_handler: MinerDataHandler, input_df: DataFrame, scored_time: datetime, - prompt_config: PromptConfig, + comp: CompetitionConfig, ) -> typing.Optional[list[dict]]: """Compute smoothed scores and reward weights for all miners. @@ -132,7 +134,7 @@ def compute_smoothed_score( # Apply per-asset coefficients vectorized (single pass, no per-miner loop). # Normalization is per-miner: each miner's scores are divided by the sum of - # coefficients for that miner's assets, matching the original per-miner loop behavior. + # coefficients for that miner's assets. coefs = df["asset"].map(ASSET_COEFFICIENTS).fillna(1.0) df["weighted_score"] = df["prompt_score_v3"] * coefs miner_coef_sums = coefs.groupby(df["miner_id"]).sum() @@ -173,7 +175,7 @@ def compute_smoothed_score( r["rolling_avg"] for r in filtered_moving_averages_data ] reward_weight_list = compute_softmax( - np.array(rolling_avg_list), prompt_config.softmax_beta + np.array(rolling_avg_list), comp.softmax_beta ) rewards = [] @@ -187,9 +189,9 @@ def compute_smoothed_score( "miner_uid": item["miner_uid"], "smoothed_score": item["rolling_avg"], "reward_weight": float(reward_weight) - * prompt_config.smoothed_score_coefficient, + * SMOOTHED_SCORE_COEFFICIENT, "updated_at": scored_time.isoformat(), - "prompt_name": prompt_config.label, + "prompt_name": comp.label, } ) @@ -206,9 +208,9 @@ def print_rewards_df(moving_averages_data: list[dict], label: str = ""): def combine_moving_averages( moving_averages_data: dict[str, list[dict]], ) -> list[dict]: - """Combine reward weights from low-frequency and high-frequency competitions. + """Combine reward weights from competitions. - Same miner appearing in both gets their weights summed. + Same miner appearing in multiple competitions gets their weights summed. """ map_miner_reward: dict[int, dict] = {} diff --git a/synth/validator/price_data_provider.py b/synth/validator/price_data_provider.py index 74297df3..ddef12dd 100644 --- a/synth/validator/price_data_provider.py +++ b/synth/validator/price_data_provider.py @@ -1,5 +1,4 @@ import logging -import os import time import requests @@ -17,20 +16,8 @@ from synth.utils.helpers import from_iso_to_unix_time from synth.utils.logging import print_execution_time -# Pyth API benchmarks doc: https://benchmarks.pyth.network/docs -# get the list of stocks supported by pyth: https://benchmarks.pyth.network/v1/shims/tradingview/symbol_info?group=pyth_stock -# get the list of crypto supported by pyth: https://benchmarks.pyth.network/v1/shims/tradingview/symbol_info?group=pyth_crypto -# get the ticker: https://benchmarks.pyth.network/v1/shims/tradingview/symbols?symbol=Crypto.XAUT/USD -# -# Pyth Pro Router (new) returns the same TradingView shape from -# https://pyth.dourolabs.app/v1/real_time/history with the same `Crypto.*/USD` -# symbol strings — public, no auth required. Selected by PYTH_BACKEND=pro. - class PriceDataProvider: - PYTH_BENCHMARKS_URL = ( - "https://benchmarks.pyth.network/v1/shims/tradingview/history" - ) # `fixed_rate@200ms` is the channel that meets every feed's min_channel: # stocks/metals/oil reject `real_time` with 404, but accept this. Crypto # majors accept it too. One channel, every feed. @@ -64,6 +51,7 @@ class PriceDataProvider: "GOOGLX": "Crypto.GOOGLX/USD", "XRP": "Crypto.XRP/USD", "HYPE": "Crypto.HYPE/USD", + "SPCX": "Pyth.HL.SPCX/USDC", } # Assets fetched from Hyperliquid (overrides Pyth for these assets) @@ -71,16 +59,6 @@ class PriceDataProvider: "WTIOIL": "xyz:CL", } - def __init__(self): - # Resolve once at instance construction so we honour the .env loaded - # by neurons/validator.py before instantiation. PYTH_BACKEND=pro - # routes history through the new Pyth Pro Router; default `hermes` - # keeps the legacy Benchmarks endpoint for instant rollback. - backend = os.environ.get("PYTH_BACKEND", "hermes").lower() - self.pyth_history_url = ( - self.PYTH_PRO_URL if backend == "pro" else self.PYTH_BENCHMARKS_URL - ) - @staticmethod def assert_assets_supported(asset_list: list[str]): supported = ( @@ -125,7 +103,7 @@ def fetch_data(self, validator_request: ValidatorRequest) -> list: "to": last_grid_timestamp + self.CANDLE_INTERVAL_SECONDS, } - response = requests.get(self.pyth_history_url, params=params) + response = requests.get(self.PYTH_PRO_URL, params=params) response.raise_for_status() data = response.json() diff --git a/synth/validator/prompt_config.py b/synth/validator/prompt_config.py index 025fec5a..b9268c86 100644 --- a/synth/validator/prompt_config.py +++ b/synth/validator/prompt_config.py @@ -7,13 +7,8 @@ class PromptConfig: label: str time_length: int time_increment: int - initial_delay: int cycle_interval_minutes: int timeout_extra_seconds: int - scoring_intervals: dict[str, int] # Define scoring intervals in seconds. - window_days: int - softmax_beta: float - smoothed_score_coefficient: float num_simulations: int = 1000 data_retention_days: int = 30 # Density tapering: predictions on validator_requests whose start_time is @@ -28,68 +23,35 @@ class PromptConfig: asset_list=[ "BTC", "ETH", - "XAU", "SOL", + "XRP", + "HYPE", + "XAU", "SPYX", "NVDAX", + "GOOGLX", "TSLAX", "AAPLX", - "GOOGLX", - "XRP", - "HYPE", "WTIOIL", + "SPCX", ], label="low", time_length=86400, time_increment=300, - initial_delay=60, # avoid 2 prompts to start simultaneously cycle_interval_minutes=5, timeout_extra_seconds=60, - scoring_intervals={ - "5min": 300, # 5 minutes - "30min": 1800, # 30 minutes - "3hour": 10800, # 3 hours - "24hour_abs": 86400, # 24 hours - }, - window_days=10, - softmax_beta=-0.1, - smoothed_score_coefficient=0.5, data_retention_days=12, thin_after_minutes=30, thin_bucket_seconds=3600, ) HIGH_FREQUENCY = PromptConfig( - asset_list=["BTC", "ETH", "XAU", "SOL", "HYPE"], + asset_list=["BTC", "ETH", "SOL", "XRP", "HYPE"], label="high", time_length=3600, time_increment=60, - initial_delay=0, cycle_interval_minutes=2, timeout_extra_seconds=60, - scoring_intervals={ - "1min": 60, - "2min": 120, - "5min": 300, - "15min": 900, - "30min": 1800, - "60min_abs": 3600, - "0_5min_gaps": 300, - "0_10min_gaps": 600, - "0_15min_gaps": 900, - "0_20min_gaps": 1200, - "0_25min_gaps": 1500, - "0_30min_gaps": 1800, - "0_35min_gaps": 2100, - "0_40min_gaps": 2400, - "0_45min_gaps": 2700, - "0_50min_gaps": 3000, - "0_55min_gaps": 3300, - "0_60min_gaps": 3600, - }, - window_days=3, - softmax_beta=-0.2, - smoothed_score_coefficient=0.5, data_retention_days=4, thin_after_minutes=10, thin_bucket_seconds=600, diff --git a/synth/validator/reward.py b/synth/validator/reward.py index 5d6fcd7e..ff9ff4bc 100644 --- a/synth/validator/reward.py +++ b/synth/validator/reward.py @@ -33,7 +33,7 @@ from synth.validator.miner_data_handler import MinerDataHandler from synth.validator.price_data_provider import PriceDataProvider from synth.validator import response_validation_v2 -from synth.validator import prompt_config +from synth.validator import competition_config # Module level - must be picklable @@ -151,17 +151,6 @@ def get_process_executor(nprocs: int = 2) -> ProcessPoolExecutor: return _PROCESS_EXECUTOR -def _get_scoring_intervals(validator_request: ValidatorRequest) -> dict: - """Determine scoring intervals based on time length.""" - is_high_freq = ( - validator_request.time_length - == prompt_config.HIGH_FREQUENCY.time_length - ) - if is_high_freq: - return prompt_config.HIGH_FREQUENCY.scoring_intervals - return prompt_config.LOW_FREQUENCY.scoring_intervals - - def _prepare_work_items( predictions: list[MinerPrediction], shm_name: str, @@ -253,6 +242,7 @@ def get_rewards_multiprocess( miner_data_handler: MinerDataHandler, price_data_provider: PriceDataProvider, validator_request: ValidatorRequest, + comp: competition_config.CompetitionConfig, nprocs: int = 2, ) -> tuple[typing.Optional[np.ndarray], list, list[dict]]: """ @@ -263,6 +253,7 @@ def get_rewards_multiprocess( - miner_data_handler (MinerDataHandler): The handler for miner data. - price_data_provider (PriceDataProvider): The provider for price data. - validator_request (ValidatorRequest): The validator request object. + - comp (CompetitionConfig): The competition being scored; supplies the scoring intervals used for CRPS. - nprocs (int): Number of processes to use for parallel computation. Returns: @@ -302,13 +293,12 @@ def get_rewards_multiprocess( shared_prices[:] = prices_array[:] # Prepare work items - scoring_intervals = _get_scoring_intervals(validator_request) work_items = _prepare_work_items( predictions, shm.name, prices_array.shape, validator_request, - scoring_intervals, + comp.scoring_intervals, ) # Process in parallel (CPU bound - use ProcessPool) diff --git a/tests/conftest.py b/tests/conftest.py index 95c37425..cfff9ecf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,12 +5,6 @@ from sqlalchemy import create_engine from testcontainers.postgres import PostgresContainer -# The scoring tests fetch real prices live from Pyth. Default to the Pro -# router (public, no API key) like CI does — the legacy hermes/benchmarks -# endpoint rate-limits (429) under the suite's call volume. `setdefault` -# leaves an explicit `PYTH_BACKEND=hermes` override untouched. -os.environ.setdefault("PYTH_BACKEND", "pro") - postgres = PostgresContainer("postgres:16-alpine") diff --git a/tests/test_calculate_crps.py b/tests/test_calculate_crps.py index 79ed42f8..4ccf8dc8 100644 --- a/tests/test_calculate_crps.py +++ b/tests/test_calculate_crps.py @@ -2,7 +2,7 @@ import numpy as np -from synth.validator import prompt_config +from synth.validator import competition_config from synth.validator.crps_calculation import ( calculate_crps_for_miner, calculate_price_changes_over_intervals, @@ -21,7 +21,7 @@ def test_calculate_crps_for_miner_1(self): np.array(predictions_path), np.array(real_price_path), time_increment, - prompt_config.LOW_FREQUENCY.scoring_intervals, + competition_config.CRYPTO_24H.scoring_intervals, ) self.assertEqual(sum_all_scores, 284.1200564488584) @@ -34,7 +34,7 @@ def test_calculate_crps_for_miner_1_b(self): np.array(predictions_path), np.array(real_price_path), time_increment, - prompt_config.LOW_FREQUENCY.scoring_intervals, + competition_config.CRYPTO_24H.scoring_intervals, ) self.assertEqual(sum_all_scores, 284.1200564488584) @@ -47,7 +47,7 @@ def test_calculate_crps_for_miner_zero(self): np.array(predictions_path), np.array(real_price_path), time_increment, - prompt_config.LOW_FREQUENCY.scoring_intervals, + competition_config.CRYPTO_24H.scoring_intervals, ) self.assertEqual(sum_all_scores, 0) @@ -60,7 +60,7 @@ def test_calculate_crps_for_miner_2(self): np.array([predictions_path]), np.array(real_price_path), time_increment, - prompt_config.LOW_FREQUENCY.scoring_intervals, + competition_config.CRYPTO_24H.scoring_intervals, ) self.assertEqual(sum_all_scores, 479.6904902048716) @@ -74,7 +74,7 @@ def test_calculate_crps_for_miner_3(self): np.array([predictions_path]), np.array(real_price_path), time_increment, - prompt_config.LOW_FREQUENCY.scoring_intervals, + competition_config.CRYPTO_24H.scoring_intervals, ) self.assertEqual(sum_all_scores, 4737.272133130346) @@ -126,7 +126,7 @@ def test_calculate_crps_for_miner_4(self): np.array(predictions_path), np.array(real_price_path), time_increment, - prompt_config.LOW_FREQUENCY.scoring_intervals, + competition_config.CRYPTO_24H.scoring_intervals, ) self.assertEqual(sum_all_scores, 13413.599141058676) @@ -143,7 +143,7 @@ def test_calculate_crps_for_miner_5(self): np.array([predictions_path]), np.array(real_price_path), time_increment, - prompt_config.LOW_FREQUENCY.scoring_intervals, + competition_config.CRYPTO_24H.scoring_intervals, ) self.assertEqual(sum_all_scores, 0.0) @@ -160,7 +160,7 @@ def test_calculate_crps_for_miner_6(self): np.array([predictions_path]), np.array(real_price_path), time_increment, - prompt_config.LOW_FREQUENCY.scoring_intervals, + competition_config.CRYPTO_24H.scoring_intervals, ) self.assertEqual(sum_all_scores, 0.0) @@ -177,7 +177,7 @@ def test_calculate_crps_for_miner_7(self): np.array([predictions_path]), np.array(real_price_path), time_increment, - prompt_config.LOW_FREQUENCY.scoring_intervals, + competition_config.CRYPTO_24H.scoring_intervals, ) self.assertEqual(sum_all_scores, 0.0) @@ -204,7 +204,7 @@ def test_calculate_crps_for_miner_8(self): np.array([predictions_path]), np.array(real_price_path), time_increment, - prompt_config.LOW_FREQUENCY.scoring_intervals, + competition_config.CRYPTO_24H.scoring_intervals, ) self.assertEqual(sum_all_scores, 0.0) @@ -223,14 +223,14 @@ def test_calculate_crps_for_miner_9(self): np.array([predictions_path]), np.array(real_price_path), time_increment, - prompt_config.LOW_FREQUENCY.scoring_intervals, + competition_config.CRYPTO_24H.scoring_intervals, ) sum_all_scores_2, _ = calculate_crps_for_miner( np.array([predictions_path]), np.array(real_price_path_full), time_increment, - prompt_config.LOW_FREQUENCY.scoring_intervals, + competition_config.CRYPTO_24H.scoring_intervals, ) with self.subTest("Check sum_all_scores equals expected"): @@ -250,7 +250,7 @@ def test_calculate_crps_for_miner_10(self): np.array([predictions_path]), np.array(real_price_path), time_increment, - prompt_config.LOW_FREQUENCY.scoring_intervals, + competition_config.CRYPTO_24H.scoring_intervals, ) self.assertEqual(np.isnan(sum_all_scores), True) @@ -264,7 +264,7 @@ def test_calculate_crps_for_miner_11(self): np.array([predictions_path]), np.array(real_price_path), time_increment, - prompt_config.LOW_FREQUENCY.scoring_intervals, + competition_config.CRYPTO_24H.scoring_intervals, ) self.assertEqual(sum_all_scores, 1061.3650577065207) @@ -288,7 +288,7 @@ def test_calculate_crps_for_miner_12(self): np.array([predictions_path]).astype(float), np.array(real_price_path), time_increment, - prompt_config.LOW_FREQUENCY.scoring_intervals, + competition_config.CRYPTO_24H.scoring_intervals, ) self.assertEqual(sum_all_scores, 12697.728694070156) @@ -312,7 +312,7 @@ def test_calculate_crps_for_miner_13(self): np.array([predictions_path]).astype(float), np.array(real_price_path), time_increment, - prompt_config.LOW_FREQUENCY.scoring_intervals, + competition_config.CRYPTO_24H.scoring_intervals, ) self.assertEqual(sum_all_scores, -1) @@ -336,7 +336,7 @@ def test_calculate_crps_for_miner_gap_1(self): np.array([predictions_path]).astype(float), np.array(real_price_path), time_increment, - prompt_config.HIGH_FREQUENCY.scoring_intervals, + competition_config.CRYPTO_1H.scoring_intervals, ) self.assertEqual(sum_all_scores, -1) @@ -454,7 +454,7 @@ def test_gap_vs_regular_not_identical(self): self.assertNotEqual(regular.shape, gap.shape) def test_high_freq_gap_intervals_produce_different_scores(self): - """Full integration test: gap intervals in HIGH_FREQUENCY config should + """Full integration test: gap intervals in the Crypto 1h config should produce different CRPS than if they were treated as regular intervals. """ time_increment = 60 diff --git a/tests/test_forward.py b/tests/test_forward.py index 67230703..80fcd3b2 100644 --- a/tests/test_forward.py +++ b/tests/test_forward.py @@ -17,7 +17,7 @@ from synth.db.models import Miner, MinerReward from synth.validator.miner_data_handler import MinerDataHandler from synth.validator.price_data_provider import PriceDataProvider -from synth.validator import prompt_config +from synth.validator import competition_config from tests.utils import prepare_random_predictions, recent_start_time @@ -35,12 +35,17 @@ def test_calculate_rewards_and_update_scores(db_engine: Engine): miner_data_handler=handler, price_data_provider=price_data_provider, scored_time=scored_time, - prompt=prompt_config.LOW_FREQUENCY, + comp=competition_config.CRYPTO_24H, ) assert success - miner_scores_df = handler.get_miner_scores(scored_time, 10) + miner_scores_df = handler.get_miner_scores( + scored_time, + 10, + competition_config.CRYPTO_24H.time_length, + competition_config.CRYPTO_24H.asset_list, + ) assert len(miner_scores_df) == len(miner_uids) @@ -61,7 +66,7 @@ def test_calculate_moving_average_and_update_rewards(db_engine: Engine): miner_data_handler=handler, price_data_provider=price_data_provider, scored_time=scored_time, - prompt=prompt_config.LOW_FREQUENCY, + comp=competition_config.CRYPTO_24H, ) assert success @@ -74,7 +79,7 @@ def test_calculate_moving_average_and_update_rewards(db_engine: Engine): print("moving_averages_data", moving_averages_data) -# Pin the miner's live-price fetch (Pyth Lazer on PYTH_BACKEND=pro) so these +# Pin the miner's live-price fetch so these # tests don't depend on it. PriceDataProvider's history fetch stays live — it # hits the public Pyth Pro history endpoint and is part of what's scored. @patch("synth.miner.simulations.get_asset_price", return_value=90000.0) @@ -162,10 +167,15 @@ def test_calculate_moving_average_and_update_rewards_new_miner( miner_data_handler=handler, price_data_provider=price_data_provider, scored_time=scored_time, - prompt=prompt_config.LOW_FREQUENCY, + comp=competition_config.CRYPTO_24H, ) - miner_scores_df = handler.get_miner_scores(scored_time, 10) + miner_scores_df = handler.get_miner_scores( + scored_time, + 10, + competition_config.CRYPTO_24H.time_length, + competition_config.CRYPTO_24H.asset_list, + ) print("miner_scores_df", miner_scores_df) @@ -288,10 +298,15 @@ def test_calculate_moving_average_and_update_rewards_new_miner_registration( miner_data_handler=handler, price_data_provider=price_data_provider, scored_time=scored_time, - prompt=prompt_config.LOW_FREQUENCY, + comp=competition_config.CRYPTO_24H, ) - miner_scores_df = handler.get_miner_scores(scored_time, 10) + miner_scores_df = handler.get_miner_scores( + scored_time, + 10, + competition_config.CRYPTO_24H.time_length, + competition_config.CRYPTO_24H.asset_list, + ) print("miner_scores_df: ", miner_scores_df) @@ -407,10 +422,15 @@ def test_calculate_moving_average_and_update_rewards_only_invalid( miner_data_handler=handler, price_data_provider=price_data_provider, scored_time=scored_time, - prompt=prompt_config.LOW_FREQUENCY, + comp=competition_config.CRYPTO_24H, ) - miner_scores_df = handler.get_miner_scores(scored_time, 10) + miner_scores_df = handler.get_miner_scores( + scored_time, + 10, + competition_config.CRYPTO_24H.time_length, + competition_config.CRYPTO_24H.asset_list, + ) print("miner_scores_df", miner_scores_df) diff --git a/tests/test_miner_data_handler.py b/tests/test_miner_data_handler.py index 89dbd742..b8563821 100644 --- a/tests/test_miner_data_handler.py +++ b/tests/test_miner_data_handler.py @@ -12,6 +12,7 @@ Miner, ) from synth.validator import response_validation_v2 +from synth.validator import competition_config from synth.validator.prompt_config import PromptConfig from synth.simulation_input import SimulationInput from synth.validator.miner_data_handler import MinerDataHandler @@ -68,7 +69,7 @@ def test_get_values_within_range(db_engine: Engine): handler.save_responses(simulation_data, simulation_input, datetime.now()) validator_requests = handler.get_validator_requests_to_score( - scored_time, 7 + scored_time, 7, 86400, ["BTC"] ) assert validator_requests is not None assert len(validator_requests) == 1 @@ -127,7 +128,7 @@ def test_get_values_ongoing_range(db_engine: Engine): handler.save_responses(simulation_data, simulation_input, datetime.now()) validator_requests = handler.get_validator_requests_to_score( - scored_time, 7 + scored_time, 7, 86400, ["BTC"] ) assert validator_requests is not None @@ -196,7 +197,7 @@ def test_multiple_records_for_same_miner(db_engine: Engine): ) validator_requests = handler.get_validator_requests_to_score( - scored_time, 7 + scored_time, 7, 86400, ["BTC"] ) assert validator_requests is not None assert len(validator_requests) == 2 @@ -281,7 +282,7 @@ def test_multiple_records_for_same_miner_with_overlapping(db_engine: Engine): ) validator_requests = handler.get_validator_requests_to_score( - scored_time, 7 + scored_time, 7, 86400, ["BTC"] ) assert validator_requests is not None assert len(validator_requests) == 1 @@ -313,7 +314,7 @@ def test_no_data_for_miner(db_engine: Engine): handler = MinerDataHandler(db_engine) validator_requests = handler.get_validator_requests_to_score( - scored_time, 7 + scored_time, 7, 86400, ["BTC"] ) assert validator_requests is not None assert len(validator_requests) == 0 @@ -353,7 +354,7 @@ def test_get_values_incorrect_format(db_engine: Engine): handler.save_responses(simulation_data, simulation_input, datetime.now()) validator_requests = handler.get_validator_requests_to_score( - scored_time, 7 + scored_time, 7, 86400, ["BTC"] ) assert validator_requests is not None assert len(validator_requests) == 1 @@ -378,7 +379,7 @@ def test_set_get_scores(db_engine: Engine): handler, _, _ = prepare_random_predictions(db_engine, start_time) validator_requests = handler.get_validator_requests_to_score( - scored_time, 7 + scored_time, 7, 86400, ["BTC"] ) assert validator_requests is not None assert len(validator_requests) == 1 @@ -387,6 +388,7 @@ def test_set_get_scores(db_engine: Engine): handler, price_data_provider, validator_requests[0], + competition_config.CRYPTO_24H, ) assert prompt_scores is not None @@ -398,6 +400,8 @@ def test_set_get_scores(db_engine: Engine): miner_scores_df = handler.get_miner_scores( scored_time=scored_time, window_days=4, + time_length=86400, + asset_list=["BTC"], ) print("miner_scores_df", miner_scores_df) @@ -479,7 +483,7 @@ def test_set_miner_scores_upsert_preserves_individual_values( ) validator_requests = handler.get_validator_requests_to_score( - scored_time, 7 + scored_time, 7, 86400, ["BTC"] ) assert len(validator_requests) >= 1 @@ -576,13 +580,8 @@ def test_set_miner_scores_upsert_preserves_individual_values( label="low", time_length=86400, time_increment=300, - initial_delay=0, cycle_interval_minutes=5, timeout_extra_seconds=60, - scoring_intervals={}, - window_days=10, - softmax_beta=-0.1, - smoothed_score_coefficient=0.5, thin_after_minutes=30, thin_bucket_seconds=3600, ) @@ -592,13 +591,8 @@ def test_set_miner_scores_upsert_preserves_individual_values( label="high", time_length=3600, time_increment=60, - initial_delay=0, cycle_interval_minutes=2, timeout_extra_seconds=60, - scoring_intervals={}, - window_days=3, - softmax_beta=-0.2, - smoothed_score_coefficient=0.5, thin_after_minutes=10, thin_bucket_seconds=600, ) @@ -981,8 +975,9 @@ def test_scoring_path_skips_thinned_requests(db_engine: Engine): requests = handler.get_validator_requests_to_score( scored_time=scored_time, - window_days=LOW_TEST_CONFIG.window_days, + window_days=10, time_length=LOW_TEST_CONFIG.time_length, + asset_list=["ETH"], ) returned_ids = {int(r.id) for r in (requests or [])} assert keeper_vr_id in returned_ids @@ -1128,7 +1123,10 @@ def read_predictions(self, validator_request, keys): ) validator_request = handler.get_validator_requests_to_score( - datetime.fromisoformat(start_time) + timedelta(days=2), 7 + datetime.fromisoformat(start_time) + timedelta(days=2), + 7, + 86400, + ["BTC"], )[0] result = handler.get_predictions_by_request(validator_request) assert len(result) == 1 @@ -1183,7 +1181,10 @@ def read_predictions(self, validator_request, keys): ) validator_request = handler.get_validator_requests_to_score( - datetime.fromisoformat(start_time) + timedelta(days=2), 7 + datetime.fromisoformat(start_time) + timedelta(days=2), + 7, + 86400, + ["BTC"], )[0] result = handler.get_predictions_by_request(validator_request) assert result[0].prediction == [] diff --git a/tests/test_miner_price_simulation.py b/tests/test_miner_price_simulation.py index 1aeab2fb..3ec7fb41 100644 --- a/tests/test_miner_price_simulation.py +++ b/tests/test_miner_price_simulation.py @@ -1,21 +1,27 @@ +import os import unittest from unittest.mock import patch, MagicMock from synth.miner import price_simulation from synth.miner.price_simulation import ( + HYPERLIQUID_ASSET_MAP, LAZER_FEED_ID_MAP, + TOKEN_MAP, get_asset_price, ) class TestGetAssetPriceHermes(unittest.TestCase): - def test_hermes_backend_parses_int_mantissa_and_expo(self): + def test_without_api_key_reads_hermes(self): + # No PYTH_API_KEY -> the keyless Hermes endpoint, even for assets + # that also have a Lazer feed. mock_resp = MagicMock() mock_resp.status_code = 200 mock_resp.json.return_value = { "parsed": [{"price": {"price": "7930115688547", "expo": "-8"}}] } - with patch.dict("os.environ", {"PYTH_BACKEND": "hermes"}): + with patch.dict("os.environ", {}, clear=False): + os.environ.pop("PYTH_API_KEY", None) with patch("requests.get", return_value=mock_resp) as mock_get: price = get_asset_price("BTC") called_url = mock_get.call_args[0][0] @@ -24,8 +30,9 @@ def test_hermes_backend_parses_int_mantissa_and_expo(self): assert price == 79301.15688547 -class TestGetAssetPriceProLazer(unittest.TestCase): - def test_pro_backend_posts_lazer_with_bearer(self): +class TestGetAssetPriceLazer(unittest.TestCase): + def test_with_api_key_posts_lazer_with_bearer(self): + # PYTH_API_KEY present -> Lazer (paid) for assets with a Lazer feed. mock_resp = MagicMock() mock_resp.status_code = 200 mock_resp.json.return_value = { @@ -40,8 +47,7 @@ def test_pro_backend_posts_lazer_with_bearer(self): } } - env = {"PYTH_BACKEND": "pro", "PYTH_API_KEY": "test-token"} - with patch.dict("os.environ", env): + with patch.dict("os.environ", {"PYTH_API_KEY": "test-token"}): with patch("requests.post", return_value=mock_resp) as mock_post: price = get_asset_price("BTC") @@ -57,27 +63,28 @@ def test_pro_backend_posts_lazer_with_bearer(self): assert body["priceFeedIds"] == [LAZER_FEED_ID_MAP["BTC"]] assert body["parsed"] is True - def test_pro_backend_routes_wtioil_to_hyperliquid(self): - # WTIOIL has no working Lazer feed; in the pro branch the miner - # mirrors the validator and pulls WTIOIL spot from Hyperliquid - # (same `coin` code as PriceDataProvider.HYPERLIQUID_SYMBOL_MAP). - from synth.miner.price_simulation import HYPERLIQUID_ASSET_MAP - - assert "WTIOIL" not in LAZER_FEED_ID_MAP - assert HYPERLIQUID_ASSET_MAP["WTIOIL"] == "xyz:CL" +class TestGetAssetPriceHyperliquid(unittest.TestCase): + @staticmethod + def _candles_resp(close: str) -> MagicMock: mock_resp = MagicMock() mock_resp.status_code = 200 mock_resp.json.return_value = [ - {"t": 1, "c": "64.10"}, - {"t": 2, "c": "65.00"}, + {"t": 1, "c": "1.0"}, + {"t": 2, "c": close}, ] - env = {"PYTH_BACKEND": "pro", "PYTH_API_KEY": "test-token"} - with patch.dict("os.environ", env): + return mock_resp + + def test_wtioil_routes_to_hyperliquid(self): + # WTIOIL has no usable Lazer feed; the miner pulls it from + # Hyperliquid (same coin the validator scores against), key or not. + assert "WTIOIL" not in LAZER_FEED_ID_MAP + assert HYPERLIQUID_ASSET_MAP["WTIOIL"] == "xyz:CL" + + resp = self._candles_resp("65.00") + with patch.dict("os.environ", {"PYTH_API_KEY": "test-token"}): with patch("requests.get") as mock_get: - with patch( - "requests.post", return_value=mock_resp - ) as mock_post: + with patch("requests.post", return_value=resp) as mock_post: price = get_asset_price("WTIOIL") assert price == 65.0 @@ -91,18 +98,28 @@ def test_pro_backend_routes_wtioil_to_hyperliquid(self): assert body["req"]["coin"] == "xyz:CL" assert body["req"]["interval"] == "1m" - def test_pro_backend_returns_none_when_api_key_missing(self): - # Missing key returns None *without* raising — tenacity's @retry only - # acts on exceptions, so call_count must stay at 0 (we don't want 5 - # useless network attempts once a key is later added). - env = {"PYTH_BACKEND": "pro"} - with patch.dict("os.environ", env, clear=False): - # Make sure PYTH_API_KEY is absent. - with patch.dict("os.environ", {"PYTH_API_KEY": ""}): - with patch("requests.post") as mock_post: - price = get_asset_price("BTC") - assert price is None - mock_post.assert_not_called() + def test_spcx_routes_to_hyperliquid_even_with_key(self): + # SPCX is not on Hermes and has no Lazer feed — it exists only on + # Hyperliquid, so it must resolve there regardless of PYTH_API_KEY. + assert "SPCX" not in TOKEN_MAP + assert "SPCX" not in LAZER_FEED_ID_MAP + assert HYPERLIQUID_ASSET_MAP["SPCX"] == "xyz:SPCX" + + resp = self._candles_resp("187.08") + with patch.dict("os.environ", {"PYTH_API_KEY": "test-token"}): + with patch("requests.get") as mock_get: + with patch("requests.post", return_value=resp) as mock_post: + price = get_asset_price("SPCX") + + assert price == 187.08 + + mock_get.assert_not_called() + called_url = mock_post.call_args[0][0] + assert called_url == price_simulation.hyperliquid_base_url + body = mock_post.call_args.kwargs["json"] + assert body["type"] == "candleSnapshot" + assert body["req"]["coin"] == "xyz:SPCX" + assert body["req"]["interval"] == "1m" if __name__ == "__main__": diff --git a/tests/test_moving_average.py b/tests/test_moving_average.py index 6ee1dee4..d171af9a 100644 --- a/tests/test_moving_average.py +++ b/tests/test_moving_average.py @@ -6,7 +6,7 @@ from synth.validator.miner_data_handler import MinerDataHandler from synth.validator.moving_average import compute_smoothed_score -from synth.validator.prompt_config import LOW_FREQUENCY +from synth.validator.competition_config import CRYPTO_24H def read_csv(file_name): @@ -27,7 +27,7 @@ def test_moving_average_1(db_engine: Engine): handler, input_df=df, scored_time=scored_time, - prompt_config=LOW_FREQUENCY, + comp=CRYPTO_24H, ) # The miner id you want to search for @@ -61,7 +61,7 @@ def test_moving_average_2(db_engine: Engine): handler, input_df=df, scored_time=scored_time, - prompt_config=LOW_FREQUENCY, + comp=CRYPTO_24H, ) # The miner id you want to search for diff --git a/tests/test_moving_average_unit.py b/tests/test_moving_average_unit.py index 2d68b6d2..351f46b3 100644 --- a/tests/test_moving_average_unit.py +++ b/tests/test_moving_average_unit.py @@ -23,7 +23,11 @@ compute_smoothed_score, prepare_df_for_moving_average, ) -from synth.validator.prompt_config import LOW_FREQUENCY, HIGH_FREQUENCY +from synth.validator.competition_config import ( + CRYPTO_1H, + CRYPTO_24H, + SMOOTHED_SCORE_COEFFICIENT, +) # --------------------------------------------------------------------------- # Helpers @@ -344,9 +348,7 @@ def test_basic_single_miner(self): handler = _mock_handler({1: 10}) scored_time = _ts(120) - result = compute_smoothed_score( - handler, df, scored_time, LOW_FREQUENCY - ) + result = compute_smoothed_score(handler, df, scored_time, CRYPTO_24H) assert result is not None assert len(result) == 1 @@ -354,7 +356,7 @@ def test_basic_single_miner(self): assert result[0]["miner_uid"] == 10 assert "smoothed_score" in result[0] assert "reward_weight" in result[0] - assert result[0]["prompt_name"] == "low" + assert result[0]["prompt_name"] == "Crypto 24h" def test_multiple_miners_rewards_sum_to_coefficient(self): """Reward weights across all miners should sum to smoothed_score_coefficient.""" @@ -365,14 +367,12 @@ def test_multiple_miners_rewards_sum_to_coefficient(self): handler = _mock_handler({1: 10, 2: 11, 3: 12}) scored_time = _ts(120) - result = compute_smoothed_score( - handler, df, scored_time, LOW_FREQUENCY - ) + result = compute_smoothed_score(handler, df, scored_time, CRYPTO_24H) assert result is not None total_weight = sum(r["reward_weight"] for r in result) assert total_weight == pytest.approx( - LOW_FREQUENCY.smoothed_score_coefficient, abs=1e-6 + SMOOTHED_SCORE_COEFFICIENT, abs=1e-6 ) def test_empty_df_returns_none(self): @@ -381,7 +381,7 @@ def test_empty_df_returns_none(self): columns=["scored_time", "miner_id", "prompt_score_v3", "asset"] ) handler = _mock_handler({}) - result = compute_smoothed_score(handler, df, _ts(0), LOW_FREQUENCY) + result = compute_smoothed_score(handler, df, _ts(0), CRYPTO_24H) assert result is None def test_scored_time_filters_future(self): @@ -420,11 +420,11 @@ def test_scored_time_filters_future(self): # scored_time before the last entry — includes only t=0 and t=60 result_partial = compute_smoothed_score( - handler, prepared, _ts(60), LOW_FREQUENCY + handler, prepared, _ts(60), CRYPTO_24H ) # scored_time includes all three result_full = compute_smoothed_score( - handler, prepared, _ts(120), LOW_FREQUENCY + handler, prepared, _ts(120), CRYPTO_24H ) assert result_partial is not None and result_full is not None @@ -442,23 +442,23 @@ def test_miner_with_no_uid_filtered_out(self): handler = _mock_handler({1: 10, 2: None}) # miner 2 has no uid - result = compute_smoothed_score(handler, df, _ts(60), LOW_FREQUENCY) + result = compute_smoothed_score(handler, df, _ts(60), CRYPTO_24H) assert result is not None miner_ids = [r["miner_id"] for r in result] assert 2 not in miner_ids def test_high_frequency_config(self): - """Should work with HIGH_FREQUENCY config and set correct prompt_name.""" + """Should work with CRYPTO_1H config and set correct prompt_name.""" times = [_ts(0), _ts(10)] df = _make_scores_df([1], times, assets=["BTC", "BTC"], scores=[0.005]) df = prepare_df_for_moving_average(df) handler = _mock_handler({1: 10}) - result = compute_smoothed_score(handler, df, _ts(60), HIGH_FREQUENCY) + result = compute_smoothed_score(handler, df, _ts(60), CRYPTO_1H) assert result is not None - assert result[0]["prompt_name"] == "high" + assert result[0]["prompt_name"] == "Crypto 1h" def test_better_miner_gets_more_weight(self): """Miner with lower scores (better) should get higher reward weight.""" @@ -490,7 +490,7 @@ def test_better_miner_gets_more_weight(self): df = prepare_df_for_moving_average(df) handler = _mock_handler({1: 10, 2: 11}) - result = compute_smoothed_score(handler, df, _ts(120), LOW_FREQUENCY) + result = compute_smoothed_score(handler, df, _ts(120), CRYPTO_24H) assert result is not None rewards = {r["miner_id"]: r["reward_weight"] for r in result} @@ -505,9 +505,7 @@ def test_result_has_updated_at(self): handler = _mock_handler({1: 10}) scored_time = _ts(60) - result = compute_smoothed_score( - handler, df, scored_time, LOW_FREQUENCY - ) + result = compute_smoothed_score(handler, df, scored_time, CRYPTO_24H) assert result is not None assert result[0]["updated_at"] == scored_time.isoformat() @@ -526,7 +524,7 @@ def test_nan_scores_dropped(self): df["scored_time"] = pd.to_datetime(df["scored_time"], utc=True) handler = _mock_handler({1: 10}) - result = compute_smoothed_score(handler, df, _ts(60), LOW_FREQUENCY) + result = compute_smoothed_score(handler, df, _ts(60), CRYPTO_24H) # All scores are NaN -> no valid data -> None or empty assert result is None or len(result) == 0 @@ -545,7 +543,7 @@ def test_miner_with_all_nan_among_valid_miners(self): df["scored_time"] = pd.to_datetime(df["scored_time"], utc=True) handler = _mock_handler({1: 10, 2: 11}) - result = compute_smoothed_score(handler, df, _ts(120), LOW_FREQUENCY) + result = compute_smoothed_score(handler, df, _ts(120), CRYPTO_24H) assert result is not None # Miner 2 has inf rolling avg -> zero softmax weight -> filtered out @@ -578,7 +576,7 @@ def test_multiple_assets_weighted(self): df["scored_time"] = pd.to_datetime(df["scored_time"], utc=True) handler = _mock_handler({1: 10}) - result = compute_smoothed_score(handler, df, _ts(60), LOW_FREQUENCY) + result = compute_smoothed_score(handler, df, _ts(60), CRYPTO_24H) assert result is not None # XAU has higher coefficient than BTC, so weighted sum != simple sum @@ -641,10 +639,10 @@ def test_single_asset_equivalence(self): scored_time = _ts(180) new_result = compute_smoothed_score( - handler, prepared, scored_time, LOW_FREQUENCY + handler, prepared, scored_time, CRYPTO_24H ) old_result = self._old_compute_smoothed_score( - prepared, scored_time, LOW_FREQUENCY + prepared, scored_time, CRYPTO_24H ) assert new_result is not None @@ -678,10 +676,10 @@ def test_multi_asset_equivalence(self): scored_time = _ts(300) new_result = compute_smoothed_score( - handler, prepared, scored_time, LOW_FREQUENCY + handler, prepared, scored_time, CRYPTO_24H ) old_result = self._old_compute_smoothed_score( - prepared, scored_time, LOW_FREQUENCY + prepared, scored_time, CRYPTO_24H ) assert new_result is not None @@ -732,10 +730,10 @@ def test_realistic_equivalence(self): scored_time = _ts(n_times * 65 + 60) new_result = compute_smoothed_score( - handler, prepared, scored_time, LOW_FREQUENCY + handler, prepared, scored_time, CRYPTO_24H ) old_result = self._old_compute_smoothed_score( - prepared, scored_time, LOW_FREQUENCY + prepared, scored_time, CRYPTO_24H ) assert new_result is not None @@ -927,7 +925,7 @@ def test_compute_smoothed_realistic(self): scored_time = df["scored_time"].max() + timedelta(hours=1) result = compute_smoothed_score( - handler, prepared, scored_time, LOW_FREQUENCY + handler, prepared, scored_time, CRYPTO_24H ) assert result is not None @@ -936,7 +934,7 @@ def test_compute_smoothed_realistic(self): # Reward weights should sum to smoothed_score_coefficient total_weight = sum(r["reward_weight"] for r in result) assert total_weight == pytest.approx( - LOW_FREQUENCY.smoothed_score_coefficient, abs=1e-4 + SMOOTHED_SCORE_COEFFICIENT, abs=1e-4 ) # All smoothed scores should be non-negative @@ -987,7 +985,7 @@ def test_zero_scores_handled(self): handler = _mock_handler({1: 10, 2: 11, 3: 12}) result = compute_smoothed_score( - handler, prepared, _ts(120), LOW_FREQUENCY + handler, prepared, _ts(120), CRYPTO_24H ) assert result is not None @@ -1012,14 +1010,14 @@ def test_compute_smoothed_realistic_hft(self): scored_time = df["scored_time"].max() + timedelta(hours=1) result = compute_smoothed_score( - handler, prepared, scored_time, HIGH_FREQUENCY + handler, prepared, scored_time, CRYPTO_1H ) assert result is not None assert len(result) > 0 total_weight = sum(r["reward_weight"] for r in result) assert total_weight == pytest.approx( - HIGH_FREQUENCY.smoothed_score_coefficient, abs=1e-4 + SMOOTHED_SCORE_COEFFICIENT, abs=1e-4 ) def test_asset_coefficient_weighting(self): @@ -1044,7 +1042,7 @@ def test_asset_coefficient_weighting(self): handler = _mock_handler({1: 10}) prepared = prepare_df_for_moving_average(df) result = compute_smoothed_score( - handler, prepared, _ts(180), LOW_FREQUENCY + handler, prepared, _ts(180), CRYPTO_24H ) assert result is not None @@ -1133,7 +1131,7 @@ def test_existing_test_csv_compatible(self): scored_time = df["scored_time"].max() + timedelta(hours=1) result = compute_smoothed_score( - handler, prepared, scored_time, LOW_FREQUENCY + handler, prepared, scored_time, CRYPTO_24H ) assert result is not None assert len(result) > 200 # ~248 miners expected @@ -1153,10 +1151,10 @@ def test_full_pipeline_basic(self): scored_time = _ts(180) low_result = compute_smoothed_score( - handler, prepared, scored_time, LOW_FREQUENCY + handler, prepared, scored_time, CRYPTO_24H ) high_result = compute_smoothed_score( - handler, prepared, scored_time, HIGH_FREQUENCY + handler, prepared, scored_time, CRYPTO_1H ) assert low_result is not None @@ -1194,7 +1192,7 @@ def test_full_pipeline_with_new_miner(self): handler = _mock_handler({1: 10, 2: 11, 3: 12}) result = compute_smoothed_score( - handler, prepared, _ts(180), LOW_FREQUENCY + handler, prepared, _ts(180), CRYPTO_24H ) assert result is not None @@ -1215,12 +1213,12 @@ def test_full_pipeline_deterministic(self): prepared1 = prepare_df_for_moving_average(df) result1 = compute_smoothed_score( - handler, prepared1, scored_time, LOW_FREQUENCY + handler, prepared1, scored_time, CRYPTO_24H ) prepared2 = prepare_df_for_moving_average(df) result2 = compute_smoothed_score( - handler, prepared2, scored_time, LOW_FREQUENCY + handler, prepared2, scored_time, CRYPTO_24H ) assert result1 is not None and result2 is not None diff --git a/tests/test_price_data_provider.py b/tests/test_price_data_provider.py index 28c0822e..79c209c8 100644 --- a/tests/test_price_data_provider.py +++ b/tests/test_price_data_provider.py @@ -419,10 +419,6 @@ def test_accepts_when_witness_candle_present(self): class TestPriceDataProviderProBackend(unittest.TestCase): - """Same regression suite, but with PYTH_BACKEND=pro selected so the - provider hits the Pyth Pro Router URL. Response shape is identical to - the legacy Benchmarks API, so the price-extraction output must match.""" - def test_pro_backend_uses_pro_url(self): # 1739974740 is the settlement-witness candle past the last grid # point at 1739974680 (= start + time_length). @@ -437,40 +433,16 @@ def test_pro_backend_uses_pro_url(self): "c": [100000.23, 99000.55, 103000.55, 108000.867, 108500.0], } - with patch.dict("os.environ", {"PYTH_BACKEND": "pro"}): - provider = PriceDataProvider() - with patch("requests.get") as mock_get: - mock_get.return_value.json.return_value = mock_response - result = provider.fetch_data(validator_request) - - called_url = mock_get.call_args[0][0] - called_params = mock_get.call_args.kwargs["params"] - assert called_url == PriceDataProvider.PYTH_PRO_URL - # The fetch window must extend one minute past the last grid - # point so the settlement witness can land in the response. - assert called_params["to"] == 1739974680 + 60 - assert result == [100000.23, 99000.55, 103000.55, 108000.867] - - def test_hermes_backend_uses_benchmarks_url(self): - mock_response = { - "t": [ - 1739974320, - 1739974440, - 1739974560, - 1739974680, - 1739974740, - ], - "c": [100000.23, 99000.55, 103000.55, 108000.867, 108500.0], - } - - with patch.dict("os.environ", {"PYTH_BACKEND": "hermes"}): - provider = PriceDataProvider() - with patch("requests.get") as mock_get: - mock_get.return_value.json.return_value = mock_response - provider.fetch_data(validator_request) + provider = PriceDataProvider() + with patch("requests.get") as mock_get: + mock_get.return_value.json.return_value = mock_response + result = provider.fetch_data(validator_request) - called_url = mock_get.call_args[0][0] - assert called_url == PriceDataProvider.PYTH_BENCHMARKS_URL + called_params = mock_get.call_args.kwargs["params"] + # The fetch window must extend one minute past the last grid + # point so the settlement witness can land in the response. + assert called_params["to"] == 1739974680 + 60 + assert result == [100000.23, 99000.55, 103000.55, 108000.867] class TestPriceDataProviderLiveProBackend(unittest.TestCase): @@ -485,36 +457,29 @@ def test_live_history_from_pro_router_per_asset(self): ) - timedelta(minutes=5) start = end - timedelta(minutes=10) - with patch.dict("os.environ", {"PYTH_BACKEND": "pro"}): - provider = PriceDataProvider() - self.assertEqual( - provider.pyth_history_url, - PriceDataProvider.PYTH_PRO_URL, - ) - - for asset in PriceDataProvider.PYTH_SYMBOL_MAP.keys(): - with self.subTest(asset=asset): - req = ValidatorRequest( - asset=asset, - start_time=start, - time_length=600, - time_increment=60, - ) - prices = provider.fetch_data(req) - - # time_length=600s @ time_increment=60s => 11 grid points. - self.assertEqual(len(prices), 11) - finite = [p for p in prices if not np.isnan(p)] - self.assertGreater( - len(finite), - 5, - f"{asset}: too many gaps: {prices}", + provider = PriceDataProvider() + + for asset in PriceDataProvider.PYTH_SYMBOL_MAP.keys(): + with self.subTest(asset=asset): + req = ValidatorRequest( + asset=asset, + start_time=start, + time_length=600, + time_increment=60, + ) + prices = provider.fetch_data(req) + + # time_length=600s @ time_increment=60s => 11 grid points. + self.assertEqual(len(prices), 11) + finite = [p for p in prices if not np.isnan(p)] + self.assertGreater( + len(finite), + 5, + f"{asset}: too many gaps: {prices}", + ) + for p in finite: + # Loose sanity bounds — XAU ~$5k, HYPE ~$40, BTC ~$80k. + self.assertGreater(p, 0, f"{asset}: non-positive price") + self.assertLess( + p, 10_000_000, f"{asset}: suspicious magnitude" ) - for p in finite: - # Loose sanity bounds — XAU ~$5k, HYPE ~$40, BTC ~$80k. - self.assertGreater( - p, 0, f"{asset}: non-positive price" - ) - self.assertLess( - p, 10_000_000, f"{asset}: suspicious magnitude" - ) diff --git a/tests/test_rewards.py b/tests/test_rewards.py index 640f1d9e..e401039f 100644 --- a/tests/test_rewards.py +++ b/tests/test_rewards.py @@ -12,6 +12,7 @@ compute_softmax, get_rewards_multiprocess, ) +from synth.validator import competition_config from tests.utils import prepare_random_predictions, recent_start_time @@ -81,13 +82,14 @@ def test_get_rewards(db_engine): price_data_provider = PriceDataProvider() validator_requests = handler.get_validator_requests_to_score( - scored_time, 7 + scored_time, 7, 86400, ["BTC"] ) prompt_scores, detailed_info, real_prices = get_rewards_multiprocess( handler, price_data_provider, validator_requests[0], + competition_config.CRYPTO_24H, ) assert prompt_scores is not None diff --git a/tests/test_sequential_scheduler.py b/tests/test_sequential_scheduler.py index 58d94a16..7d6a4bcc 100644 --- a/tests/test_sequential_scheduler.py +++ b/tests/test_sequential_scheduler.py @@ -16,13 +16,8 @@ def _config(cycle_interval_minutes: int) -> PromptConfig: label="test", time_length=86400, time_increment=300, - initial_delay=0, cycle_interval_minutes=cycle_interval_minutes, timeout_extra_seconds=60, - scoring_intervals={}, - window_days=10, - softmax_beta=-0.1, - smoothed_score_coefficient=0.5, ) diff --git a/tests/test_simulations.py b/tests/test_simulations.py index 678e6bbc..5dce14f0 100644 --- a/tests/test_simulations.py +++ b/tests/test_simulations.py @@ -6,7 +6,7 @@ from synth.validator.response_validation_v2 import CORRECT, validate_responses -# get_asset_price hits a live price feed (Pyth Lazer on PYTH_BACKEND=pro); +# get_asset_price hits a live price feed; # pin it so these tests exercise the simulation math without a network call. @patch("synth.miner.simulations.get_asset_price", return_value=90000.0) def test_generate_simulations(mock_get_asset_price): diff --git a/tests/utils.py b/tests/utils.py index 112a08bd..175a51dd 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -55,8 +55,7 @@ def prepare_random_predictions(db_engine: Engine, start_time: str): ) # generate_simulations() fetches a live current price via get_asset_price - # (Pyth Lazer when PYTH_BACKEND=pro). Tests must not depend on that - # external call, so pin the price and let the GBM math run on it. + # Tests must not depend on that external call, so pin the price and let the GBM math run on it. with patch( "synth.miner.simulations.get_asset_price", return_value=90000.0 ): diff --git a/validator.config.js b/validator.config.js index c11271e1..67f5e385 100644 --- a/validator.config.js +++ b/validator.config.js @@ -22,7 +22,7 @@ module.exports = { name: "validator cycle scoring", interpreter: "python3", script: "./neurons/validator.py", - args: "--netuid 50 --logging.debug --wallet.name validator --wallet.hotkey default --neuron.nprocs 8 --softmax.low.beta -0.4 --softmax.high.beta -0.4 --sma.low.days 10 --sma.high.days 5 --validator.mode light --retention.low.days 11 --retention.high.days 6 --validator.cycle_name scoring", + args: "--netuid 50 --logging.debug --wallet.name validator --wallet.hotkey default --neuron.nprocs 8 --validator.mode light --retention.low.days 11 --retention.high.days 6 --validator.cycle_name scoring", env: { PYTHONPATH: ".", }, diff --git a/verify/pyth-lazer-listing.py b/verify/pyth-lazer-listing.py index 8eaa9e72..775248bb 100644 --- a/verify/pyth-lazer-listing.py +++ b/verify/pyth-lazer-listing.py @@ -54,6 +54,15 @@ "WTIOIL": "925ca92ff005ae943c158e3563f59698ce7e75c5a8c8dd43303a0a154887b3e6", } +# Assets with no hermes_id in the Lazer catalog — Pyth-Pro / Lazer-only feeds. +# SPCX is `Pyth.HL.SPCX/USDC` (pyth_lazer_id=99934, exponent=-4): the legacy +# hermes/Benchmarks endpoint returns "Symbol ... doesn't exist" for it, so the +# validator can only score SPCX with PYTH_BACKEND=pro. The hermes_id bridge +# can't resolve these, so we match them by symbol string instead. +LAZER_SYMBOL_MAP = { + "SPCX": "Pyth.HL.SPCX/USDC", +} + def lazer_auth_headers() -> dict: key = os.environ.get("PYTH_API_KEY") @@ -96,10 +105,15 @@ def map_assets_to_lazer(symbols: list) -> dict: if hermes_id: by_hermes[hermes_id] = entry + by_symbol = {entry.get("symbol"): entry for entry in symbols} + mapping = {} for asset, hermes_id in HERMES_ID_MAP.items(): match = by_hermes.get(normalize_hex(hermes_id)) mapping[asset] = match + # Lazer-only feeds have no hermes_id; resolve them by symbol instead. + for asset, symbol in LAZER_SYMBOL_MAP.items(): + mapping[asset] = by_symbol.get(symbol) return mapping