Skip to content

Follow ups + challenge fixes #634

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ward-taoshi merged 9 commits into from
Dec 11, 2025
Merged

Follow ups + challenge fixes #634

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
1418ea4
Follow up
ward-taoshi Dec 9, 2025
7d84d02
Use challenge period as source for miner asset scores
ward-taoshi Dec 9, 2025
ce4f46d
Enable sltp fields for market orders
ward-taoshi Dec 10, 2025
7fca32a
Fix tests
ward-taoshi Dec 10, 2025
c0799e5
Better logging message
ward-taoshi Dec 10, 2025
892e5d2
Bump tiingo data interval for full day
ward-taoshi Dec 10, 2025
cc0bc40
Raise SignalException on sltp error
ward-taoshi Dec 11, 2025
b8db5ac
fix tests
ward-taoshi Dec 11, 2025
efe15a9
Bump meta
ward-taoshi Dec 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion data_generator/tiingo_data_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ async def connect(self, handle_msg):
"""
Asynchronous connect method for the Tiingo websocket client
"""
POLLING_INTERVAL_S = 5
POLLING_INTERVAL_S = 8.6
last_poll_time = 0

while not self._should_close:
Expand Down
12 changes: 6 additions & 6 deletions docs/miner.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,12 +110,12 @@ $$

| Metric | Scoring Weight |
|------------------------|----------------|
| Average Daily PnL | 90% |
| Calmar Ratio | 2% |
| Sharpe Ratio | 2% |
| Omega Ratio | 2% |
| Sortino Ratio | 2% |
| Statistical Confidence | 2% |
| Average Daily PnL | 100% |
| Calmar Ratio | 0% |
| Sharpe Ratio | 0% |
| Omega Ratio | 0% |
| Sortino Ratio | 0% |
| Statistical Confidence | 0% |

### Scoring Penalties

Expand Down
56 changes: 29 additions & 27 deletions tests/vali_tests/test_challengeperiod_unit.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
from tests.vali_tests.base_objects.test_base import TestBase
from vali_objects.enums.order_type_enum import OrderType
from vali_objects.vali_dataclasses.position import Position
from vali_objects.scoring.scoring import Scoring
from vali_objects.challenge_period import ChallengePeriodManager
from vali_objects.vali_dataclasses.ledger.ledger_utils import LedgerUtils
from vali_objects.enums.miner_bucket_enum import MinerBucket
Expand Down Expand Up @@ -197,35 +196,23 @@ def save_and_get_positions(self, base_positions, hotkeys):
# Re-raise to preserve original test failure behavior
raise

def get_combined_scores_dict(self, miner_scores: dict[str, float], asset_class=None):
def get_asset_softmaxed_scores(self, miner_scores: dict[str, float], asset_class=None):
"""
Create a combined scores dict for testing.
Create asset_softmaxed_scores dict for testing.

Args:
miner_scores: dict mapping hotkey to score (0.0 to 1.0)
asset_class: TradePairCategory, defaults to CRYPTO

Returns:
combined_scores_dict in the format expected by inspect()
asset_softmaxed_scores in the format expected by inspect()
Format: {TradePairCategory: {hotkey: score, ...}}
"""
if asset_class is None:
asset_class = vali_file.TradePairCategory.CRYPTO

combined_scores_dict = {asset_class: {"metrics": {}, "penalties": {}}}
asset_class_dict = combined_scores_dict[asset_class]

# Create scores for each metric
for config_name, config in Scoring.scoring_config.items():
scores_list = [(hotkey, score) for hotkey, score in miner_scores.items()]
asset_class_dict["metrics"][config_name] = {
'scores': scores_list,
'weight': config['weight']
}

# All miners get penalty multiplier of 1 (no penalty)
asset_class_dict["penalties"] = {hotkey: 1.0 for hotkey in miner_scores.keys()}

return combined_scores_dict
# asset_softmaxed_scores format: {asset_class: {hotkey: score}}
return {asset_class: miner_scores}

def _populate_active_miners(self, *, maincomp=[], challenge=[], probation=[]):
"""Populate active miners using RPC client methods with error handling."""
Expand Down Expand Up @@ -309,7 +296,7 @@ def test_failing_remaining_time(self):
# Top 25 success miners get scores from 1.0 down to 0.76 (25 miners)
miner_scores[self.SUCCESS_MINER_NAMES[i]] = 1.0 - (i * 0.01)

combined_scores_dict = self.get_combined_scores_dict(miner_scores)
asset_softmaxed_scores = self.get_asset_softmaxed_scores(miner_scores)

# Check that the miner continues in challenge (time remaining, so not eliminated)
passing, demoted, failing = self.challenge_period_client.inspect(
Expand All @@ -320,7 +307,7 @@ def test_failing_remaining_time(self):
inspection_hotkeys={"miner": current_time},
current_time=current_time,
hk_to_first_order_time=hk_to_first_order_time,
combined_scores_dict=combined_scores_dict,
asset_softmaxed_scores=asset_softmaxed_scores,
)
self.assertNotIn("miner", passing)
self.assertNotIn("miner", list(failing.keys()))
Expand Down Expand Up @@ -367,6 +354,10 @@ def test_passing_remaining_time(self):
inspection_hotkeys = {"miner": self.START_TIME}
current_time = self.CURRENTLY_IN_CHALLENGE

# Create scores where miner is in top 25 (passing)
miner_scores = {"miner": 1.0}
asset_softmaxed_scores = self.get_asset_softmaxed_scores(miner_scores)

# Check that the miner is screened as passing
passing, demoted, failing = self.challenge_period_client.inspect(
positions=inspection_positions,
Expand All @@ -376,6 +367,7 @@ def test_passing_remaining_time(self):
inspection_hotkeys=inspection_hotkeys,
current_time=current_time,
hk_to_first_order_time=hk_to_first_order_time,
asset_softmaxed_scores=asset_softmaxed_scores,
)

self.assertIn("miner", passing)
Expand All @@ -395,6 +387,10 @@ def test_passing_no_remaining_time(self):
inspection_hotkeys = {"miner": self.START_TIME}
current_time = self.CURRENTLY_IN_CHALLENGE

# Create scores where miner is in top 25 (passing)
miner_scores = {"miner": 1.0}
asset_softmaxed_scores = self.get_asset_softmaxed_scores(miner_scores)

# Check that the miner is screened as passing
passing, demoted, failing = self.challenge_period_client.inspect(
positions=inspection_positions,
Expand All @@ -404,6 +400,7 @@ def test_passing_no_remaining_time(self):
inspection_hotkeys=inspection_hotkeys,
current_time=current_time,
hk_to_first_order_time=hk_to_first_order_time,
asset_softmaxed_scores=asset_softmaxed_scores,
)

self.assertIn("miner", passing)
Expand Down Expand Up @@ -499,7 +496,7 @@ def test_just_above_threshold(self):
miner_scores[self.SUCCESS_MINER_NAMES[23]] = 0.76
miner_scores[self.SUCCESS_MINER_NAMES[24]] = 0.75

combined_scores_dict = self.get_combined_scores_dict(miner_scores)
asset_softmaxed_scores = self.get_asset_softmaxed_scores(miner_scores)

# Check that the miner is promoted (in top 25)
passing, demoted, failing = self.challenge_period_client.inspect(
Expand All @@ -510,7 +507,7 @@ def test_just_above_threshold(self):
inspection_hotkeys={"miner": current_time},
current_time=current_time,
hk_to_first_order_time=hk_to_first_order_time,
combined_scores_dict=combined_scores_dict,
asset_softmaxed_scores=asset_softmaxed_scores,
)
self.assertIn("miner", passing)
self.assertNotIn("miner", list(failing.keys()))
Expand Down Expand Up @@ -539,7 +536,7 @@ def test_just_below_threshold(self):

miner_scores["miner"] = 0.74 # Rank 26 (just below rank 25's score of 0.76)

combined_scores_dict = self.get_combined_scores_dict(miner_scores)
asset_softmaxed_scores = self.get_asset_softmaxed_scores(miner_scores)

# Check that the miner continues in challenge (not promoted, not eliminated)
passing, demoted, failing = self.challenge_period_client.inspect(
Expand All @@ -550,7 +547,7 @@ def test_just_below_threshold(self):
inspection_hotkeys={"miner": current_time},
current_time=current_time,
hk_to_first_order_time=hk_to_first_order_time,
combined_scores_dict=combined_scores_dict,
asset_softmaxed_scores=asset_softmaxed_scores,
)
self.assertNotIn("miner", passing)
self.assertNotIn("miner", list(failing.keys()))
Expand Down Expand Up @@ -578,7 +575,7 @@ def test_at_threshold(self):
miner_scores["miner"] = 0.76 # Ties for rank 25
miner_scores[self.SUCCESS_MINER_NAMES[24]] = 0.75 # Rank 26, will be demoted

combined_scores_dict = self.get_combined_scores_dict(miner_scores)
asset_softmaxed_scores = self.get_asset_softmaxed_scores(miner_scores)

# Check that the miner is promoted (at threshold rank 25)
passing, demoted, failing = self.challenge_period_client.inspect(
Expand All @@ -589,7 +586,7 @@ def test_at_threshold(self):
inspection_hotkeys={"miner": current_time},
current_time=current_time,
hk_to_first_order_time=hk_to_first_order_time,
combined_scores_dict=combined_scores_dict,
asset_softmaxed_scores=asset_softmaxed_scores,
)

self.assertIn("miner", passing)
Expand Down Expand Up @@ -621,6 +618,10 @@ def test_screen_minimum_interaction(self):
portfolio_cps = [cp for cp in base_ledger_portfolio.cps if cp.last_update_ms < current_time]
base_ledger_portfolio.cps = portfolio_cps

# Create scores where miner is in top 25 (passing)
miner_scores = {"miner": 1.0}
asset_softmaxed_scores = self.get_asset_softmaxed_scores(miner_scores)

# Check that miner with a passing score passes when they have enough trading days
passing, demoted, failing = self.challenge_period_client.inspect(
positions=inspection_positions,
Expand All @@ -630,6 +631,7 @@ def test_screen_minimum_interaction(self):
inspection_hotkeys={"miner": current_time},
current_time=current_time,
hk_to_first_order_time=hk_to_first_order_time,
asset_softmaxed_scores=asset_softmaxed_scores,
)

self.assertIn("miner", passing)
Expand Down
Loading
Loading