diff --git a/runnable/generate_request_minerstatistics.py b/runnable/generate_request_minerstatistics.py index 5c9ab6aa9..4f324a661 100644 --- a/runnable/generate_request_minerstatistics.py +++ b/runnable/generate_request_minerstatistics.py @@ -114,9 +114,14 @@ def generate_request_minerstatistics(time_now:int): all_miner_hotkeys = challengeperiod_success_hotkeys + challengeperiod_testing_hotkeys filtered_ledger = subtensor_weight_setter.filtered_ledger(hotkeys=all_miner_hotkeys) + filtered_positions = subtensor_weight_setter.get_all_miner_positions_by_hotkey( + challengeperiod_success_hotkeys, + sort_positions=True, + acceptable_position_end_ms=time_now - ValiConfig.SET_WEIGHT_LOOKBACK_RANGE_MS + ) ## Penalties - miner_penalties = Scoring.miner_penalties(filtered_ledger) + miner_penalties = Scoring.miner_penalties(filtered_ledger, filtered_positions) fullpenalty_miners: list[tuple[str, float]] = [ ( miner, 0 ) for miner, penalty in miner_penalties.items() if penalty == 0 ] consistency_penalties = {} @@ -172,7 +177,12 @@ def generate_request_minerstatistics(time_now:int): ## This is when we only want to look at the successful miners successful_ledger = subtensor_weight_setter.filtered_ledger(hotkeys=challengeperiod_success_hotkeys) - checkpoint_results = Scoring.compute_results_checkpoint(successful_ledger, evaluation_time_ms=time_now, verbose=False) + checkpoint_results = Scoring.compute_results_checkpoint( + ledger_dict=successful_ledger, + filtered_positions=filtered_positions, + evaluation_time_ms=time_now, + verbose=False + ) challengeperiod_scores = [ (x, ValiConfig.SET_WEIGHT_MINER_CHALLENGE_PERIOD_WEIGHT) for x in challengeperiod_testing_hotkeys ] scoring_results = checkpoint_results + challengeperiod_scores diff --git a/runnable/run_incentive_review.py b/runnable/run_incentive_review.py index 3820f9617..0217f2035 100644 --- a/runnable/run_incentive_review.py +++ b/runnable/run_incentive_review.py @@ -48,13 +48,19 @@ ) filtered_ledger = subtensor_weight_setter.filtered_ledger(hotkeys=passing_hotkeys + challengeperiod_success) + filtered_positions = subtensor_weight_setter.get_all_miner_positions_by_hotkey( + passing_hotkeys + challengeperiod_success, + sort_positions=True, + acceptable_position_end_ms=current_time - ValiConfig.SET_WEIGHT_LOOKBACK_RANGE_MS + ) + return_decay_coefficient_short = ValiConfig.HISTORICAL_DECAY_COEFFICIENT_RETURNS_SHORT return_decay_coefficient_long = ValiConfig.HISTORICAL_DECAY_COEFFICIENT_RETURNS_LONG risk_adjusted_decay_coefficient = ValiConfig.HISTORICAL_DECAY_COEFFICIENT_RISKMETRIC return_decay_short_lookback_time_ms = ValiConfig.RETURN_DECAY_SHORT_LOOKBACK_TIME_MS # Compute miner penalties - miner_penalties = Scoring.miner_penalties(filtered_ledger) + miner_penalties = Scoring.miner_penalties(filtered_ledger, filtered_positions) ## Miners with full penalty fullpenalty_miner_scores: list[tuple[str, float]] = [ ( miner, 0 ) for miner, penalty in miner_penalties.items() if penalty == 0 ] @@ -63,11 +69,16 @@ ## Individual miner penalties consistency_penalties = {} drawdown_penalties = {} + positional_penalties = {} for miner, ledger in filtered_ledger.items(): minercps = ledger.cps consistency_penalties[miner] = PositionUtils.compute_consistency_penalty_cps(minercps) drawdown_penalties[miner] = PositionUtils.compute_drawdown_penalty_cps(minercps) + positional_penalties[miner] = PositionUtils.compute_positional_penalty_cps( + minercps, + filtered_positions[miner] + ) # Augmented returns ledgers returns_ledger_short = PositionManager.limit_perf_ledger( @@ -142,7 +153,7 @@ combined_scores[miner] = 1 combined_scores[miner] *= config['weight'] * score + (1 - config['weight']) - combined_weighed = Scoring.weigh_miner_scores(list(combined_scores.items())) + fullpenalty_miner_scores + combined_weighed = Scoring.softmax_scores(list(combined_scores.items())) + fullpenalty_miner_scores combined_scores = dict(combined_weighed) ## Normalize the scores @@ -168,22 +179,24 @@ combined_data[miner]['Penalty'] = miner_penalties.get(miner, 0) combined_data[miner]['Drawdown Penalty'] = drawdown_penalties.get(miner, 0) combined_data[miner]['Consistency Penalty'] = consistency_penalties.get(miner, 0) + combined_data[miner]['Positional Penalty'] = positional_penalties.get(miner, 0) df = pd.DataFrame.from_dict(combined_data, orient='index') - # printing_columns = [ - # 'return_cps_short Weighted Score', - # 'return_cps_long Weighted Score', - # 'Final Normalized Score', - # 'Final Rank', - # 'Penalty', - # 'Drawdown Penalty', - # 'Consistency Penalty', - # ] - - # df_subset = df[printing_columns].round(3) - # df_subset = df_subset.sort_values(by='Final Rank', ascending=True) - # # print(df_subset) + printing_columns = [ + 'return_cps_short Weighted Score', + 'return_cps_long Weighted Score', + 'Final Normalized Score', + 'Final Rank', + 'Penalty', + 'Drawdown Penalty', + 'Consistency Penalty', + 'Positional Penalty' + ] + + df_subset = df[printing_columns].round(3) + df_subset = df_subset.sort_values(by='Final Rank', ascending=True) + print(df_subset.head(30)) # Print rankings and original scores for each metric for metric, ranks in rankings.items(): diff --git a/vali_config.py b/vali_config.py index 9070cc919..3815645cc 100644 --- a/vali_config.py +++ b/vali_config.py @@ -81,6 +81,10 @@ class ValiConfig: SET_WEIGHT_MINIMUM_POSITION_DURATION_MS = 1 * 60 * 1000 # 1 minutes SET_WEIGHT_MINIMUM_POSITION_DURATION_TOTAL_MS = SET_WEIGHT_MINIMUM_POSITION_DURATION_MS * SET_WEIGHT_MINIMUM_POSITIONS RETURN_DECAY_SHORT_LOOKBACK_TIME_MS = 3 * 24 * 60 * 60 * 1000 # 3 days + REALIZED_RETURN_EXPONENT = 4 + SOFTMAX_TEMPERATURE = 0.5 + MAX_RETURN_SIGMOID_SPREAD = 15 + MAX_RETURN_SIGMOID_SHIFT = 0.5 SET_WEIGHT_MINER_CHECKPOINT_CONSISTENCY_DISPLACEMENT = 15 SET_WEIGHT_CHECKPOINT_CONSISTENCY_TAPER = SET_WEIGHT_MINER_CHECKPOINT_CONSISTENCY_DISPLACEMENT * 1.25 @@ -300,4 +304,4 @@ def __str__(self): TRADE_PAIR_ID_TO_TRADE_PAIR = {x.trade_pair_id: x for x in TradePair} -TRADE_PAIR_STR_TO_TRADE_PAIR = {x.trade_pair: x for x in TradePair} \ No newline at end of file +TRADE_PAIR_STR_TO_TRADE_PAIR = {x.trade_pair: x for x in TradePair} diff --git a/vali_objects/scoring/scoring.py b/vali_objects/scoring/scoring.py index e5f5a5f77..415680527 100644 --- a/vali_objects/scoring/scoring.py +++ b/vali_objects/scoring/scoring.py @@ -13,6 +13,8 @@ from vali_objects.utils.vali_bkp_utils import ValiBkpUtils from vali_objects.utils.vali_utils import ValiUtils from vali_objects.vali_dataclasses.perf_ledger import PerfCheckpoint, PerfLedger +from vali_objects.position import Position + from vali_objects.utils.position_manager import PositionManager from time_util.time_util import TimeUtil from vali_objects.utils.position_utils import PositionUtils @@ -63,6 +65,7 @@ class Scoring: @staticmethod def compute_results_checkpoint( ledger_dict: Dict[str, PerfLedger], + filtered_positions: Dict[str, list[Position]], evaluation_time_ms: int = None, verbose=True ) -> List[Tuple[str, float]]: @@ -82,7 +85,7 @@ def compute_results_checkpoint( return_decay_short_lookback_time_ms = ValiConfig.RETURN_DECAY_SHORT_LOOKBACK_TIME_MS # Compute miner penalties - miner_penalties = Scoring.miner_penalties(ledger_dict) + miner_penalties = Scoring.miner_penalties(ledger_dict, filtered_positions) ## Miners with full penalty fullpenalty_miner_scores: list[tuple[str, float]] = [ ( miner, 0 ) for miner, penalty in miner_penalties.items() if penalty == 0 ] @@ -134,15 +137,14 @@ def compute_results_checkpoint( combined_scores[miner] *= config['weight'] * score + (1 - config['weight']) # ## Force good performance of all error metrics - combined_weighed = Scoring.weigh_miner_scores(list(combined_scores.items())) + fullpenalty_miner_scores + combined_weighed = Scoring.softmax_scores(list(combined_scores.items())) + fullpenalty_miner_scores combined_scores = dict(combined_weighed) ## Normalize the scores - normalized_scores = Scoring.normalize_scores(combined_scores) - return sorted(normalized_scores.items(), key=lambda x: x[1], reverse=True) + return sorted(combined_scores.items(), key=lambda x: x[1], reverse=True) @staticmethod - def miner_penalties(ledger_dict: dict[str, PerfLedger]) -> dict[str, float]: + def miner_penalties(ledger_dict: dict[str, PerfLedger], miner_positions: dict[str, list[Position]]) -> dict[str, float]: # Compute miner penalties miner_penalties = {} for miner, perfledger in ledger_dict.items(): @@ -150,7 +152,9 @@ def miner_penalties(ledger_dict: dict[str, PerfLedger]) -> dict[str, float]: consistency_penalty = PositionUtils.compute_consistency_penalty_cps(ledgercps) drawdown_penalty = PositionUtils.compute_drawdown_penalty_cps(ledgercps) - miner_penalties[miner] = drawdown_penalty * consistency_penalty + positional_penalty = PositionUtils.compute_positional_penalty_cps(ledgercps, miner_positions.get(miner, [])) + + miner_penalties[miner] = drawdown_penalty * consistency_penalty * positional_penalty return miner_penalties @@ -438,7 +442,43 @@ def miner_scores_percentiles(miner_scores: list[tuple[str, float]]) -> list[tupl return miner_percentiles + @staticmethod + def softmax_scores(returns: list[tuple[str, float]]) -> list[tuple[str, float]]: + """ + Assign weights to the returns based on their relative position and apply softmax with a temperature parameter. + + The softmax function is used to convert the scores into probabilities that sum to 1. + Subtracting the max value from the scores before exponentiation improves numerical stability. + Parameters: + returns (list[tuple[str, float]]): List of tuples with miner names and their scores. + temperature (float): Temperature parameter to control the sharpness of the softmax distribution. Default is 1.0. + + Returns: + list[tuple[str, float]]: List of tuples with miner names and their softmax weights. + """ + epsilon = ValiConfig.EPSILON + temperature = ValiConfig.SOFTMAX_TEMPERATURE + + if not returns: + bt.debug("No returns to score, returning empty list") + return [] + + if len(returns) == 1: + bt.info("Only one miner, returning 1.0 for the solo miner weight") + return [(returns[0][0], 1.0)] + + # Extract scores and apply softmax with temperature + scores = [score for _, score in returns] + max_score = np.max(scores) + exp_scores = np.exp((scores - max_score) / temperature) + softmax_scores = exp_scores / max(np.sum(exp_scores), epsilon) + + # Combine miners with their respective softmax scores + weighted_returns = [(returns[i][0], softmax_scores[i]) for i in range(len(returns))] + + return weighted_returns + @staticmethod def weigh_miner_scores(returns: list[tuple[str, float]]) -> list[tuple[str, float]]: """ diff --git a/vali_objects/utils/position_utils.py b/vali_objects/utils/position_utils.py index 2dd3d77dd..f87d2e4ad 100644 --- a/vali_objects/utils/position_utils.py +++ b/vali_objects/utils/position_utils.py @@ -286,6 +286,89 @@ def mdd_augmentation(recent_drawdown: float) -> float: drawdown_penalty = base_augmentation * lower_augmentation * upper_augmentation return float(drawdown_penalty) + @staticmethod + def positional_realized_returns_distribution(realized_returns: list[float]) -> float: + """ + Returns the penalty associated with uneven distributions for realized returns + + Args: + realized_returns: list[float] - list of realized (closed position) returns + """ + epsilon = ValiConfig.EPSILON + max_return_spread = ValiConfig.MAX_RETURN_SIGMOID_SPREAD + max_return_shift = ValiConfig.MAX_RETURN_SIGMOID_SHIFT + + if len(realized_returns) <= 0: + return 0 + + ## look piecewise + realized_return = sum(realized_returns) + + if realized_return == 0: + return 0 + + if realized_return > 0: + positive_returns = [ x for x in realized_returns if x > 0 ] + numerator = max(positive_returns) + else: + negative_returns = [ x for x in realized_returns if x < 0 ] + numerator = min(negative_returns) + + denominator = realized_return + max_return_ratio = np.clip(numerator / denominator, 0, 1) + + return np.clip(1 / (1 + np.exp(max_return_spread*(max_return_ratio-max_return_shift))), 0, 1) + + @staticmethod + def positional_realized_unrealized_ratio(realized_return: float, unrealized_return: float) -> float: + """ + Returns the expected penalty for large discrepancies between realized and unrealized gains + + Args: + returns_ratio: float - ratio of realized and unrealized gains + """ + epsilon = ValiConfig.EPSILON + realized_return_ratio_exponent = ValiConfig.REALIZED_RETURN_EXPONENT + + numerator = abs(unrealized_return - realized_return) + denominator = abs(unrealized_return) + + if denominator <= epsilon: + return 0 + + returns_ratio = numerator / denominator + return np.clip(1 - returns_ratio**realized_return_ratio_exponent, 0, 1) + + @staticmethod + def compute_positional_penalty_cps(checkpoints: list[PerfCheckpoint], positions: list[Position]) -> float: + """ + This function looks at: + 1. The number of closed positions within the past 30 days, which were also opened in the past 30 days + 2. The ratio between the largest position and the total closed position returns + 3. Ratio between the realized and unrealized returns + + Args: + checkpoints: list[PerfCheckpoint] - the list of checkpoints + positions: list[Position] - the list of positions + """ + epsilon = ValiConfig.EPSILON + + # Closed Positions + closed_positions = [ x for x in positions if x.is_closed_position and x.close_ms > 0 ] + + # Total realized return + realized_returns = [ math.log(x.return_at_close) for x in closed_positions ] + realized_return = sum(realized_returns) + + # Unrealized return - already in log format + unrealized_returns = [ x.gain + x.loss for x in checkpoints ] + unrealized_return = sum(unrealized_returns) + + returns_ratio_penalty = PositionUtils.positional_realized_unrealized_ratio(realized_return, unrealized_return) + max_returns_penalty = PositionUtils.positional_realized_returns_distribution(realized_returns) + + return returns_ratio_penalty * max_returns_penalty + def compute_drawdown_penalty_cps(checkpoints: list[PerfCheckpoint]) -> float: """ Args: @@ -373,77 +456,6 @@ def compute_consistency_penalty( consistency_penalties = PositionUtils.compute_consistency(lookback_fractions) return consistency_penalties - @staticmethod - def compute_consistency_penalty_positions( - positions: list[Position], - evaluation_time_ms: int - ) -> float: - """ - Args: - positions: list[Position] - the list of positions - evaluation_time_ms: int - the evaluation time - """ - lookback_fractions = [ - PositionUtils.compute_lookback_fraction( - position.open_ms, - position.close_ms, - evaluation_time_ms - ) for position in positions - if position.is_closed_position and position.max_leverage_seen() >= ValiConfig.MIN_LEVERAGE_CONSITENCY_PENALTY - ] - - # Sort the lookback fractions in ascending order - lookback_fractions = sorted(lookback_fractions) - consistency_penalties = PositionUtils.compute_consistency(lookback_fractions) - return consistency_penalties - - @staticmethod - def compute_consistency( - lookback_fractions: list[Position] - ) -> float: - """ - Args: - close_ms_list: list[int] - the list of close times for the positions - """ - if len(lookback_fractions) == 0: - return 0 - - window_size = ValiConfig.HISTORICAL_PENALTY_WINDOW - stride = ValiConfig.HISTORICAL_PENALTY_STRIDE - - # Initialize variables - total_windows = int((1 - window_size) / stride) + 1 - represented_windows = 0 - - # Iterate over the sliding windows - for i in range(total_windows): - window_start = i * stride - window_end = window_start + window_size - - # Check if any lookback fraction falls within the current window - for fraction in lookback_fractions: - if window_start <= fraction < window_end: - represented_windows += 1 - break - - # Calculate the penalty score - penalty_score = represented_windows / total_windows - - if penalty_score >= 0.6: - return 1 - elif penalty_score >= 0.5: - return 0.9 - elif penalty_score >= 0.4: - return 0.8 - elif penalty_score >= 0.3: - return 0.5 - elif penalty_score >= 0.2: - return 0.25 - elif penalty_score >= 0.1: - return 0.1 - - return 0.1 - @staticmethod def flatten_positions( positions: dict[str, list[Position]] diff --git a/vali_objects/utils/subtensor_weight_setter.py b/vali_objects/utils/subtensor_weight_setter.py index 46d8c907d..bdf53009d 100644 --- a/vali_objects/utils/subtensor_weight_setter.py +++ b/vali_objects/utils/subtensor_weight_setter.py @@ -44,6 +44,11 @@ def set_weights(self, current_time: int = None): # only collect ledger elements for the miners that passed the challenge period filtered_ledger = self.filtered_ledger(hotkeys=success_hotkeys) + filtered_positions = self.position_manager.get_all_miner_positions_by_hotkey( + success_hotkeys, + sort_positions=True, + acceptable_position_end_ms=current_time - ValiConfig.SET_WEIGHT_LOOKBACK_RANGE_MS + ) if len(filtered_ledger) == 0: bt.logging.info("No returns to set weights with. Do nothing for now.") @@ -51,6 +56,7 @@ def set_weights(self, current_time: int = None): bt.logging.info("Calculating new subtensor weights...") checkpoint_results = Scoring.compute_results_checkpoint( filtered_ledger, + filtered_positions, evaluation_time_ms=current_time ) bt.logging.info(f"Sorted results for weight setting: [{sorted(checkpoint_results, key=lambda x: x[1], reverse=True)}]")