naive_routing_analysis.py

#!/usr/bin/env python3
"""
Naive (Random) Routing Analysis

Calculates the expected accuracy of a router that randomly selects 
a model for each prompt using probability theory.

For each prompt (original_id):
- If there are N models and K have performance=1.0
- Probability of selecting a correct model = K/N
- Expected accuracy = sum of all probabilities / total prompts
"""

import pandas as pd
import sys
import csv

csv.field_size_limit(sys.maxsize)

def analyze_naive_routing(csv_path: str) -> None:
    print(f"Loading data from: {csv_path}")
    
    # Read the CSV file
    df = pd.read_csv(csv_path)
    
    print(f"Total rows: {len(df)}")
    
    # Get unique original_ids
    unique_ids = df['original_id'].unique()
    total_prompts = len(unique_ids)
    print(f"Unique prompts (original_id): {total_prompts}")
    
    # Get unique models
    unique_models = df['llm'].unique()
    num_models = len(unique_models)
    print(f"Unique models: {num_models}")
    
    # Calculate expected accuracy using probability
    # For each prompt: P(correct) = (# correct models) / (# total models for that prompt)
    # Expected total correct = sum of P(correct) for all prompts
    # Expected accuracy = Expected total correct / total prompts
    
    expected_correct = 0.0
    probability_details = []
    
    for orig_id in unique_ids:
        group = df[df['original_id'] == orig_id]
        total_models_for_prompt = len(group)
        correct_models = (group['performance'] == 1.0).sum()
        
        # Probability of randomly selecting a correct model for this prompt
        p_correct = correct_models / total_models_for_prompt
        expected_correct += p_correct
        
        probability_details.append({
            'original_id': orig_id,
            'total_models': total_models_for_prompt,
            'correct_models': correct_models,
            'p_correct': p_correct
        })
    
    expected_accuracy = (expected_correct / total_prompts) * 100
    
    print("\n" + "="*50)
    print("NAIVE (RANDOM) ROUTING ANALYSIS RESULTS")
    print("="*50)
    print(f"Total unique prompts: {total_prompts}")
    print(f"Expected number of correct answers: {expected_correct:.2f}")
    print(f"\nNaive Router Expected Accuracy: {expected_accuracy:.2f}%")
    print("="*50)
    
    # Additional statistics
    prob_df = pd.DataFrame(probability_details)
    
    print("\nProbability Distribution Statistics:")
    print(f"  Mean P(correct) per prompt: {prob_df['p_correct'].mean():.4f}")
    print(f"  Std Dev P(correct): {prob_df['p_correct'].std():.4f}")
    print(f"  Min P(correct): {prob_df['p_correct'].min():.4f}")
    print(f"  Max P(correct): {prob_df['p_correct'].max():.4f}")
    
    # Distribution of P(correct) values
    print("\nDistribution of P(correct) by correct model count:")
    for k in sorted(prob_df['correct_models'].unique()):
        count = (prob_df['correct_models'] == k).sum()
        subset = prob_df[prob_df['correct_models'] == k]
        avg_p = subset['p_correct'].mean()
        print(f"  {k:2d} correct model(s): {count:3d} prompts, P(correct)={avg_p:.4f}")
    
    # Comparison summary
    print("\n" + "="*50)
    print("COMPARISON WITH ORACLE ROUTING")
    print("="*50)
    oracle_correct = (prob_df['correct_models'] > 0).sum()
    oracle_accuracy = (oracle_correct / total_prompts) * 100
    print(f"Oracle Router Accuracy: {oracle_accuracy:.2f}%")
    print(f"Naive Router Expected Accuracy: {expected_accuracy:.2f}%")
    print(f"Gap (Oracle - Naive): {oracle_accuracy - expected_accuracy:.2f}%")
    print("="*50)


if __name__ == "__main__":
    csv_path = sys.argv[1] if len(sys.argv) > 1 else "new_data/routerarena_irtrouter_format.csv"
    analyze_naive_routing(csv_path)