From 5be830da9d8b194ab80d84f5cba3543e9f8fb112 Mon Sep 17 00:00:00 2001 From: shivangi221b Date: Tue, 24 Feb 2026 20:31:45 -0500 Subject: [PATCH 1/2] consolidate eval scripts --- .gitignore | 1 + COMPREHENSIVE_DEMO.py | 2 +- README.md | 82 +-- docs/EVALUATION_SUMMARY.md | 304 +++++++---- docs/EVALUATION_VERIFICATION_SUMMARY.md | 150 ----- docs/QUICK_REFERENCE.md | 2 +- docs/SETUP_INSTRUCTIONS.md | 45 +- .../evaluation_outputs/evaluation_report.json | 152 +----- .../evaluation_outputs/evaluation_report.txt | 8 + experiments/kavya_evaluation_framework.py | 515 ------------------ experiments/run_benchmark.py | 39 -- experiments/run_comprehensive_evaluations.py | 426 --------------- experiments/run_evaluation.py | 321 +++++++++++ experiments/verify_evaluation_numbers.py | 181 ------ requirements.txt | 2 +- scripts/setup_gcp_gpu.sh | 2 +- src/agent/ollama_llm.py | 44 +- src/evaluation/metrics.py | 165 +++++- tests/test_metrics.py | 48 +- 19 files changed, 817 insertions(+), 1672 deletions(-) delete mode 100644 docs/EVALUATION_VERIFICATION_SUMMARY.md create mode 100644 experiments/evaluation_outputs/evaluation_report.txt delete mode 100644 experiments/kavya_evaluation_framework.py delete mode 100644 experiments/run_benchmark.py delete mode 100755 experiments/run_comprehensive_evaluations.py create mode 100644 experiments/run_evaluation.py delete mode 100644 experiments/verify_evaluation_numbers.py diff --git a/.gitignore b/.gitignore index 3cdabfd..d166729 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ ENV/ build/ dist/ *.egg-info/ +.venv # Data (never commit large datasets) data/raw/* diff --git a/COMPREHENSIVE_DEMO.py b/COMPREHENSIVE_DEMO.py index a0fb517..8ed9c59 100644 --- a/COMPREHENSIVE_DEMO.py +++ b/COMPREHENSIVE_DEMO.py @@ -228,7 +228,7 @@ def demo_evaluation_framework(): print_section("DEMO 5: Evaluation Framework") print("The evaluation framework can be run with:") - print(" python experiments/kavya_evaluation_framework.py") + print(" python experiments/run_evaluation.py --eval-set path/to/eval_set.json") print("\nIt provides:") print(" ✅ WER/CER calculation") print(" ✅ Error analysis") diff --git a/README.md b/README.md index 8e3d6ac..02cb37c 100644 --- a/README.md +++ b/README.md @@ -77,9 +77,8 @@ Adaptive-Self-Learning-Agentic-AI-System/ │ ├── test_agent.py # Test agent functionality │ ├── test_api.py # Test API endpoints │ ├── test_data_management.py # Test data management -│ ├── kavya_evaluation_framework.py # Comprehensive evaluation +│ ├── run_evaluation.py # Evaluate dataset on baseline + improved models │ ├── evaluate_models.py # Model evaluation -│ ├── run_benchmark.py # Performance benchmarking │ ├── visualize_evaluation_results.py # Generate charts │ └── example_usage.py # Usage examples │ @@ -267,27 +266,25 @@ system.record_training_performance( report = system.generate_comprehensive_report() ``` -### 4. Evaluation Framework (`experiments/kavya_evaluation_framework.py`) +### 4. Evaluation (`experiments/run_evaluation.py`) -Comprehensive evaluation with metrics and visualization. +Evaluate any dataset (audio + ground-truth references) on the baseline and on improved (fine-tuned) models if they exist. **Features:** -- WER/CER calculation -- Error analysis -- Performance benchmarking -- Visualization generation +- WER/CER for baseline (Whisper) and all fine-tuned Wav2Vec2 versions +- Optional latency/throughput benchmark (`--benchmark`) **Usage:** -```python -from experiments.kavya_evaluation_framework import EvaluationFramework +```bash +# Eval set file (JSON/JSONL/CSV with audio_path + reference) +python experiments/run_evaluation.py --eval-set path/to/eval_set.json -framework = EvaluationFramework(model_name="whisper") -results = framework.run_comprehensive_evaluation( - eval_datasets=["data/processed/test_dataset"], - output_report=True -) +# Or audio directory + refs file +python experiments/run_evaluation.py --audio-dir data/recordings_for_test --refs path/to/refs.json ``` +See **`docs/EVALUATION_SUMMARY.md`** for full options and examples. + ## 🌐 Running the System ### 1. Baseline API (Simple Transcription) @@ -342,37 +339,24 @@ curl -X POST "http://localhost:8000/transcribe" \ -F "file=@data/test_audio/test_1.wav" ``` -### 3. Evaluation & Benchmarking +### 3. Evaluation -#### Run Comprehensive Evaluation +#### Run evaluation (baseline + improved models) ```bash -cd experiments -python kavya_evaluation_framework.py +python experiments/run_evaluation.py --eval-set path/to/eval_set.json ``` -Output: -- `evaluation_outputs/evaluation_report.json` - Detailed results -- `evaluation_outputs/evaluation_summary.json` - Summary metrics -- `docs/EVALUATION_SUMMARY.md` - Human-readable report -- `evaluation_outputs/visualizations/` - Charts and graphs - -#### Run Benchmark Tests +With latency/throughput benchmark: ```bash -python experiments/run_benchmark.py +python experiments/run_evaluation.py --eval-set path/to/eval_set.json --benchmark ``` -Output: -- `evaluation_outputs/benchmark_report.json` - Performance metrics - -#### Visualize Results -```bash -python experiments/visualize_evaluation_results.py -``` +Output (in `experiments/evaluation_outputs/`): +- `evaluation_report.json` - Baseline and improved-model WER/CER +- `evaluation_report.txt` - Short summary +- `benchmark_report.json` - If `--benchmark` was used -Generates: -- WER/CER comparison charts -- Error distribution histograms -- Comprehensive dashboards +See **`docs/EVALUATION_SUMMARY.md`** for input format (eval set, audio-dir + refs, or --gold-from-llm) and all options. ### 4. Testing Components @@ -630,25 +614,15 @@ if user_correction: data_system.add_correction(case_id, user_correction) ``` -### Workflow 2: Model Evaluation & Comparison +### Workflow 2: Model Evaluation -```python -from experiments.kavya_evaluation_framework import EvaluationFramework - -# Evaluate baseline model -framework = EvaluationFramework(model_name="whisper") -results = framework.run_comprehensive_evaluation( - eval_datasets=["data/processed/test_dataset"] -) - -# Generate visualizations -framework.generate_visualizations() - -# Get metrics -print(f"WER: {results['overall_metrics']['mean_wer']:.4f}") -print(f"CER: {results['overall_metrics']['mean_cer']:.4f}") +```bash +# Run evaluation on a dataset (eval set = JSON/JSONL/CSV with audio_path + reference) +python experiments/run_evaluation.py --eval-set path/to/eval_set.json ``` +Then read `experiments/evaluation_outputs/evaluation_report.json` for baseline and improved-model WER/CER. See `docs/EVALUATION_SUMMARY.md` for details. + ### Workflow 3: Fine-tuning Pipeline ```python diff --git a/docs/EVALUATION_SUMMARY.md b/docs/EVALUATION_SUMMARY.md index e2b7dbd..6eab6b3 100644 --- a/docs/EVALUATION_SUMMARY.md +++ b/docs/EVALUATION_SUMMARY.md @@ -1,124 +1,198 @@ -# Evaluation Framework - Week 1 Deliverables Summary - -## ✅ All Required Outputs Generated Successfully - -### 📊 Generated Files - -#### 1. **Evaluation Reports (JSON)** -- ✅ `evaluation_report.json` - Complete detailed evaluation report with: - - Model information (Whisper-base, 72.6M parameters) - - Per-dataset metrics (WER, CER) - - Detailed results for each sample - - Error analysis - - Inference statistics - -- ✅ `evaluation_summary.json` - Summary metrics: - - Overall WER: 0.1000 (10%) - - Overall CER: 0.0227 (2.27%) - - Per-dataset breakdown - - Model metadata - -- ✅ `benchmark_report.json` - Performance benchmarks: - - Latency: Mean 0.72s, Std 0.61s - - Throughput: 2.97 samples/second - - Cost estimates: $180/month for 1.8M inferences - -#### 2. **Visualizations (PNG)** -- ✅ `wer_cer_comparison.png` - Bar chart comparing WER and CER across datasets -- ✅ `error_distribution.png` - Histogram showing distribution of WER across samples -- ✅ `evaluation_dashboard.png` - Comprehensive 4-panel dashboard with: - - WER by dataset - - CER by dataset - - Overall metrics summary - - Sample counts - -#### 3. **Additional Benchmark Output** -- ✅ `baseline_benchmark.json` - Standalone benchmark report from run_benchmark.py - -## 📈 Key Metrics Achieved - -### Model Performance -- **Model**: Whisper-base (openai/whisper-base) -- **Parameters**: 72,593,920 (72.6M) -- **Device**: CPU -- **WER**: 0.1000 (10% word error rate) -- **CER**: 0.0227 (2.27% character error rate) - -### Performance Benchmarks -- **Mean Latency**: 0.72 seconds per sample -- **Throughput**: 2.97 samples/second -- **Cost Estimate**: $1.80 per hour transcribed - -### Evaluation Coverage -- **Datasets Evaluated**: 1 (test_dataset) -- **Total Samples**: 2 test samples -- **Error Analysis**: Complete with worst errors identified - -## 🔧 Framework Components Verified - -### ✅ Core Evaluation Components -1. **STTEvaluator** - WER/CER calculation ✓ -2. **BaselineSTTModel** - Model inference wrapper ✓ -3. **BaselineBenchmark** - Performance benchmarking ✓ -4. **EvaluationFramework** - Comprehensive evaluation system ✓ - -### ✅ Data Pipeline -1. **Test Dataset Creation** - Created test dataset with ground truth ✓ -2. **Dataset Loading** - Successfully loads HuggingFace datasets ✓ -3. **Audio Processing** - Handles audio files correctly ✓ - -### ✅ Output Generation -1. **JSON Reports** - All required reports generated ✓ -2. **Visualizations** - All charts and dashboards created ✓ -3. **Error Analysis** - Detailed error breakdown included ✓ - -## 📝 Week 1 Deliverables Checklist - -- [x] Development environment setup -- [x] Dataset curation (test dataset created) -- [x] Preprocessing pipelines -- [x] **Evaluation framework implementation** -- [x] **WER/CER metrics calculation** -- [x] **Error analysis** -- [x] **Performance benchmarking (latency, throughput, cost)** -- [x] **Report generation (JSON)** -- [x] **Visualization generation (PNG charts)** -- [x] **Comprehensive evaluation outputs** - -## 🎯 Framework Capabilities Demonstrated - -1. **Multi-Dataset Evaluation** - Can evaluate on multiple datasets -2. **Comprehensive Metrics** - WER, CER, latency, throughput, cost -3. **Error Analysis** - Identifies worst errors and patterns -4. **Visualization** - Generates charts and dashboards -5. **Report Generation** - Creates detailed JSON reports -6. **Benchmarking** - Performance and cost analysis - -## 📂 File Structure +# Evaluation Summary +Single place for how to run evaluation, input options (including LLM as gold), outputs, verification, and the evaluation module. + +--- + +## Script: `experiments/run_evaluation.py` + +**Purpose:** Run any dataset on the **baseline (Whisper)** and on **improved (fine-tuned) models** if they exist. Supports three ways to get references (gold standard): from an eval set file, from a refs file, or **from the LLM** (Ollama). + +**Outputs:** WER/CER (and optional benchmark) in `experiments/evaluation_outputs/`. + +--- + +## Input: How to provide the dataset + +### 1. Evaluation set file (recommended when you have ground truth) + +A single file listing each audio path and its reference text. + +```bash +python experiments/run_evaluation.py --eval-set path/to/eval_set.json ``` -experiments/evaluation_outputs/ -├── benchmark_report.json # Performance benchmarks -├── evaluation_report.json # Full detailed report -├── evaluation_summary.json # Summary metrics -├── visualizations/ -│ ├── wer_cer_comparison.png # WER/CER comparison chart -│ ├── error_distribution.png # Error distribution histogram -│ └── evaluation_dashboard.png # Comprehensive dashboard -└── (see docs/EVALUATION_SUMMARY.md for this summary) + +**Formats:** JSON, JSONL, CSV. + +**Fields (any of these names):** Audio: `audio_path`, `audio`, or `path`. Reference: `reference`, `text`, or `target_text`. + +**Example JSON:** +```json +[ + {"audio_path": "data/test_audio/sample1.wav", "reference": "hello world"}, + {"audio_path": "data/test_audio/sample2.wav", "reference": "goodbye"} +] +``` + +### 2. Audio directory + refs file + +Discover WAV/MP3 in a directory and pair with references from a file. + +```bash +python experiments/run_evaluation.py --audio-dir data/recordings_for_test --refs path/to/refs.json ``` -## 🚀 Next Steps +- **`--audio-dir`:** Directory of `.wav` / `.mp3` (sorted by name). +- **`--refs`:** Same format as `--eval-set`. Matched by path or filename. Without `--refs`, WER/CER cannot be computed. -The evaluation framework is fully functional and ready for: -1. Scaling to larger datasets -2. Adding more evaluation metrics -3. Comparing multiple models -4. Integration with continuous evaluation pipeline +### 3. Audio directory with LLM as gold standard + +Point to a directory of audio files; the script uses the **LLM (Ollama)** as gold: it checks that Ollama is running, gets a baseline transcript for each file, then asks the LLM to correct it and uses that corrected text as the reference for WER/CER. + +```bash +python experiments/run_evaluation.py --audio-dir data/recordings_for_test --gold-from-llm +``` + +- **Requires:** Ollama running (e.g. `ollama serve`) and the model pulled (e.g. `ollama pull llama3.2:3b`). +- **Behavior:** Same LLM-gold logic as `scripts/finetune_wav2vec2.py`: baseline transcribes each file, then `LlamaLLMCorrector.correct_transcript(transcript, errors=[], context={})` gives the gold reference. +- **Optional:** `--llm-model llama3.2:3b` (default) to choose the Ollama model. + +If the LLM is not available, the script exits with a clear message to start Ollama and pull the model. --- -**Generated**: 2025-11-18 -**Framework Version**: Week 1 - Kavya Evaluation Framework -**Status**: ✅ All deliverables complete +## Output + +- **`experiments/evaluation_outputs/evaluation_report.json`** – Full report: baseline and improved-model WER/CER, `gold_source` (eval_set / refs / llm), optional benchmark. +- **`experiments/evaluation_outputs/evaluation_report.txt`** – Short summary. +- With **`--benchmark`**: **`experiments/evaluation_outputs/benchmark_report.json`** – Latency/throughput from baseline. + +--- + +## Options + +| Option | Description | +|--------|-------------| +| `--eval-set PATH` | Eval set file (JSON/JSONL/CSV) with audio_path + reference. | +| `--audio-dir DIR` | Directory of WAV/MP3. Use with `--refs` or `--gold-from-llm`. | +| `--refs PATH` | Refs file (same format as --eval-set). Ignored if --gold-from-llm. | +| `--gold-from-llm` | Use LLM (Ollama) as gold; requires --audio-dir. | +| `--llm-model NAME` | Ollama model for --gold-from-llm (default: llama3.2:3b). | +| `--output-dir DIR` | Output directory (default: experiments/evaluation_outputs). | +| `--baseline-only` | Run only baseline, skip improved models. | +| `--benchmark` | Include latency/throughput benchmark. | + +--- + +## Quick examples + +```bash +# Eval set file +python experiments/run_evaluation.py --eval-set data/eval_set.json + +# Audio dir + refs file +python experiments/run_evaluation.py --audio-dir data/recordings_for_test --refs data/refs.json + +# Audio dir + LLM as gold (Ollama must be running) +python experiments/run_evaluation.py --audio-dir data/recordings_for_test --gold-from-llm + +# Baseline only, with benchmark +python experiments/run_evaluation.py --eval-set data/eval_set.json --baseline-only --benchmark +``` + +--- + +## Verifying results + +There is **no separate verification script**. To verify or report evaluation numbers: + +1. **Run evaluation** with your chosen input (eval set, refs, or `--gold-from-llm`). +2. **Inspect the report:** Open `experiments/evaluation_outputs/evaluation_report.json` and check: + - `baseline.wer`, `baseline.cer` for baseline metrics + - `improved_models[*].wer`, `improved_models[*].cer` for fine-tuned models (if any) + - `gold_source` to see how references were obtained (`eval_set` / `refs` / `llm`) + - With `--benchmark`: latency/throughput in `benchmark_report.json` or inside the main report + +**For reports/papers:** Cite values from `evaluation_report.json` as the single source of truth. + +### What gets verified + +When you run `run_evaluation.py` with a proper eval set (audio + reference), the following are **measured**: + +- **Baseline WER/CER** – `baseline.wer`, `baseline.cer` +- **Improved model WER/CER** – `improved_models[*].wer`, `.cer` (if any improved models exist) +- **Model info** – `baseline.model_info` +- **Num samples** – `num_samples` +- **Latency/throughput** – If `--benchmark` was used + +### Numbers that require ground truth + +The following metrics require actual ground-truth reference transcripts (not LLM-generated) to verify: + +- **Full system performance** (if you have a full system with error detection/correction) +- **Ablation study results** (component-specific contributions) +- **Statistical significance** (p-values, Cohen's d, confidence intervals) +- **Error detection precision/recall** (requires known errors) + +**Note:** When using `--gold-from-llm`, the gold standard comes from LLM correction of baseline transcripts, which is useful for comparing model versions but not a substitute for human-verified ground truth for absolute accuracy claims. + +--- + +## Unified evaluation module (`src.evaluation.metrics`) + +Evaluation logic lives in one module: **streaming (inference)** and **batch/offline** test sets. + +### Components + +- **STTEvaluator** – Low-level WER/CER (single pair and batch). +- **EvaluationModule** – Unified API: + - **Streaming:** `add_prediction(reference, hypothesis)` then `get_metrics()`; per-sample in `.results`. + - **Batch:** `evaluate_batch(references, hypotheses)` or `evaluate_from_file(path)` for JSON/JSONL/CSV. + +### Example + +```python +from src.evaluation.metrics import EvaluationModule + +eval_mod = EvaluationModule() +for ref, hyp in stream_of_predictions: + eval_mod.add_prediction(ref, hyp) +metrics = eval_mod.get_metrics() # {"wer", "cer", "num_samples"} +``` + +--- + +## Other components + +- **BaselineSTTModel** – Loads Whisper or fine-tuned Wav2Vec2; used by `run_evaluation.py`. +- **LlamaLLMCorrector** (`src.agent.llm_corrector`) – Used for `--gold-from-llm`; same pattern as `scripts/finetune_wav2vec2.py`. +- **BaselineBenchmark** – Latency/throughput when you pass `--benchmark`. +- **Model versioning** – `get_all_model_versions()` / `get_current_model_path()` to discover improved models under `models/`. +- **scripts/finetune_wav2vec2.py** – Fine-tuning script; also runs baseline vs fine-tuned evaluation with LLM gold on its own test set. +- **src/data/model_validator.py** – Library to compare a model vs baseline on an evaluation set. +- **experiments/test_baseline.py** – Smoke test (load baseline, one inference). + +--- + +## File structure (after a run) + +``` +experiments/evaluation_outputs/ +├── evaluation_report.json # Full report (baseline + improved models, gold_source) +├── evaluation_report.txt # Short summary +└── benchmark_report.json # If --benchmark was used +``` + +--- + +## Report metrics reference + +| Metric | Source in report | +|--------|------------------| +| Baseline WER / CER | `evaluation_report.json` → `baseline.wer`, `baseline.cer` | +| Improved model WER / CER | `evaluation_report.json` → `improved_models[*].wer`, `.cer` | +| Latency / throughput | `benchmark_report.json` (when using `--benchmark`) | +| Num samples | `evaluation_report.json` → `num_samples` | +| Gold source | `evaluation_report.json` → `gold_source` | +**Conclusion:** Run `experiments/run_evaluation.py` with a proper eval set (audio + reference) to obtain verified metrics; use the generated report as the single source of truth for baseline and improved models. diff --git a/docs/EVALUATION_VERIFICATION_SUMMARY.md b/docs/EVALUATION_VERIFICATION_SUMMARY.md deleted file mode 100644 index c2e2ac4..0000000 --- a/docs/EVALUATION_VERIFICATION_SUMMARY.md +++ /dev/null @@ -1,150 +0,0 @@ -# Evaluation Numbers Verification Summary - -**Date**: December 2024 -**Status**: Verification Complete - ---- - -## Purpose and Context - -This document tracks the **verification process** for quantitative metrics reported in our research paper (`report.md`). The verification process ensures scientific accuracy by: - -1. **Cross-referencing reported values** against actual evaluation outputs (JSON files from `experiments/evaluation_outputs/`) -2. **Identifying discrepancies** between initial estimates and measured values -3. **Distinguishing verified metrics** (from actual evaluation runs) from **estimated metrics** (theoretical/expected based on component analysis) -4. **Documenting limitations** (e.g., lack of ground truth data) that prevent full verification -5. **Providing transparency** about which numbers in the report are measured vs. estimated - -**Why This Matters**: In academic/research contexts, distinguishing between verified measurements and theoretical estimates is essential for reproducibility and credibility. This document serves as an audit trail for the evaluation numbers in our report, ensuring readers can trust baseline metrics while understanding where improvements are estimated rather than measured. - ---- - -## ✅ VERIFIED NUMBERS (From Actual Evaluation Files) - -### Baseline Model Performance -- **WER**: 10.0% (0.1000) - ✅ Verified from `evaluation_summary.json` -- **CER**: 2.27% (0.0227) - ✅ Verified from `evaluation_summary.json` -- **Model Parameters**: 72,593,920 (72.6M) - ✅ Verified -- **Mean Latency**: 5.29 seconds - ✅ Verified from `benchmark_report.json` -- **Throughput**: 2.65 samples/second - ✅ Verified from `benchmark_report.json` -- **Device**: CPU - ✅ Verified - -### Evaluation Dataset -- **Total Samples Evaluated**: 2 samples (from evaluation_summary.json) -- **Note**: Small sample size limits statistical power - ---- - -## ⚠️ NUMBERS UPDATED IN REPORT - -### Fixed Discrepancies: -1. **Latency**: Updated from 0.72s → **5.29s** (actual measured value) -2. **Throughput**: Updated from 2.97 → **2.65 samples/s** (actual measured value) -3. **Baseline WER in comparison table**: Updated from 25-30% → **10.0%** (actual measured value) - ---- - -## 📊 NUMBERS THAT REQUIRE GROUND TRUTH DATA - -The following numbers in the report require actual ground truth reference transcripts to verify: - -### Full System Performance -- Full system WER (currently estimated as 8.0-9.0%) -- Full system CER (currently estimated as 1.8-2.0%) -- Error detection precision/recall -- Correction success rates - -### Ablation Study Results -- Component-specific WER contributions -- Configuration-specific performance metrics - -### Statistical Analysis -- Paired t-test p-values -- Cohen's d effect sizes -- Confidence intervals - -### Why These Need Ground Truth: -- Current evaluation uses baseline transcription as reference (WER = 0%) -- Need actual human-verified transcripts to measure real improvements -- Error detection and correction metrics require known errors - ---- - -## 🔧 HOW TO VERIFY REMAINING NUMBERS - -### Option 1: Use Existing Ground Truth Dataset -```python -# If you have a dataset with ground truth transcripts: -from src.integration import UnifiedSTTSystem - -system = UnifiedSTTSystem() -results = system.evaluate_batch( - audio_files=["audio1.wav", "audio2.wav"], - reference_transcripts=["ground truth 1", "ground truth 2"] -) -``` - -### Option 2: Create Synthetic Test Cases -```python -# Create test cases with known errors: -# 1. Transcribe audio with baseline -# 2. Introduce known errors -# 3. Use as reference -# 4. Measure correction effectiveness -``` - -### Option 3: Use Public STT Datasets -- LibriSpeech -- Common Voice -- TIMIT -- Any dataset with ground truth transcripts - ---- - -## 📝 REPORT STATUS - -### ✅ Verified Sections: -- Baseline model performance (WER, CER, latency, throughput) -- Model parameters and configuration -- Evaluation framework description - -### ⚠️ Estimated/Theoretical Sections: -- Full system performance improvements -- Ablation study results -- Statistical significance values -- Component contributions -- Error detection metrics - -### 📌 Notes Added to Report: -- Italicized notes indicating verified vs estimated numbers -- Disclaimers about dataset limitations -- Framework capabilities vs actual measured results - ---- - -## 🎯 RECOMMENDATIONS - -1. **For Full Verification**: Obtain ground truth transcripts for test audio files -2. **For Report**: Current report accurately reflects verified baseline metrics -3. **For Future Work**: Run comprehensive evaluation with ground truth when available -4. **For Presentation**: Clearly distinguish between verified metrics and theoretical estimates - ---- - -## 📊 ACTUAL MEASURED VALUES SUMMARY - -| Metric | Verified Value | Source | -|--------|---------------|--------| -| Baseline WER | 10.0% | evaluation_summary.json | -| Baseline CER | 2.27% | evaluation_summary.json | -| Mean Latency | 5.29s | benchmark_report.json | -| Throughput | 2.65 samples/s | benchmark_report.json | -| Model Params | 72.6M | evaluation_summary.json | -| Device | CPU | evaluation_summary.json | -| Samples Evaluated | 2 | evaluation_summary.json | - ---- - -**Conclusion**: Baseline metrics are verified and accurate. Full system improvements require ground truth data for proper evaluation. - - diff --git a/docs/QUICK_REFERENCE.md b/docs/QUICK_REFERENCE.md index e2a7c19..b5ed848 100644 --- a/docs/QUICK_REFERENCE.md +++ b/docs/QUICK_REFERENCE.md @@ -38,7 +38,7 @@ curl "http://localhost:8000/agent/stats" python experiments/test_baseline.py python experiments/test_agent.py python experiments/test_data_management.py -python experiments/kavya_evaluation_framework.py +python experiments/run_evaluation.py --eval-set path/to/eval_set.json pytest tests/ ``` diff --git a/docs/SETUP_INSTRUCTIONS.md b/docs/SETUP_INSTRUCTIONS.md index 2d2b314..eac14c3 100644 --- a/docs/SETUP_INSTRUCTIONS.md +++ b/docs/SETUP_INSTRUCTIONS.md @@ -300,33 +300,16 @@ print(f"Total cases: {stats['data_management']['total_failed_cases']}") print(f"Correction rate: {stats['data_management']['correction_rate']:.1%}") ``` -### 4. Evaluation Framework Setup +### 4. Evaluation Setup -Set up and run comprehensive evaluation: +Run evaluation with an evaluation set file (JSON/JSONL/CSV with `audio_path` and `reference`): -```python -from experiments.kavya_evaluation_framework import EvaluationFramework - -# Initialize framework -framework = EvaluationFramework( - model_name="whisper", - output_dir="experiments/evaluation_outputs" -) - -# Run evaluation (requires test dataset) -results = framework.run_comprehensive_evaluation( - eval_datasets=["data/processed/test_dataset"], - output_report=True, - generate_visualizations=True -) - -# Results saved to: -# - experiments/evaluation_outputs/evaluation_report.json -# - experiments/evaluation_outputs/evaluation_summary.json -# - experiments/evaluation_outputs/benchmark_report.json -# - experiments/evaluation_outputs/visualizations/*.png +```bash +python experiments/run_evaluation.py --eval-set path/to/eval_set.json ``` +Results are saved to `experiments/evaluation_outputs/` (evaluation_report.json, evaluation_report.txt; use `--benchmark` for benchmark_report.json). See **docs/EVALUATION_SUMMARY.md** for full options. + ## 🏃 Running the System ### Mode 1: API Server (Production) @@ -521,23 +504,19 @@ python experiments/test_api.py pkill -f "uvicorn src.agent_api:app" ``` -### Run Evaluation & Benchmarks +### Run Evaluation -**Comprehensive Evaluation:** +**Evaluation (baseline + improved models):** ```bash -cd experiments -python kavya_evaluation_framework.py +python experiments/run_evaluation.py --eval-set path/to/eval_set.json ``` -**Performance Benchmarking:** +With latency/throughput benchmark: ```bash -python experiments/run_benchmark.py +python experiments/run_evaluation.py --eval-set path/to/eval_set.json --benchmark ``` -**Generate Visualizations:** -```bash -python experiments/visualize_evaluation_results.py -``` +See **docs/EVALUATION_SUMMARY.md** for input format and options. ## ☁️ GCP Setup (Optional) diff --git a/experiments/evaluation_outputs/evaluation_report.json b/experiments/evaluation_outputs/evaluation_report.json index 8df689b..9db8fc4 100644 --- a/experiments/evaluation_outputs/evaluation_report.json +++ b/experiments/evaluation_outputs/evaluation_report.json @@ -1,146 +1,22 @@ { - "summary": { - "model": "whisper", + "num_samples": 12, + "baseline": { + "model_id": "whisper", "model_info": { "name": "whisper", "parameters": 72593920, "device": "cpu", - "trainable_params": 71825920 + "framework": "pytorch", + "trainable_params": 71825920, + "model_path": "openai/whisper-base", + "is_finetuned": false }, - "evaluation_date": "2025-11-18T12:04:35.094231", - "total_datasets": 1, - "total_samples_evaluated": 2, - "overall_metrics": { - "mean_wer": 0.1, - "std_wer": 0.0, - "mean_cer": 0.022727272727272728, - "std_cer": 0.0, - "best_wer": 0.1, - "worst_wer": 0.1 - }, - "per_dataset_metrics": { - "test_dataset": { - "wer": 0.1, - "cer": 0.022727272727272728, - "num_samples": 2 - } - } - }, - "detailed_results": { - "test_dataset": { - "metrics": { - "wer": 0.1, - "cer": 0.022727272727272728, - "num_samples": 2, - "num_errors": 0, - "inference_stats": { - "mean_time_seconds": 0.8749442100524902, - "std_time_seconds": 0.5617508888244629, - "min_time_seconds": 0.31319332122802734, - "max_time_seconds": 1.4366950988769531, - "total_time_seconds": 1.7498884201049805 - } - }, - "references": [ - "add the sum to the product of these three", - "you" - ], - "hypotheses": [ - " add the sum to the product of these three.", - " you" - ], - "errors": [], - "error_analysis": { - "worst_errors": [ - { - "reference": "add the sum to the product of these three", - "hypothesis": " add the sum to the product of these three.", - "wer": 0.1111111111111111, - "cer": 0.024390243902439025, - "ref_length": 9, - "hyp_length": 9, - "length_diff": 0 - }, - { - "reference": "you", - "hypothesis": " you", - "wer": 0.0, - "cer": 0.0, - "ref_length": 1, - "hyp_length": 1, - "length_diff": 0 - } - ], - "error_statistics": { - "high_wer_count": 0, - "medium_wer_count": 0, - "low_wer_count": 2, - "avg_ref_length": 5.0, - "avg_hyp_length": 5.0, - "avg_length_diff": 0.0 - } - } - } + "wer": 0.12987012987012986, + "cer": 0.10275689223057644, + "num_samples": 12, + "latency_mean_sec": 0.1947083274523417 }, - "full_evaluation": { - "model_info": { - "name": "whisper", - "parameters": 72593920, - "device": "cpu", - "trainable_params": 71825920 - }, - "evaluation_date": "2025-11-18T12:04:33.299511", - "datasets": { - "test_dataset": { - "metrics": { - "wer": 0.1, - "cer": 0.022727272727272728, - "num_samples": 2, - "num_errors": 0, - "inference_stats": { - "mean_time_seconds": 0.8749442100524902, - "std_time_seconds": 0.5617508888244629, - "min_time_seconds": 0.31319332122802734, - "max_time_seconds": 1.4366950988769531, - "total_time_seconds": 1.7498884201049805 - } - }, - "error_analysis_summary": { - "high_wer_count": 0, - "medium_wer_count": 0, - "low_wer_count": 2, - "avg_ref_length": 5.0, - "avg_hyp_length": 5.0, - "avg_length_diff": 0.0 - } - } - }, - "summary": { - "model": "whisper", - "model_info": { - "name": "whisper", - "parameters": 72593920, - "device": "cpu", - "trainable_params": 71825920 - }, - "evaluation_date": "2025-11-18T12:04:35.094231", - "total_datasets": 1, - "total_samples_evaluated": 2, - "overall_metrics": { - "mean_wer": 0.1, - "std_wer": 0.0, - "mean_cer": 0.022727272727272728, - "std_cer": 0.0, - "best_wer": 0.1, - "worst_wer": 0.1 - }, - "per_dataset_metrics": { - "test_dataset": { - "wer": 0.1, - "cer": 0.022727272727272728, - "num_samples": 2 - } - } - } - } + "improved_models": [], + "timestamp": "2026-02-24T20:26:19", + "gold_source": "llm" } \ No newline at end of file diff --git a/experiments/evaluation_outputs/evaluation_report.txt b/experiments/evaluation_outputs/evaluation_report.txt new file mode 100644 index 0000000..4692ce4 --- /dev/null +++ b/experiments/evaluation_outputs/evaluation_report.txt @@ -0,0 +1,8 @@ +============================================================ +EVALUATION REPORT +Samples: 12 + +Baseline (Whisper) + WER: 0.1299 CER: 0.1028 + +============================================================ \ No newline at end of file diff --git a/experiments/kavya_evaluation_framework.py b/experiments/kavya_evaluation_framework.py deleted file mode 100644 index aa2460a..0000000 --- a/experiments/kavya_evaluation_framework.py +++ /dev/null @@ -1,515 +0,0 @@ -""" -Kavya Evaluation Framework - Week 1 -Comprehensive evaluation framework for STT baseline models -Generates all relevant outputs for Week 1 deliverables -""" - -import sys -import json -import time -from pathlib import Path -from typing import List, Dict, Tuple, Optional -from datetime import datetime -import numpy as np -import pandas as pd -from tqdm import tqdm -from datasets import load_from_disk, Dataset - -# Add src to path -sys.path.append(str(Path(__file__).parent.parent)) -from src.baseline_model import BaselineSTTModel -from src.evaluation.metrics import STTEvaluator -from src.benchmark import BaselineBenchmark -import logging - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -class EvaluationFramework: - """ - Comprehensive evaluation framework for STT models. - Handles dataset evaluation, error analysis, and report generation. - """ - - def __init__(self, model_name: str = "whisper", output_dir: str = "experiments/evaluation_outputs"): - """ - Initialize evaluation framework. - - Args: - model_name: Name of model to evaluate - output_dir: Directory to save evaluation outputs - """ - self.model_name = model_name - self.output_dir = Path(output_dir) - self.output_dir.mkdir(parents=True, exist_ok=True) - - # Initialize components - logger.info(f"Loading {model_name} model...") - self.model = BaselineSTTModel(model_name=model_name) - self.evaluator = STTEvaluator() - self.benchmark = BaselineBenchmark(model_name=model_name) - - # Results storage - self.evaluation_results = { - "model_info": self.model.get_model_info(), - "evaluation_date": datetime.now().isoformat(), - "datasets": {}, - "summary": {} - } - - def evaluate_dataset( - self, - dataset_path: str, - split: str = "test", - max_samples: Optional[int] = None, - audio_column: str = "audio", - text_column: str = "text" - ) -> Dict: - """ - Evaluate model on a dataset split. - - Args: - dataset_path: Path to dataset - split: Dataset split to evaluate (train/dev/test) - max_samples: Maximum number of samples to evaluate (None for all) - audio_column: Name of audio column in dataset - text_column: Name of text column in dataset - - Returns: - Dictionary with evaluation results - """ - logger.info(f"Loading dataset from {dataset_path} (split: {split})...") - - try: - # Load dataset - dataset = load_from_disk(dataset_path) - - # Handle DatasetDict vs Dataset - if isinstance(dataset, dict): - if split not in dataset: - logger.warning(f"Split '{split}' not found. Available: {list(dataset.keys())}") - split = list(dataset.keys())[0] - dataset = dataset[split] - - # Limit samples if specified - if max_samples and len(dataset) > max_samples: - logger.info(f"Limiting evaluation to {max_samples} samples") - dataset = dataset.select(range(max_samples)) - - logger.info(f"Evaluating on {len(dataset)} samples...") - - # Run evaluation - references = [] - hypotheses = [] - inference_times = [] - errors = [] - - for idx, sample in enumerate(tqdm(dataset, desc="Evaluating")): - try: - # Get reference text - reference = sample.get(text_column, "") - if not reference: - logger.warning(f"Sample {idx} missing text column") - continue - - # Get audio - audio_data = sample.get(audio_column) - if audio_data is None: - logger.warning(f"Sample {idx} missing audio column") - continue - - # Save audio temporarily if it's an Audio object - import tempfile - import soundfile as sf - - with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp: - if hasattr(audio_data, 'array'): - # HuggingFace Audio object - sf.write(tmp.name, audio_data['array'], audio_data['sampling_rate']) - elif isinstance(audio_data, dict): - sf.write(tmp.name, audio_data['array'], audio_data['sampling_rate']) - else: - # Assume it's a path - tmp.name = audio_data - - # Transcribe - start_time = time.time() - result = self.model.transcribe(tmp.name) - inference_time = time.time() - start_time - - # Cleanup - import os - if os.path.exists(tmp.name): - os.remove(tmp.name) - - hypothesis = result['transcript'] - - references.append(reference) - hypotheses.append(hypothesis) - inference_times.append(inference_time) - - except Exception as e: - logger.error(f"Error processing sample {idx}: {e}") - errors.append({"sample_idx": idx, "error": str(e)}) - continue - - # Calculate metrics - if len(references) == 0: - logger.error("No valid samples processed") - return {} - - metrics = self.evaluator.evaluate_batch(references, hypotheses) - - # Add inference statistics - metrics.update({ - "num_samples": len(references), - "num_errors": len(errors), - "inference_stats": { - "mean_time_seconds": np.mean(inference_times), - "std_time_seconds": np.std(inference_times), - "min_time_seconds": np.min(inference_times), - "max_time_seconds": np.max(inference_times), - "total_time_seconds": sum(inference_times) - } - }) - - logger.info(f"Evaluation complete: WER={metrics['wer']:.4f}, CER={metrics['cer']:.4f}") - - return { - "metrics": metrics, - "references": references, - "hypotheses": hypotheses, - "errors": errors - } - - except Exception as e: - logger.error(f"Error evaluating dataset: {e}") - return {} - - def perform_error_analysis( - self, - references: List[str], - hypotheses: List[str], - top_n: int = 20 - ) -> Dict: - """ - Perform detailed error analysis on predictions. - - Args: - references: List of reference transcriptions - hypotheses: List of predicted transcriptions - top_n: Number of worst errors to analyze - - Returns: - Dictionary with error analysis results - """ - logger.info("Performing error analysis...") - - # Calculate per-sample errors - sample_errors = [] - for ref, hyp in zip(references, hypotheses): - wer = self.evaluator.calculate_wer(ref, hyp) - cer = self.evaluator.calculate_cer(ref, hyp) - sample_errors.append({ - "reference": ref, - "hypothesis": hyp, - "wer": wer, - "cer": cer, - "ref_length": len(ref.split()), - "hyp_length": len(hyp.split()), - "length_diff": len(hyp.split()) - len(ref.split()) - }) - - # Sort by WER (worst first) - sample_errors.sort(key=lambda x: x['wer'], reverse=True) - - # Analyze error patterns - error_analysis = { - "worst_errors": sample_errors[:top_n], - "error_statistics": { - "high_wer_count": sum(1 for e in sample_errors if e['wer'] > 0.5), - "medium_wer_count": sum(1 for e in sample_errors if 0.2 < e['wer'] <= 0.5), - "low_wer_count": sum(1 for e in sample_errors if e['wer'] <= 0.2), - "avg_ref_length": np.mean([e['ref_length'] for e in sample_errors]), - "avg_hyp_length": np.mean([e['hyp_length'] for e in sample_errors]), - "avg_length_diff": np.mean([e['length_diff'] for e in sample_errors]) - } - } - - return error_analysis - - def evaluate_multiple_datasets( - self, - dataset_configs: List[Dict], - max_samples_per_dataset: Optional[int] = 100 - ) -> Dict: - """ - Evaluate model on multiple datasets. - - Args: - dataset_configs: List of dataset configuration dicts - max_samples_per_dataset: Max samples to evaluate per dataset - - Returns: - Dictionary with results for all datasets - """ - logger.info(f"Evaluating on {len(dataset_configs)} datasets...") - - all_results = {} - - for config in dataset_configs: - dataset_name = config.get("name", "unknown") - dataset_path = config.get("path") - split = config.get("split", "test") - - if not dataset_path or not Path(dataset_path).exists(): - logger.warning(f"Dataset {dataset_name} not found at {dataset_path}") - continue - - logger.info(f"\n{'='*60}") - logger.info(f"Evaluating {dataset_name}") - logger.info(f"{'='*60}") - - results = self.evaluate_dataset( - dataset_path=dataset_path, - split=split, - max_samples=max_samples_per_dataset, - audio_column=config.get("audio_column", "audio"), - text_column=config.get("text_column", "text") - ) - - if results: - # Perform error analysis - if "references" in results and "hypotheses" in results: - error_analysis = self.perform_error_analysis( - results["references"], - results["hypotheses"] - ) - results["error_analysis"] = error_analysis - - all_results[dataset_name] = results - self.evaluation_results["datasets"][dataset_name] = { - "metrics": results["metrics"], - "error_analysis_summary": results.get("error_analysis", {}).get("error_statistics", {}) - } - - return all_results - - def generate_summary_report(self, all_results: Dict) -> Dict: - """ - Generate summary report across all datasets. - - Args: - all_results: Results from evaluate_multiple_datasets - - Returns: - Summary report dictionary - """ - logger.info("Generating summary report...") - - # Aggregate metrics - all_wers = [] - all_cers = [] - total_samples = 0 - - for dataset_name, results in all_results.items(): - if "metrics" in results: - metrics = results["metrics"] - all_wers.append(metrics["wer"]) - all_cers.append(metrics["cer"]) - total_samples += metrics.get("num_samples", 0) - - summary = { - "model": self.model_name, - "model_info": self.model.get_model_info(), - "evaluation_date": datetime.now().isoformat(), - "total_datasets": len(all_results), - "total_samples_evaluated": total_samples, - "overall_metrics": { - "mean_wer": np.mean(all_wers) if all_wers else None, - "std_wer": np.std(all_wers) if all_wers else None, - "mean_cer": np.mean(all_cers) if all_cers else None, - "std_cer": np.std(all_cers) if all_cers else None, - "best_wer": np.min(all_wers) if all_wers else None, - "worst_wer": np.max(all_wers) if all_wers else None - }, - "per_dataset_metrics": { - name: { - "wer": results["metrics"]["wer"], - "cer": results["metrics"]["cer"], - "num_samples": results["metrics"]["num_samples"] - } - for name, results in all_results.items() - if "metrics" in results - } - } - - self.evaluation_results["summary"] = summary - return summary - - def save_evaluation_report( - self, - all_results: Dict, - summary: Dict, - filename: str = "evaluation_report.json" - ): - """ - Save comprehensive evaluation report. - - Args: - all_results: Detailed results from all datasets - summary: Summary report - filename: Output filename - """ - report = { - "summary": summary, - "detailed_results": all_results, - "full_evaluation": self.evaluation_results - } - - output_path = self.output_dir / filename - with open(output_path, 'w') as f: - json.dump(report, f, indent=2) - - logger.info(f"✅ Evaluation report saved to {output_path}") - - # Also save summary as separate file - summary_path = self.output_dir / "evaluation_summary.json" - with open(summary_path, 'w') as f: - json.dump(summary, f, indent=2) - - logger.info(f"✅ Summary saved to {summary_path}") - - def print_evaluation_summary(self, summary: Dict): - """ - Print formatted evaluation summary. - - Args: - summary: Summary report dictionary - """ - print("\n" + "="*70) - print("EVALUATION FRAMEWORK SUMMARY REPORT") - print("="*70) - - print(f"\nModel: {summary['model']}") - print(f"Evaluation Date: {summary['evaluation_date']}") - print(f"Total Datasets: {summary['total_datasets']}") - print(f"Total Samples Evaluated: {summary['total_samples_evaluated']}") - - if summary['overall_metrics']['mean_wer'] is not None: - print("\n" + "-"*70) - print("OVERALL METRICS") - print("-"*70) - print(f"Mean WER: {summary['overall_metrics']['mean_wer']:.4f} ± {summary['overall_metrics']['std_wer']:.4f}") - print(f"Mean CER: {summary['overall_metrics']['mean_cer']:.4f} ± {summary['overall_metrics']['std_cer']:.4f}") - print(f"Best WER: {summary['overall_metrics']['best_wer']:.4f}") - print(f"Worst WER: {summary['overall_metrics']['worst_wer']:.4f}") - - if summary['per_dataset_metrics']: - print("\n" + "-"*70) - print("PER-DATASET METRICS") - print("-"*70) - for dataset_name, metrics in summary['per_dataset_metrics'].items(): - print(f"\n{dataset_name}:") - print(f" WER: {metrics['wer']:.4f}") - print(f" CER: {metrics['cer']:.4f}") - print(f" Samples: {metrics['num_samples']}") - - print("\n" + "="*70) - - -def main(): - """ - Main evaluation framework execution. - """ - print("="*70) - print("KAVYA EVALUATION FRAMEWORK - WEEK 1") - print("="*70) - - # Initialize framework - framework = EvaluationFramework( - model_name="whisper", - output_dir="experiments/evaluation_outputs" - ) - - # Define dataset configurations - # Note: Update these paths based on your actual dataset locations - dataset_configs = [ - { - "name": "test_dataset", - "path": "data/evaluation/test_dataset", - "split": "test", - "audio_column": "audio", - "text_column": "text" - }, - { - "name": "common_voice_test", - "path": "data/evaluation/common_voice_accents", - "split": "test", - "audio_column": "audio", - "text_column": "sentence" - }, - { - "name": "librispeech_test", - "path": "data/evaluation/librispeech_clean", - "split": "test", - "audio_column": "audio", - "text_column": "text" - } - ] - - # Filter to only existing datasets - existing_configs = [ - config for config in dataset_configs - if Path(config["path"]).exists() - ] - - if not existing_configs: - logger.warning("No evaluation datasets found. Using test audio for demonstration...") - # Fallback: evaluate on test audio files - test_audio_files = [ - "data/test_audio/addf8-Alaw-GW.wav", - "data/data/test_audio/test_1.wav" - ] - - # Create a simple evaluation with test files - print("\nRunning benchmark on test audio files...") - benchmark_report = framework.benchmark.generate_report( - [f for f in test_audio_files if Path(f).exists()] - ) - - # Save benchmark report - framework.benchmark.save_report( - benchmark_report, - str(framework.output_dir / "benchmark_report.json") - ) - - print("\n✅ Benchmark complete. Check evaluation_outputs/ for results.") - return - - # Evaluate on all datasets - all_results = framework.evaluate_multiple_datasets( - dataset_configs=existing_configs, - max_samples_per_dataset=100 # Adjust as needed - ) - - # Generate summary - summary = framework.generate_summary_report(all_results) - - # Print summary - framework.print_evaluation_summary(summary) - - # Save reports - framework.save_evaluation_report(all_results, summary) - - print("\n✅ Evaluation framework complete!") - print(f"📁 All outputs saved to: {framework.output_dir}") - print("\nGenerated files:") - print(" - evaluation_report.json (full detailed report)") - print(" - evaluation_summary.json (summary metrics)") - print(" - benchmark_report.json (performance benchmarks)") - - -if __name__ == "__main__": - main() diff --git a/experiments/run_benchmark.py b/experiments/run_benchmark.py deleted file mode 100644 index 6d978a0..0000000 --- a/experiments/run_benchmark.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -Task 4: Benchmark the baseline model -""" - -import sys -from pathlib import Path -sys.path.append(str(Path(__file__).parent.parent)) - -from src.benchmark import BaselineBenchmark -import os -import json - -if __name__ == "__main__": - print("=" * 50) - print("TASK 4: Baseline Benchmarking") - print("=" * 50) - - # Use test audio - test_audio_files = ["data/test_audio/test_1.wav"] - - if not all(os.path.exists(f) for f in test_audio_files): - print("⚠️ Test audio files not found. Run Task 1 first.") - exit(1) - - benchmark = BaselineBenchmark(model_name="whisper") - - print("\n📊 Running benchmarks (this may take a few minutes)...\n") - report = benchmark.generate_report(test_audio_files) - - print("\n✅ BENCHMARK REPORT:") - print("-" * 50) - print(json.dumps(report, indent=2)) - - # Save report - benchmark.save_report(report, "baseline_benchmark.json") - - print("\n" + "=" * 50) - print("✅ Benchmarking complete!") - print("=" * 50) diff --git a/experiments/run_comprehensive_evaluations.py b/experiments/run_comprehensive_evaluations.py deleted file mode 100755 index c29caee..0000000 --- a/experiments/run_comprehensive_evaluations.py +++ /dev/null @@ -1,426 +0,0 @@ -#!/usr/bin/env python3 -""" -Comprehensive Evaluation Script -Runs actual evaluations to get measured numbers for the report. -""" - -import sys -from pathlib import Path -import json -import logging -from typing import List, Dict, Optional -import numpy as np -from datetime import datetime -import time - -# Add src to path -sys.path.insert(0, str(Path(__file__).parent)) - -from src.baseline_model import BaselineSTTModel -from src.integration import UnifiedSTTSystem, StatisticalAnalyzer, AblationStudy -from src.evaluation.metrics import STTEvaluator - -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger(__name__) - - -class ComprehensiveEvaluator: - """Comprehensive evaluator that runs all evaluation types.""" - - def __init__(self, output_dir: str = "experiments/evaluation_results"): - self.output_dir = Path(output_dir) - self.output_dir.mkdir(parents=True, exist_ok=True) - self.results = {} - - def find_test_audio_files(self) -> List[str]: - """Find available test audio files.""" - audio_files = [] - - # Check test_audio directory - test_audio_dir = Path("data/test_audio") - if test_audio_dir.exists(): - audio_files.extend(list(test_audio_dir.glob("*.wav"))) - - # Check recordings_for_test directory - recordings_dir = Path("data/recordings_for_test") - if recordings_dir.exists(): - audio_files.extend(list(recordings_dir.glob("*.wav"))[:5]) # Limit to 5 for speed - - return [str(f) for f in audio_files if f.exists()] - - def create_reference_transcripts(self, audio_files: List[str]) -> List[str]: - """Create placeholder reference transcripts for evaluation.""" - # In real scenario, these would be actual ground truth transcripts - # For now, we'll use baseline transcription as reference - logger.info("Creating reference transcripts from baseline model...") - baseline = BaselineSTTModel(model_name="whisper") - references = [] - - for audio_file in audio_files: - try: - result = baseline.transcribe(audio_file) - references.append(result.get('transcript', '')) - logger.info(f" Created reference for {Path(audio_file).name}") - except Exception as e: - logger.warning(f" Failed to transcribe {audio_file}: {e}") - references.append("") # Empty reference - - return references - - def evaluate_baseline(self, audio_files: List[str], references: List[str]) -> Dict: - """Evaluate baseline model.""" - logger.info("="*70) - logger.info("EVALUATING BASELINE MODEL") - logger.info("="*70) - - baseline = BaselineSTTModel(model_name="whisper") - evaluator = STTEvaluator() - - wers = [] - cers = [] - latencies = [] - - for i, (audio_file, reference) in enumerate(zip(audio_files, references)): - if not reference.strip(): - continue - - logger.info(f"Processing {i+1}/{len(audio_files)}: {Path(audio_file).name}") - - start_time = time.time() - result = baseline.transcribe(audio_file) - latency = time.time() - start_time - - transcript = result.get('transcript', '') - if transcript: - wer = evaluator.calculate_wer(reference, transcript) - cer = evaluator.calculate_cer(reference, transcript) - wers.append(wer) - cers.append(cer) - latencies.append(latency) - - results = { - 'num_samples': len(wers), - 'wer': { - 'mean': np.mean(wers) if wers else None, - 'std': np.std(wers) if wers else None, - 'min': np.min(wers) if wers else None, - 'max': np.max(wers) if wers else None, - 'values': wers - }, - 'cer': { - 'mean': np.mean(cers) if cers else None, - 'std': np.std(cers) if cers else None, - 'values': cers - }, - 'latency': { - 'mean': np.mean(latencies) if latencies else None, - 'std': np.std(latencies) if latencies else None, - 'values': latencies - } - } - - logger.info(f"Baseline WER: {results['wer']['mean']:.4f} ({results['wer']['mean']*100:.2f}%)") - logger.info(f"Baseline CER: {results['cer']['mean']:.4f} ({results['cer']['mean']*100:.2f}%)") - logger.info(f"Mean Latency: {results['latency']['mean']:.2f}s") - - return results - - def evaluate_full_system(self, audio_files: List[str], references: List[str]) -> Dict: - """Evaluate full system.""" - logger.info("="*70) - logger.info("EVALUATING FULL SYSTEM") - logger.info("="*70) - - system = UnifiedSTTSystem( - model_name="whisper", - enable_error_detection=True, - enable_llm_correction=True, - enable_adaptive_fine_tuning=False # Disable for faster evaluation - ) - - wers = [] - cers = [] - latencies = [] - errors_detected = [] - corrections_applied = [] - - for i, (audio_file, reference) in enumerate(zip(audio_files, references)): - if not reference.strip(): - continue - - logger.info(f"Processing {i+1}/{len(audio_files)}: {Path(audio_file).name}") - - start_time = time.time() - result = system.transcribe(audio_file, reference_transcript=reference) - latency = time.time() - start_time - - if 'evaluation' in result: - wers.append(result['evaluation']['wer']) - cers.append(result['evaluation']['cer']) - latencies.append(latency) - - # Track error detection and correction - if result.get('error_detection', {}).get('has_errors', False): - errors_detected.append(result['error_detection']['error_count']) - else: - errors_detected.append(0) - - if result.get('corrections', {}).get('applied', False): - corrections_applied.append(result['corrections']['count']) - else: - corrections_applied.append(0) - - results = { - 'num_samples': len(wers), - 'wer': { - 'mean': np.mean(wers) if wers else None, - 'std': np.std(wers) if wers else None, - 'values': wers - }, - 'cer': { - 'mean': np.mean(cers) if cers else None, - 'std': np.std(cers) if cers else None, - 'values': cers - }, - 'latency': { - 'mean': np.mean(latencies) if latencies else None, - 'std': np.std(latencies) if latencies else None, - 'values': latencies - }, - 'errors_detected': { - 'total': sum(errors_detected), - 'mean': np.mean(errors_detected) if errors_detected else None, - 'values': errors_detected - }, - 'corrections_applied': { - 'total': sum(corrections_applied), - 'mean': np.mean(corrections_applied) if corrections_applied else None, - 'values': corrections_applied - } - } - - logger.info(f"Full System WER: {results['wer']['mean']:.4f} ({results['wer']['mean']*100:.2f}%)") - logger.info(f"Full System CER: {results['cer']['mean']:.4f} ({results['cer']['mean']*100:.2f}%)") - logger.info(f"Mean Latency: {results['latency']['mean']:.2f}s") - logger.info(f"Total Errors Detected: {results['errors_detected']['total']}") - logger.info(f"Total Corrections Applied: {results['corrections_applied']['total']}") - - return results - - def run_statistical_analysis(self, baseline_wers: List[float], full_system_wers: List[float]) -> Dict: - """Run statistical analysis comparing baseline vs full system.""" - logger.info("="*70) - logger.info("RUNNING STATISTICAL ANALYSIS") - logger.info("="*70) - - if len(baseline_wers) != len(full_system_wers) or len(baseline_wers) < 2: - logger.warning("Insufficient data for statistical analysis") - return {} - - analyzer = StatisticalAnalyzer() - - # Paired t-test - t_test_result = analyzer.paired_t_test(baseline_wers, full_system_wers) - - # System comparison - comparison = analyzer.compare_systems( - baseline_wers, - full_system_wers, - "Baseline", - "Full System" - ) - - results = { - 'paired_t_test': t_test_result, - 'system_comparison': comparison - } - - logger.info(f"Mean Baseline WER: {t_test_result['mean_baseline']:.4f}") - logger.info(f"Mean Full System WER: {t_test_result['mean_treatment']:.4f}") - logger.info(f"Mean Difference: {t_test_result['mean_difference']:.4f}") - logger.info(f"p-value: {t_test_result['p_value']:.4f}") - logger.info(f"Statistically Significant: {t_test_result['is_significant']}") - logger.info(f"Cohen's d: {t_test_result['cohens_d']:.4f}") - - return results - - def run_ablation_study(self, audio_files: List[str], references: List[str]) -> Dict: - """Run ablation study.""" - logger.info("="*70) - logger.info("RUNNING ABLATION STUDY") - logger.info("="*70) - - try: - study = AblationStudy() - results = study.run_ablation_study( - audio_files=audio_files, - reference_transcripts=references, - model_name="whisper" - ) - - logger.info("Ablation study completed") - if 'summary' in results: - summary = results['summary'] - logger.info(f"Baseline WER: {summary.get('baseline_performance', 'N/A')}") - logger.info(f"Full System WER: {summary.get('full_system_performance', 'N/A')}") - - return results - except Exception as e: - logger.error(f"Error running ablation study: {e}") - import traceback - traceback.print_exc() - return {} - - def generate_report(self) -> str: - """Generate comprehensive evaluation report.""" - report_lines = [] - report_lines.append("="*70) - report_lines.append("COMPREHENSIVE EVALUATION REPORT") - report_lines.append(f"Generated: {datetime.now().isoformat()}") - report_lines.append("="*70) - report_lines.append("") - - # Baseline Results - if 'baseline' in self.results: - baseline = self.results['baseline'] - report_lines.append("BASELINE MODEL RESULTS") - report_lines.append("-"*70) - report_lines.append(f"Number of Samples: {baseline['num_samples']}") - if baseline['wer']['mean'] is not None: - report_lines.append(f"WER: {baseline['wer']['mean']:.4f} ({baseline['wer']['mean']*100:.2f}%)") - report_lines.append(f" Std: {baseline['wer']['std']:.4f}") - report_lines.append(f" Range: [{baseline['wer']['min']:.4f}, {baseline['wer']['max']:.4f}]") - if baseline['cer']['mean'] is not None: - report_lines.append(f"CER: {baseline['cer']['mean']:.4f} ({baseline['cer']['mean']*100:.2f}%)") - if baseline['latency']['mean'] is not None: - report_lines.append(f"Mean Latency: {baseline['latency']['mean']:.2f}s") - report_lines.append("") - - # Full System Results - if 'full_system' in self.results: - full = self.results['full_system'] - report_lines.append("FULL SYSTEM RESULTS") - report_lines.append("-"*70) - report_lines.append(f"Number of Samples: {full['num_samples']}") - if full['wer']['mean'] is not None: - report_lines.append(f"WER: {full['wer']['mean']:.4f} ({full['wer']['mean']*100:.2f}%)") - if full['cer']['mean'] is not None: - report_lines.append(f"CER: {full['cer']['mean']:.4f} ({full['cer']['mean']*100:.2f}%)") - if full['latency']['mean'] is not None: - report_lines.append(f"Mean Latency: {full['latency']['mean']:.2f}s") - if 'errors_detected' in full: - report_lines.append(f"Total Errors Detected: {full['errors_detected']['total']}") - report_lines.append(f"Total Corrections Applied: {full['corrections_applied']['total']}") - report_lines.append("") - - # Statistical Analysis - if 'statistical' in self.results: - stats = self.results['statistical'] - report_lines.append("STATISTICAL ANALYSIS") - report_lines.append("-"*70) - if 'paired_t_test' in stats: - t_test = stats['paired_t_test'] - report_lines.append(f"Mean Difference: {t_test['mean_difference']:.4f}") - report_lines.append(f"p-value: {t_test['p_value']:.4f}") - report_lines.append(f"Statistically Significant: {t_test['is_significant']}") - report_lines.append(f"Cohen's d: {t_test['cohens_d']:.4f}") - report_lines.append(f"95% CI: [{t_test['confidence_interval'][0]:.4f}, {t_test['confidence_interval'][1]:.4f}]") - report_lines.append("") - - # Ablation Study - if 'ablation' in self.results: - ablation = self.results['ablation'] - report_lines.append("ABLATION STUDY RESULTS") - report_lines.append("-"*70) - if 'summary' in ablation: - summary = ablation['summary'] - report_lines.append(f"Baseline WER: {summary.get('baseline_performance', 'N/A')}") - report_lines.append(f"Full System WER: {summary.get('full_system_performance', 'N/A')}") - report_lines.append("") - - # Comparison - if 'baseline' in self.results and 'full_system' in self.results: - baseline_wer = self.results['baseline']['wer']['mean'] - full_wer = self.results['full_system']['wer']['mean'] - if baseline_wer and full_wer: - improvement = ((baseline_wer - full_wer) / baseline_wer) * 100 - report_lines.append("IMPROVEMENT SUMMARY") - report_lines.append("-"*70) - report_lines.append(f"WER Improvement: {improvement:.2f}% relative reduction") - report_lines.append(f" ({baseline_wer*100:.2f}% → {full_wer*100:.2f}%)") - - report_lines.append("") - report_lines.append("="*70) - - return "\n".join(report_lines) - - def run_all_evaluations(self): - """Run all evaluation types.""" - logger.info("Starting comprehensive evaluation...") - - # Find test files - audio_files = self.find_test_audio_files() - if not audio_files: - logger.error("No audio files found!") - return - - logger.info(f"Found {len(audio_files)} audio files") - - # Create references (using baseline as proxy) - references = self.create_reference_transcripts(audio_files) - - # Filter out files without references - valid_pairs = [(a, r) for a, r in zip(audio_files, references) if r.strip()] - audio_files = [a for a, r in valid_pairs] - references = [r for a, r in valid_pairs] - - logger.info(f"Evaluating {len(audio_files)} files with references") - - # Run evaluations - self.results['baseline'] = self.evaluate_baseline(audio_files, references) - self.results['full_system'] = self.evaluate_full_system(audio_files, references) - - # Statistical analysis - if (self.results['baseline']['wer']['values'] and - self.results['full_system']['wer']['values']): - self.results['statistical'] = self.run_statistical_analysis( - self.results['baseline']['wer']['values'], - self.results['full_system']['wer']['values'] - ) - - # Ablation study (may take longer) - logger.info("\nRunning ablation study (this may take a while)...") - self.results['ablation'] = self.run_ablation_study(audio_files, references) - - # Save results - results_file = self.output_dir / f"evaluation_results_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" - with open(results_file, 'w') as f: - json.dump(self.results, f, indent=2, default=str) - logger.info(f"\nResults saved to: {results_file}") - - # Generate report - report = self.generate_report() - report_file = self.output_dir / f"evaluation_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.txt" - with open(report_file, 'w') as f: - f.write(report) - logger.info(f"Report saved to: {report_file}") - - print("\n" + "="*70) - print("EVALUATION COMPLETE") - print("="*70) - print(report) - - return self.results - - -def main(): - """Main function.""" - evaluator = ComprehensiveEvaluator() - results = evaluator.run_all_evaluations() - return results - - -if __name__ == "__main__": - main() - - diff --git a/experiments/run_evaluation.py b/experiments/run_evaluation.py new file mode 100644 index 0000000..2f36a20 --- /dev/null +++ b/experiments/run_evaluation.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 +""" +Unified evaluation: run any dataset on baseline and on improved (fine-tuned) models if they exist. +Replaces run_comprehensive_evaluations, kavya_evaluation_framework, run_benchmark, verify_evaluation_numbers. +""" + +import argparse +import json +import logging +import re +import sys +import time +from pathlib import Path +from typing import List, Dict, Any, Optional, Tuple + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from src.baseline_model import BaselineSTTModel +from src.evaluation.metrics import EvaluationModule +from src.utils.model_versioning import get_all_model_versions, get_current_model_path + +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") +logger = logging.getLogger(__name__) + + +def _get_gold_from_llm(stt_transcript: str, llm_corrector) -> str: + """Get LLM-corrected transcript as gold (same pattern as scripts/finetune_wav2vec2.py).""" + if not stt_transcript: + return "" + if not llm_corrector or not llm_corrector.is_available(): + return stt_transcript + llm_result = llm_corrector.correct_transcript(stt_transcript, errors=[], context={}) + gold = llm_result.get("corrected_transcript", stt_transcript).strip() + gold = re.sub(r'^["\'](.*)["\']$', r"\1", gold.strip()) + return gold.strip() + + +def discover_audio_with_llm_gold(audio_dir: Path, llm_model_name: str = "llama3.2:3b") -> List[Tuple[str, str]]: + """ + Discover WAV/MP3 in audio_dir and use LLM as gold standard. + Requires Ollama running with the given model. Same LLM gold logic as scripts/finetune_wav2vec2.py. + Returns list of (audio_path, gold_reference). + """ + from src.agent.llm_corrector import LlamaLLMCorrector + + if not audio_dir.exists(): + raise FileNotFoundError(f"Audio dir not found: {audio_dir}") + files = sorted( + list(audio_dir.glob("*.wav")) + list(audio_dir.glob("*.mp3")), + key=lambda p: p.name, + ) + paths = [str(f) for f in files] + if not paths: + logger.warning(f"No WAV/MP3 found in {audio_dir}") + return [] + + llm_corrector = LlamaLLMCorrector(model_name=llm_model_name, raise_on_error=False) + if not llm_corrector.is_available(): + logger.error( + "LLM is not available. Start Ollama (e.g. 'ollama serve') and pull the model " + f"(e.g. 'ollama pull {llm_model_name}'). Then re-run with --gold-from-llm." + ) + sys.exit(1) + logger.info(f"Using LLM ({llm_model_name}) as gold standard for {len(paths)} files.") + + baseline = BaselineSTTModel(model_name="whisper") + pairs = [] + for i, audio_path in enumerate(paths): + try: + stt_result = baseline.transcribe(audio_path) + stt_transcript = (stt_result.get("transcript") or "").strip() + gold = _get_gold_from_llm(stt_transcript, llm_corrector) + pairs.append((audio_path, gold)) + if (i + 1) % 10 == 0: + logger.info(f" LLM gold: {i + 1}/{len(paths)} files") + except Exception as e: + logger.warning(f"Failed {audio_path}: {e}") + return pairs + + +def load_eval_set(eval_set_path: Path) -> List[Tuple[str, str]]: + """Load (audio_path, reference) pairs from JSON, JSONL, or CSV.""" + path = Path(eval_set_path) + if not path.exists(): + raise FileNotFoundError(f"Eval set not found: {path}") + pairs = [] + suffix = path.suffix.lower() + if suffix == ".json": + with open(path, "r") as f: + data = json.load(f) + items = data if isinstance(data, list) else data.get("samples", data.get("data", [])) + for item in items: + ap = item.get("audio_path") or item.get("audio") or item.get("path") + ref = item.get("reference") or item.get("text") or item.get("target_text") or "" + if ap and ref is not None: + pairs.append((str(ap).strip(), str(ref).strip())) + elif suffix == ".jsonl": + with open(path, "r") as f: + for line in f: + line = line.strip() + if not line: + continue + item = json.loads(line) + ap = item.get("audio_path") or item.get("audio") or item.get("path") + ref = item.get("reference") or item.get("text") or item.get("target_text") or "" + if ap and ref is not None: + pairs.append((str(ap).strip(), str(ref).strip())) + elif suffix == ".csv": + import csv + with open(path, "r", newline="", encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + ap = row.get("audio_path") or row.get("audio") or row.get("path") + ref = row.get("reference") or row.get("text") or row.get("target_text") or "" + if ap and ref is not None: + pairs.append((str(ap).strip(), str(ref).strip())) + else: + raise ValueError(f"Unsupported format: {suffix}. Use .json, .jsonl, or .csv") + return pairs + + +def discover_audio_and_refs(audio_dir: Path, refs_path: Optional[Path]) -> List[Tuple[str, str]]: + """Discover *.wav and *.mp3 in audio_dir; pair with refs from refs_path if given.""" + if not audio_dir.exists(): + raise FileNotFoundError(f"Audio dir not found: {audio_dir}") + files = sorted( + list(audio_dir.glob("*.wav")) + list(audio_dir.glob("*.mp3")), + key=lambda p: p.name, + ) + paths = [str(f) for f in files] + if not refs_path or not refs_path.exists(): + logger.warning("No --refs file: cannot compute WER. Provide refs (JSON/JSONL/CSV with audio_path + reference) for metrics.") + return [(p, "") for p in paths] + pairs_from_refs = load_eval_set(refs_path) + refs_by_path = {str(Path(p).resolve()): r for p, r in pairs_from_refs} + refs_by_name = {Path(p).name: r for p, r in pairs_from_refs} + out = [] + for p in paths: + r = refs_by_path.get(p) or refs_by_path.get(str(Path(p).resolve())) or refs_by_name.get(Path(p).name, "") + out.append((p, r)) + return out + + +def run_model_on_set( + model: BaselineSTTModel, + pairs: List[Tuple[str, str]], + model_id: str, + skip_empty_ref: bool = True, + include_per_sample: bool = False, +) -> Dict[str, Any]: + """Run model on all (audio_path, reference) pairs; return metrics (and optionally per-sample results).""" + eval_mod = EvaluationModule() + latencies = [] + for audio_path, reference in pairs: + if skip_empty_ref and not reference.strip(): + continue + try: + start = time.time() + out = model.transcribe(audio_path) + latencies.append(time.time() - start) + hyp = (out.get("transcript") or "").strip() + eval_mod.add_prediction(reference, hyp) + except Exception as e: + logger.warning(f"{model_id} failed on {audio_path}: {e}") + metrics = eval_mod.get_metrics() + if not metrics: + out = {"wer": None, "cer": None, "num_samples": 0, "latency_mean_sec": None} + else: + out = { + "wer": metrics["wer"], + "cer": metrics["cer"], + "num_samples": metrics["num_samples"], + "latency_mean_sec": sum(latencies) / len(latencies) if latencies else None, + } + if include_per_sample: + out["results"] = eval_mod.results + return out + + +def main() -> Dict[str, Any]: + parser = argparse.ArgumentParser( + description="Evaluate a dataset on baseline and improved models (if any)." + ) + parser.add_argument( + "--eval-set", + type=Path, + help="Path to evaluation set: JSON/JSONL/CSV with audio_path (or audio) and reference (or text/target_text).", + ) + parser.add_argument( + "--audio-dir", + type=Path, + help="Directory of WAV/MP3 files. Use with --refs to provide references for WER.", + ) + parser.add_argument( + "--refs", + type=Path, + help="Path to refs file (same format as --eval-set). Used only with --audio-dir (ignored if --gold-from-llm).", + ) + parser.add_argument( + "--gold-from-llm", + action="store_true", + help="Use LLM (Ollama) as gold standard. Requires --audio-dir. Checks Ollama is running and gets reference from LLM-corrected baseline transcript.", + ) + parser.add_argument( + "--llm-model", + type=str, + default="llama3.2:3b", + help="Ollama model name for --gold-from-llm (default: llama3.2:3b).", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path("experiments/evaluation_outputs"), + help="Output directory for report and JSON.", + ) + parser.add_argument( + "--baseline-only", + action="store_true", + help="Run only baseline (Whisper), skip improved models.", + ) + parser.add_argument( + "--benchmark", + action="store_true", + help="Include latency/throughput benchmark in output.", + ) + args = parser.parse_args() + + if args.gold_from_llm: + if not args.audio_dir: + logger.error("--gold-from-llm requires --audio-dir.") + sys.exit(1) + pairs = discover_audio_with_llm_gold(args.audio_dir, args.llm_model) + elif args.eval_set: + pairs = load_eval_set(args.eval_set) + elif args.audio_dir: + pairs = discover_audio_and_refs(args.audio_dir, args.refs) + else: + logger.error("Provide --eval-set, or --audio-dir (with --refs or --gold-from-llm).") + sys.exit(1) + + pairs_with_ref = [(p, r) for p, r in pairs if r.strip()] + if not pairs_with_ref: + logger.error("No (audio_path, reference) pairs with non-empty reference. Cannot compute WER.") + sys.exit(1) + + logger.info(f"Loaded {len(pairs_with_ref)} samples with references.") + + output_dir = args.output_dir + output_dir.mkdir(parents=True, exist_ok=True) + + report = { + "num_samples": len(pairs_with_ref), + "baseline": {}, + "improved_models": [], + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"), + "gold_source": "llm" if args.gold_from_llm else ("eval_set" if args.eval_set else "refs"), + } + + baseline = BaselineSTTModel(model_name="whisper") + baseline_info = baseline.get_model_info() + report["baseline"] = { + "model_id": "whisper", + "model_info": baseline_info, + **run_model_on_set(baseline, pairs_with_ref, "baseline"), + } + logger.info(f"Baseline WER: {report['baseline']['wer']:.4f}, CER: {report['baseline']['cer']:.4f}") + + if not args.baseline_only: + versions = get_all_model_versions() + current_path = get_current_model_path() + for v in versions: + model_path = v["path"] + version_num = v["version_num"] + model_id = f"wav2vec2-finetuned-v{version_num}" + try: + model = BaselineSTTModel(model_name=model_id) + metrics = run_model_on_set(model, pairs_with_ref, model_id) + metrics["model_id"] = model_id + metrics["path"] = model_path + metrics["is_current"] = (model_path == current_path) + report["improved_models"].append(metrics) + logger.info(f"{model_id} WER: {metrics['wer']:.4f}, CER: {metrics['cer']:.4f}") + except Exception as e: + logger.warning(f"Could not load or run {model_id}: {e}") + + if args.benchmark: + from src.benchmark import BaselineBenchmark + audio_paths = [p for p, _ in pairs_with_ref[: min(50, len(pairs_with_ref))]] + if audio_paths: + bench = BaselineBenchmark(model_name="whisper") + bench_report = bench.generate_report(audio_paths) + report["benchmark"] = bench_report + bench.save_report(bench_report, str(output_dir / "benchmark_report.json")) + + out_json = output_dir / "evaluation_report.json" + with open(out_json, "w") as f: + json.dump(report, f, indent=2, default=str) + logger.info(f"Report saved to {out_json}") + + txt_lines = [ + "=" * 60, + "EVALUATION REPORT", + f"Samples: {report['num_samples']}", + "", + "Baseline (Whisper)", + f" WER: {report['baseline']['wer']:.4f} CER: {report['baseline']['cer']:.4f}", + "", + ] + for m in report["improved_models"]: + txt_lines.append(f"{m['model_id']} WER: {m['wer']:.4f} CER: {m['cer']:.4f}") + txt_lines.append("=" * 60) + out_txt = output_dir / "evaluation_report.txt" + with open(out_txt, "w") as f: + f.write("\n".join(txt_lines)) + logger.info(f"Summary saved to {out_txt}") + + return report + + +if __name__ == "__main__": + main() diff --git a/experiments/verify_evaluation_numbers.py b/experiments/verify_evaluation_numbers.py deleted file mode 100644 index f43f963..0000000 --- a/experiments/verify_evaluation_numbers.py +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env python3 -""" -Script to verify evaluation numbers in the report by running actual evaluations. -""" - -import sys -from pathlib import Path -import json -import logging -from typing import List, Dict -import numpy as np - -# Add src to path -sys.path.insert(0, str(Path(__file__).parent)) - -from src.baseline_model import BaselineSTTModel -from src.integration import UnifiedSTTSystem, StatisticalAnalyzer, AblationStudy -from src.evaluation.metrics import STTEvaluator - -logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') -logger = logging.getLogger(__name__) - - -def verify_baseline_metrics(): - """Verify baseline model metrics.""" - logger.info("="*70) - logger.info("VERIFYING BASELINE METRICS") - logger.info("="*70) - - # Load existing evaluation results - eval_summary_path = Path("experiments/evaluation_outputs/evaluation_summary.json") - benchmark_path = Path("experiments/evaluation_outputs/benchmark_report.json") - - actual_results = {} - - if eval_summary_path.exists(): - with open(eval_summary_path) as f: - eval_data = json.load(f) - actual_results['baseline_wer'] = eval_data['overall_metrics']['mean_wer'] - actual_results['baseline_cer'] = eval_data['overall_metrics']['mean_cer'] - actual_results['model_params'] = eval_data['model_info']['parameters'] - logger.info(f"✓ Found baseline WER: {actual_results['baseline_wer']:.4f} ({actual_results['baseline_wer']*100:.2f}%)") - logger.info(f"✓ Found baseline CER: {actual_results['baseline_cer']:.4f} ({actual_results['baseline_cer']*100:.2f}%)") - - if benchmark_path.exists(): - with open(benchmark_path) as f: - benchmark_data = json.load(f) - actual_results['mean_latency'] = benchmark_data['latency_benchmark']['mean_latency_seconds'] - actual_results['throughput'] = benchmark_data['throughput_benchmark']['samples_per_second'] - logger.info(f"✓ Found mean latency: {actual_results['mean_latency']:.2f}s") - logger.info(f"✓ Found throughput: {actual_results['throughput']:.2f} samples/s") - - # Report discrepancies - logger.info("\n" + "-"*70) - logger.info("REPORTED vs ACTUAL VALUES:") - logger.info("-"*70) - - discrepancies = [] - - # Baseline WER - reported_wer = 0.10 # Report says 10.0% - if abs(actual_results.get('baseline_wer', 0) - reported_wer) > 0.01: - discrepancies.append(f"Baseline WER: Report says {reported_wer*100:.1f}%, Actual: {actual_results.get('baseline_wer', 'N/A')*100:.1f}%") - else: - logger.info(f"✓ Baseline WER matches: {reported_wer*100:.1f}%") - - # Baseline CER - reported_cer = 0.0227 # Report says 2.27% - if abs(actual_results.get('baseline_cer', 0) - reported_cer) > 0.001: - discrepancies.append(f"Baseline CER: Report says {reported_cer*100:.2f}%, Actual: {actual_results.get('baseline_cer', 'N/A')*100:.2f}%") - else: - logger.info(f"✓ Baseline CER matches: {reported_cer*100:.2f}%") - - # Latency - Report says 0.72s but actual is 5.29s - reported_latency = 0.72 - if abs(actual_results.get('mean_latency', 0) - reported_latency) > 0.1: - discrepancies.append(f"⚠️ LATENCY MISMATCH: Report says {reported_latency:.2f}s, Actual: {actual_results.get('mean_latency', 'N/A'):.2f}s") - logger.warning(f"⚠️ Major discrepancy in latency!") - - # Throughput - reported_throughput = 2.97 - if abs(actual_results.get('throughput', 0) - reported_throughput) > 0.1: - discrepancies.append(f"Throughput: Report says {reported_throughput:.2f} samples/s, Actual: {actual_results.get('throughput', 'N/A'):.2f} samples/s") - else: - logger.info(f"✓ Throughput matches: {reported_throughput:.2f} samples/s") - - if discrepancies: - logger.warning("\n⚠️ DISCREPANCIES FOUND:") - for d in discrepancies: - logger.warning(f" - {d}") - else: - logger.info("\n✓ All baseline metrics match!") - - return actual_results, discrepancies - - -def check_report_numbers(): - """Check numbers mentioned in the report against what we can verify.""" - logger.info("\n" + "="*70) - logger.info("CHECKING REPORT NUMBERS") - logger.info("="*70) - - issues = [] - - # Check baseline numbers - logger.info("\n1. Baseline Performance:") - logger.info(" Report claims: WER 10.0%, CER 2.27%, Latency 0.72s, Throughput 2.97 samples/s") - logger.info(" Note: These appear to be from a small test dataset (2 samples)") - - # Check full system numbers - logger.info("\n2. Full System Performance:") - logger.info(" Report claims: WER 19-22% (improvement from 25-30%)") - logger.info(" ⚠️ WARNING: Baseline WER is actually 10%, not 25-30%") - logger.info(" ⚠️ This suggests report numbers may be from different dataset or theoretical") - issues.append("Baseline WER mismatch: Report says 25-30% but actual baseline is 10%") - - # Check ablation study numbers - logger.info("\n3. Ablation Study Results:") - logger.info(" Report claims various WER values for different configurations") - logger.info(" ⚠️ These numbers cannot be verified without running full ablation study") - logger.info(" ⚠️ Need to run actual ablation study to verify") - - # Check statistical numbers - logger.info("\n4. Statistical Analysis:") - logger.info(" Report claims: p < 0.001, Cohen's d = 0.5-0.7") - logger.info(" ⚠️ These require actual paired comparisons - cannot verify without test data") - - return issues - - -def main(): - """Main verification function.""" - logger.info("EVALUATION NUMBER VERIFICATION") - logger.info("="*70) - logger.info("This script verifies numbers in the report against actual evaluation results.\n") - - # Verify baseline metrics - actual_results, discrepancies = verify_baseline_metrics() - - # Check report numbers - issues = check_report_numbers() - - # Summary - logger.info("\n" + "="*70) - logger.info("SUMMARY") - logger.info("="*70) - - logger.info("\n✓ Verified from actual evaluation files:") - logger.info(f" - Baseline WER: {actual_results.get('baseline_wer', 'N/A')*100:.2f}%") - logger.info(f" - Baseline CER: {actual_results.get('baseline_cer', 'N/A')*100:.2f}%") - logger.info(f" - Mean Latency: {actual_results.get('mean_latency', 'N/A'):.2f}s") - logger.info(f" - Throughput: {actual_results.get('throughput', 'N/A'):.2f} samples/s") - - logger.info("\n⚠️ Numbers in report that need verification:") - logger.info(" - Full system WER (19-22%) - requires full system evaluation") - logger.info(" - Ablation study results - requires running ablation study") - logger.info(" - Statistical p-values and effect sizes - requires paired comparisons") - logger.info(" - Component contributions - requires ablation study") - - logger.info("\n⚠️ Major discrepancies found:") - if discrepancies: - for d in discrepancies: - logger.warning(f" - {d}") - if issues: - for i in issues: - logger.warning(f" - {i}") - - logger.info("\n" + "="*70) - logger.info("RECOMMENDATIONS:") - logger.info("="*70) - logger.info("1. The report contains some theoretical/estimated numbers") - logger.info("2. Baseline metrics (WER 10%, CER 2.27%) are verified from actual evaluations") - logger.info("3. Latency number (0.72s) doesn't match actual (5.29s) - may be from different test") - logger.info("4. Full system and ablation numbers need actual test runs to verify") - logger.info("5. Consider updating report with actual measured values or clearly label as estimates") - - -if __name__ == "__main__": - main() - - diff --git a/requirements.txt b/requirements.txt index 4dc889e..08c87ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ torchaudio>=2.0.0 transformers>=4.35.0 accelerate>=0.24.0 datasets>=2.14.0 -bitsandbytes>=0.43.0 # For quantization (4-bit and 8-bit support) +bitsandbytes>=0.40.0 # For quantization (4-bit/8-bit). 0.43+ is Linux-only; use 0.40–0.42 on macOS. peft>=0.8.0 # For LoRA fine-tuning # Audio processing diff --git a/scripts/setup_gcp_gpu.sh b/scripts/setup_gcp_gpu.sh index d2e3e5c..f741de5 100644 --- a/scripts/setup_gcp_gpu.sh +++ b/scripts/setup_gcp_gpu.sh @@ -91,7 +91,7 @@ echo " 3. Verify GPU access:" echo " python -c 'import torch; print(torch.cuda.is_available())'" echo "" echo " 4. Run your evaluation framework:" -echo " python experiments/kavya_evaluation_framework.py" +echo " python experiments/run_evaluation.py --eval-set path/to/eval_set.json" echo "" echo "💰 Cost Management:" echo " - Stop VM when not in use: gcloud compute instances stop $VM_NAME --zone=$ZONE" diff --git a/src/agent/ollama_llm.py b/src/agent/ollama_llm.py index 900c2d2..f74b1f8 100644 --- a/src/agent/ollama_llm.py +++ b/src/agent/ollama_llm.py @@ -237,12 +237,18 @@ def generate( **kwargs ) - # Validate response type - if not isinstance(response, dict): - raise RuntimeError(f"Unexpected response type: {type(response)}. Expected dict.") + # Handle both dict (older ollama versions) and GenerateResponse object (newer versions) + if isinstance(response, dict): + result = response.get('response', '') + elif hasattr(response, 'response'): + # GenerateResponse object (newer ollama package) + result = response.response + elif hasattr(response, 'text'): + # Alternative attribute name + result = response.text + else: + raise RuntimeError(f"Unexpected response type: {type(response)}. Expected dict or GenerateResponse.") - # Extract and validate result - result = response.get('response', '') if not result: logger.warning("Ollama returned empty response") @@ -279,16 +285,26 @@ def chat( **kwargs ) - # Validate response type - if not isinstance(response, dict): - raise RuntimeError(f"Unexpected response type: {type(response)}. Expected dict.") - - # Extract and validate result - message = response.get('message', {}) - if not isinstance(message, dict): - raise RuntimeError(f"Unexpected message type: {type(message)}. Expected dict.") + # Handle both dict (older ollama versions) and ChatResponse object (newer versions) + if isinstance(response, dict): + message = response.get('message', {}) + if isinstance(message, dict): + result = message.get('content', '') + else: + result = str(message) + elif hasattr(response, 'message'): + # ChatResponse object (newer ollama package) + message = response.message + if hasattr(message, 'content'): + result = message.content + else: + result = str(message) + elif hasattr(response, 'content'): + # Direct content attribute + result = response.content + else: + raise RuntimeError(f"Unexpected response type: {type(response)}. Expected dict or ChatResponse.") - result = message.get('content', '') if not result: logger.warning("Ollama returned empty chat response") diff --git a/src/evaluation/metrics.py b/src/evaluation/metrics.py index 54ba532..1c92880 100644 --- a/src/evaluation/metrics.py +++ b/src/evaluation/metrics.py @@ -1,16 +1,58 @@ """ -Evaluation metrics for STT models: WER and CER. +Unified evaluation module for STT models: WER and CER. +Supports streaming predictions (inference) and batch/offline test sets. """ from jiwer import wer, cer import json +import csv from pathlib import Path -from typing import List, Dict +from typing import List, Dict, Optional, Union, Any import logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) + +def _load_pairs_from_json(path: Path, ref_key: str, hyp_key: str) -> tuple: + """Load (references, hypotheses) from JSON array or {'samples': [...]}.""" + with open(path, "r") as f: + data = json.load(f) + items = data if isinstance(data, list) else data.get("samples", data.get("data", [])) + if not items: + return [], [] + refs = [item.get(ref_key, item.get("reference", "")) for item in items] + hyps = [item.get(hyp_key, item.get("hypothesis", "")) for item in items] + return refs, hyps + + +def _load_pairs_from_jsonl(path: Path, ref_key: str, hyp_key: str) -> tuple: + """Load (references, hypotheses) from JSONL.""" + refs, hyps = [], [] + with open(path, "r") as f: + for line in f: + line = line.strip() + if not line: + continue + item = json.loads(line) + refs.append(item.get(ref_key, item.get("reference", ""))) + hyps.append(item.get(hyp_key, item.get("hypothesis", ""))) + return refs, hyps + + +def _load_pairs_from_csv( + path: Path, ref_key: str, hyp_key: str +) -> tuple: + """Load (references, hypotheses) from CSV. ref_key/hyp_key are column names.""" + refs, hyps = [], [] + with open(path, "r", newline="", encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + refs.append(row.get(ref_key, row.get("reference", ""))) + hyps.append(row.get(hyp_key, row.get("hypothesis", ""))) + return refs, hyps + + class STTEvaluator: """Calculate WER and CER for STT predictions""" @@ -106,3 +148,122 @@ def save_results(self, output_path: str): logger.info(f"Average CER: {summary['average_cer']:.4f}") return summary + + +class EvaluationModule: + """ + Unified evaluation: streaming (inference) and batch/offline test sets. + Use add_prediction() for streaming; evaluate_batch() or evaluate_from_file() for batch. + """ + + def __init__(self): + self._references: List[str] = [] + self._hypotheses: List[str] = [] + self._stt_evaluator = STTEvaluator() + + def add_prediction(self, reference: str, hypothesis: str) -> None: + """ + Add a single reference/hypothesis pair (streaming inference). + Call get_metrics() for current corpus-level WER/CER; per-sample results in .results. + """ + self._references.append(reference) + self._hypotheses.append(hypothesis) + self._stt_evaluator.results.append({ + "reference": reference, + "hypothesis": hypothesis, + "wer": self._stt_evaluator.calculate_wer(reference, hypothesis), + "cer": self._stt_evaluator.calculate_cer(reference, hypothesis), + }) + + def calculate_wer(self, reference: str, hypothesis: str) -> float: + """Single-pair WER (delegates to STTEvaluator).""" + return self._stt_evaluator.calculate_wer(reference, hypothesis) + + def calculate_cer(self, reference: str, hypothesis: str) -> float: + """Single-pair CER (delegates to STTEvaluator).""" + return self._stt_evaluator.calculate_cer(reference, hypothesis) + + def get_metrics(self) -> Dict[str, Any]: + """ + Return current metrics over all pairs added so far (streaming). + Returns WER, CER, num_samples; empty dict if no pairs. + """ + if not self._references or not self._hypotheses: + return {} + n = min(len(self._references), len(self._hypotheses)) + refs = self._references[:n] + hyps = self._hypotheses[:n] + return { + "wer": wer(refs, hyps), + "cer": cer(refs, hyps), + "num_samples": n, + } + + def reset(self) -> None: + """Clear accumulated streaming state.""" + self._references.clear() + self._hypotheses.clear() + self._stt_evaluator.results = [] + + def evaluate_batch( + self, + references: List[str], + hypotheses: List[str], + ) -> Dict[str, Any]: + """ + Evaluate a batch of reference/hypothesis pairs (offline test set). + Returns WER, CER, num_samples, and populates detailed results on internal STTEvaluator. + """ + result = self._stt_evaluator.evaluate_batch(references, hypotheses) + return { + "wer": result["wer"], + "cer": result["cer"], + "num_samples": result["num_samples"], + } + + def evaluate_from_file( + self, + path: Union[str, Path], + reference_key: str = "reference", + hypothesis_key: str = "hypothesis", + ) -> Dict[str, Any]: + """ + Load reference/hypothesis pairs from a batch file and compute metrics. + Supports .json (array or {'samples': [...]}), .jsonl, and .csv. + """ + path = Path(path) + if not path.exists(): + raise FileNotFoundError(f"Evaluation file not found: {path}") + suffix = path.suffix.lower() + if suffix == ".json": + refs, hyps = _load_pairs_from_json(path, reference_key, hypothesis_key) + elif suffix == ".jsonl": + refs, hyps = _load_pairs_from_jsonl(path, reference_key, hypothesis_key) + elif suffix == ".csv": + refs, hyps = _load_pairs_from_csv(path, reference_key, hypothesis_key) + else: + raise ValueError( + f"Unsupported batch file format: {suffix}. Use .json, .jsonl, or .csv" + ) + if len(refs) != len(hyps): + logger.warning( + f"Length mismatch: {len(refs)} references, {len(hyps)} hypotheses; truncating to min" + ) + n = min(len(refs), len(hyps)) + refs, hyps = refs[:n], hyps[:n] + if not refs: + logger.warning("No pairs loaded from %s", path) + return {} + return self.evaluate_batch(refs, hyps) + + @property + def results(self) -> List[Dict]: + """Per-sample results from last evaluate_batch (or from streaming via get_metrics).""" + return self._stt_evaluator.results + + def save_results(self, output_path: str) -> Optional[Dict]: + """Save detailed results (same as STTEvaluator.save_results).""" + if not self._stt_evaluator.results: + logger.warning("No results to save") + return None + return self._stt_evaluator.save_results(output_path) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index bda10b9..495eb95 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -6,7 +6,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).parent.parent)) -from src.evaluation.metrics import STTEvaluator +from src.evaluation.metrics import STTEvaluator, EvaluationModule import pytest @@ -168,6 +168,52 @@ def test_multiple_spaces(): assert wer_score >= 0.0 +class TestEvaluationModule: + """Test unified EvaluationModule (streaming + batch).""" + + def test_streaming_add_and_get_metrics(self): + mod = EvaluationModule() + mod.add_prediction("hello world", "hello world") + mod.add_prediction("goodbye", "good bye") + m = mod.get_metrics() + assert "wer" in m and "cer" in m and m["num_samples"] == 2 + assert len(mod.results) == 2 + + def test_streaming_reset(self): + mod = EvaluationModule() + mod.add_prediction("a", "b") + mod.reset() + assert mod.get_metrics() == {} + assert mod.results == [] + + def test_batch_same_as_stt_evaluator(self): + refs = ["hello world", "test case"] + hyps = ["hello world", "test case"] + mod = EvaluationModule() + out = mod.evaluate_batch(refs, hyps) + assert out["num_samples"] == 2 + assert out["wer"] == 0.0 and out["cer"] == 0.0 + + def test_evaluate_from_file_json(self, tmp_path): + path = tmp_path / "batch.json" + path.write_text('[{"reference": "hi", "hypothesis": "hi"}]') + mod = EvaluationModule() + m = mod.evaluate_from_file(path) + assert m["num_samples"] == 1 and m["wer"] == 0.0 + + def test_evaluate_from_file_jsonl(self, tmp_path): + path = tmp_path / "batch.jsonl" + path.write_text('{"reference": "a", "hypothesis": "a"}\n') + mod = EvaluationModule() + m = mod.evaluate_from_file(path) + assert m["num_samples"] == 1 + + def test_calculate_wer_cer_delegate(self): + mod = EvaluationModule() + assert mod.calculate_wer("hello", "hello") == 0.0 + assert mod.calculate_cer("ab", "ab") == 0.0 + + if __name__ == "__main__": # Run tests pytest.main([__file__, "-v"]) From f9fe2b534ecf9be378b6f4b17cc43c9d26e0c310 Mon Sep 17 00:00:00 2001 From: shivangi221b Date: Tue, 24 Feb 2026 20:41:09 -0500 Subject: [PATCH 2/2] fix minor type error bug --- experiments/run_evaluation.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/experiments/run_evaluation.py b/experiments/run_evaluation.py index 2f36a20..9e1a2c6 100644 --- a/experiments/run_evaluation.py +++ b/experiments/run_evaluation.py @@ -263,7 +263,11 @@ def main() -> Dict[str, Any]: "model_info": baseline_info, **run_model_on_set(baseline, pairs_with_ref, "baseline"), } - logger.info(f"Baseline WER: {report['baseline']['wer']:.4f}, CER: {report['baseline']['cer']:.4f}") + b = report["baseline"] + if b.get("wer") is not None and b.get("cer") is not None: + logger.info(f"Baseline WER: {b['wer']:.4f}, CER: {b['cer']:.4f}") + else: + logger.warning("Baseline produced no valid predictions (WER/CER unavailable).") if not args.baseline_only: versions = get_all_model_versions() @@ -279,7 +283,10 @@ def main() -> Dict[str, Any]: metrics["path"] = model_path metrics["is_current"] = (model_path == current_path) report["improved_models"].append(metrics) - logger.info(f"{model_id} WER: {metrics['wer']:.4f}, CER: {metrics['cer']:.4f}") + if metrics.get("wer") is not None and metrics.get("cer") is not None: + logger.info(f"{model_id} WER: {metrics['wer']:.4f}, CER: {metrics['cer']:.4f}") + else: + logger.warning(f"{model_id}: no valid predictions (WER/CER unavailable).") except Exception as e: logger.warning(f"Could not load or run {model_id}: {e}") @@ -297,17 +304,22 @@ def main() -> Dict[str, Any]: json.dump(report, f, indent=2, default=str) logger.info(f"Report saved to {out_json}") + b = report["baseline"] + wer_str = f"{b['wer']:.4f}" if b.get("wer") is not None else "N/A" + cer_str = f"{b['cer']:.4f}" if b.get("cer") is not None else "N/A" txt_lines = [ "=" * 60, "EVALUATION REPORT", f"Samples: {report['num_samples']}", "", "Baseline (Whisper)", - f" WER: {report['baseline']['wer']:.4f} CER: {report['baseline']['cer']:.4f}", + f" WER: {wer_str} CER: {cer_str}", "", ] for m in report["improved_models"]: - txt_lines.append(f"{m['model_id']} WER: {m['wer']:.4f} CER: {m['cer']:.4f}") + m_wer = f"{m['wer']:.4f}" if m.get("wer") is not None else "N/A" + m_cer = f"{m['cer']:.4f}" if m.get("cer") is not None else "N/A" + txt_lines.append(f"{m['model_id']} WER: {m_wer} CER: {m_cer}") txt_lines.append("=" * 60) out_txt = output_dir / "evaluation_report.txt" with open(out_txt, "w") as f: