diff --git a/QUICK_REFERENCE.md b/QUICK_REFERENCE.md deleted file mode 100644 index 9b1a802..0000000 --- a/QUICK_REFERENCE.md +++ /dev/null @@ -1,311 +0,0 @@ -# Quick Reference Guide - STT System - -Fast lookup for common commands and operations. - -## ๐Ÿš€ Installation (One-Time) - -```bash -# Clone and setup -git clone -cd Adaptive-Self-Learning-Agentic-AI-System -python -m venv venv -source venv/bin/activate # Windows: venv\Scripts\activate -pip install -r requirements.txt -python scripts/verify_setup.py -``` - -## ๐ŸŽฏ Running the System - -### Start API Server - -```bash -# Agent API (recommended - includes error detection) -uvicorn src.agent_api:app --reload --port 8000 - -# Baseline API (simple transcription only) -uvicorn src.inference_api:app --reload --port 8000 -``` - -### Quick Python Script - -```python -from src.baseline_model import BaselineSTTModel -from src.agent import STTAgent -from src.data.integration import IntegratedDataManagementSystem - -# Initialize -model = BaselineSTTModel() -agent = STTAgent(model) -data_system = IntegratedDataManagementSystem() - -# Transcribe -result = agent.transcribe_with_agent("audio.wav", enable_auto_correction=True) -print(result['transcript']) - -# Record failures -if result['error_detection']['has_errors']: - case_id = data_system.record_failed_transcription( - audio_path="audio.wav", - original_transcript=result['transcript'], - error_types=list(result['error_detection']['error_types'].keys()), - error_score=result['error_detection']['error_score'] - ) -``` - -## ๐ŸŒ API Endpoints - -### Baseline Endpoints - -```bash -# Transcribe -curl -X POST "http://localhost:8000/transcribe" -F "file=@audio.wav" - -# Model info -curl "http://localhost:8000/model-info" - -# Health -curl "http://localhost:8000/health" -``` - -### Agent Endpoints - -```bash -# Agent transcribe with auto-correction -curl -X POST "http://localhost:8000/agent/transcribe?auto_correction=true" \ - -F "file=@audio.wav" - -# Submit feedback -curl -X POST "http://localhost:8000/agent/feedback" \ - -H "Content-Type: application/json" \ - -d '{"transcript_id":"123","is_correct":true,"corrected_transcript":"text"}' - -# Get statistics -curl "http://localhost:8000/agent/stats" - -# Get learning data -curl "http://localhost:8000/agent/learning-data" -``` - -## ๐Ÿงช Testing - -```bash -# Test all components -python experiments/test_baseline.py -python experiments/test_agent.py -python experiments/test_data_management.py -python experiments/test_api.py # Requires API running - -# Run evaluation -python experiments/kavya_evaluation_framework.py - -# Run benchmarks -python experiments/run_benchmark.py - -# Generate visualizations -python experiments/visualize_evaluation_results.py -``` - -## ๐Ÿ“Š Data Management - -```python -from src.data.integration import IntegratedDataManagementSystem - -system = IntegratedDataManagementSystem() - -# Record failed case -case_id = system.record_failed_transcription( - audio_path="audio.wav", - original_transcript="text", - error_types=["all_caps"], - error_score=0.75 -) - -# Add correction -system.add_correction(case_id, "Corrected text") - -# Prepare fine-tuning dataset -dataset_info = system.prepare_finetuning_dataset( - min_error_score=0.5, - max_samples=1000, - create_version=True -) - -# Get statistics -stats = system.get_system_statistics() -print(f"Cases: {stats['data_management']['total_failed_cases']}") -print(f"Correction rate: {stats['data_management']['correction_rate']:.1%}") - -# Generate report -report = system.generate_comprehensive_report("report.json") -``` - -## โ˜๏ธ GCP Commands - -```bash -# Setup -gcloud auth login -gcloud config set project stt-agentic-ai-2025 -gcloud services enable compute.googleapis.com storage-api.googleapis.com -gsutil mb gs://stt-project-datasets - -# Create GPU VM -bash scripts/setup_gcp_gpu.sh - -# Deploy to GCP -python scripts/deploy_to_gcp.py - -# Monitor costs -python scripts/monitor_gcp_costs.py - -# VM control -gcloud compute instances stop stt-gpu-vm --zone=us-central1-a -gcloud compute instances start stt-gpu-vm --zone=us-central1-a -gcloud compute instances delete stt-gpu-vm --zone=us-central1-a -``` - -## ๐Ÿ”ง Common Operations - -### Process Single Audio File - -```python -from src.agent import STTAgent -from src.baseline_model import BaselineSTTModel - -model = BaselineSTTModel() -agent = STTAgent(model) -result = agent.transcribe_with_agent("audio.wav") -print(result['transcript']) -``` - -### Process Multiple Files - -```python -from pathlib import Path -from src.agent import STTAgent -from src.baseline_model import BaselineSTTModel - -model = BaselineSTTModel() -agent = STTAgent(model) - -for audio_file in Path("data/raw").glob("*.wav"): - result = agent.transcribe_with_agent(str(audio_file)) - print(f"{audio_file.name}: {result['transcript']}") -``` - -### Check System Status - -```python -from src.data.integration import IntegratedDataManagementSystem - -system = IntegratedDataManagementSystem() -stats = system.get_system_statistics() - -print(f"Total cases: {stats['data_management']['total_failed_cases']}") -print(f"Corrected: {stats['data_management']['corrected_cases']}") -print(f"Correction rate: {stats['data_management']['correction_rate']:.1%}") -``` - -### Export Data for Analysis - -```python -from src.data.integration import IntegratedDataManagementSystem - -system = IntegratedDataManagementSystem() - -# Export to DataFrame -df = system.data_manager.to_dataframe() -print(df.head()) - -# Export learning data -learning_data = system.data_manager.export_learning_data("export.json") -``` - -## ๐Ÿ“ˆ Evaluation & Benchmarking - -```python -from experiments.kavya_evaluation_framework import EvaluationFramework - -framework = EvaluationFramework(model_name="whisper") -results = framework.run_comprehensive_evaluation( - eval_datasets=["data/processed/test_dataset"] -) - -print(f"WER: {results['overall_metrics']['mean_wer']:.4f}") -print(f"CER: {results['overall_metrics']['mean_cer']:.4f}") -``` - -## ๐Ÿ› Troubleshooting - -```bash -# Check installation -python scripts/verify_setup.py - -# Check GPU -python -c "import torch; print(f'CUDA: {torch.cuda.is_available()}')" - -# Clear model cache -rm -rf ~/.cache/huggingface/ - -# Kill process on port -kill -9 $(lsof -ti:8000) - -# Fix permissions -chmod -R 755 data/ - -# Enable debug logging -export PYTHONLOGLEVEL=DEBUG -``` - -## ๐Ÿ“ Important Files - -``` -src/baseline_model.py # Baseline STT model -src/agent_api.py # Agent API server -src/agent/agent.py # Agent orchestrator -src/data/integration.py # Data management system -experiments/test_*.py # Test scripts -experiments/kavya_evaluation_framework.py # Evaluation -requirements.txt # Dependencies -README.md # Full documentation -docs/SETUP_INSTRUCTIONS.md # Detailed setup -``` - -## ๐Ÿ”‘ Key Concepts - -- **Baseline Model**: Whisper-based STT without agent features -- **Agent**: Adds error detection and auto-correction -- **Data Management**: Tracks failures and prepares training data -- **Error Types**: 8+ heuristics (all_caps, missing_punctuation, etc.) -- **Fine-tuning Dataset**: Prepared from corrected failed cases -- **Version Control**: Tracks dataset versions with checksums - -## ๐Ÿ’ก Tips - -- Use agent API for production (includes error detection) -- Record failures automatically for continuous learning -- Prepare fine-tuning dataset when you have 500+ corrected cases -- Enable GCS for backup and collaboration -- Use GPU for 3-7x faster inference -- Monitor agent statistics to track system health -- Generate regular reports for performance tracking - -## ๐Ÿ“ž Help - -```bash -# Verify setup -python scripts/verify_setup.py - -# Run examples -python experiments/example_usage.py - -# Check documentation -ls docs/ -``` - ---- - -For detailed information, see: -- **README.md** - Full project documentation -- **docs/SETUP_INSTRUCTIONS.md** - Detailed setup guide -- **docs/** - Component-specific guides - diff --git a/README.md b/README.md index f96cd70..8e3d6ac 100644 --- a/README.md +++ b/README.md @@ -102,14 +102,17 @@ Adaptive-Self-Learning-Agentic-AI-System/ โ”‚ โ””โ”€โ”€ versions/ # Dataset versions โ”‚ โ”œโ”€โ”€ docs/ # Documentation -โ”‚ โ”œโ”€โ”€ DATA_MANAGEMENT_SYSTEM.md # Data management guide -โ”‚ โ”œโ”€โ”€ QUICK_START_DATA_MANAGEMENT.md # Quick start -โ”‚ โ””โ”€โ”€ GCP_SETUP_GUIDE.md # GCP setup instructions +โ”‚ โ”œโ”€โ”€ SETUP_INSTRUCTIONS.md # Setup guide +โ”‚ โ”œโ”€โ”€ TESTING_GUIDE.md # Testing guide +โ”‚ โ”œโ”€โ”€ DATA_MANAGEMENT_SYSTEM.md # Data management +โ”‚ โ”œโ”€โ”€ FINETUNING.md # Fine-tuning (script + orchestration) +โ”‚ โ”œโ”€โ”€ WANDB.md # Weights & Biases integration +โ”‚ โ”œโ”€โ”€ CONTROL_PANEL.md # Control panel UI +โ”‚ โ”œโ”€โ”€ GCP.md # Google Cloud setup & deployment +โ”‚ โ”œโ”€โ”€ QUICK_REFERENCE.md # Command reference +โ”‚ โ””โ”€โ”€ LLM_INTEGRATION.md # Gemma LLM integration โ”‚ -โ”œโ”€โ”€ requirements.txt # Python dependencies -โ”œโ”€โ”€ README.md # This file -โ”œโ”€โ”€ docs/SETUP_INSTRUCTIONS.md # Detailed setup guide -โ””โ”€โ”€ docs/DATA_MANAGEMENT_SYSTEM.md # Data management guide +โ””โ”€โ”€ requirements.txt # Python dependencies ``` ## ๐Ÿš€ Quick Start @@ -350,7 +353,7 @@ python kavya_evaluation_framework.py Output: - `evaluation_outputs/evaluation_report.json` - Detailed results - `evaluation_outputs/evaluation_summary.json` - Summary metrics -- `evaluation_outputs/EVALUATION_SUMMARY.md` - Human-readable report +- `docs/EVALUATION_SUMMARY.md` - Human-readable report - `evaluation_outputs/visualizations/` - Charts and graphs #### Run Benchmark Tests @@ -755,12 +758,13 @@ python scripts/monitor_gcp_costs.py ## ๐Ÿ“š Documentation -- **[docs/SETUP_INSTRUCTIONS.md](docs/SETUP_INSTRUCTIONS.md)** - Detailed setup guide -- **[docs/DATA_MANAGEMENT_SYSTEM.md](docs/DATA_MANAGEMENT_SYSTEM.md)** - Complete data management guide -- **[docs/QUICK_START_DATA_MANAGEMENT.md](docs/QUICK_START_DATA_MANAGEMENT.md)** - Quick start for data management -- **[docs/DATA_MANAGEMENT_SYSTEM.md](docs/DATA_MANAGEMENT_SYSTEM.md)** - Complete data management API -- **[docs/QUICK_START_DATA_MANAGEMENT.md](docs/QUICK_START_DATA_MANAGEMENT.md)** - Quick start for data management -- **[docs/GCP_SETUP_GUIDE.md](docs/GCP_SETUP_GUIDE.md)** - GCP setup instructions +- **[docs/SETUP_INSTRUCTIONS.md](docs/SETUP_INSTRUCTIONS.md)** - Setup guide +- **[docs/TESTING_GUIDE.md](docs/TESTING_GUIDE.md)** - Testing guide +- **[docs/DATA_MANAGEMENT_SYSTEM.md](docs/DATA_MANAGEMENT_SYSTEM.md)** - Data management +- **[docs/FINETUNING.md](docs/FINETUNING.md)** - Fine-tuning +- **[docs/CONTROL_PANEL.md](docs/CONTROL_PANEL.md)** - Control panel +- **[docs/GCP.md](docs/GCP.md)** - GCP setup & deployment +- **[docs/QUICK_REFERENCE.md](docs/QUICK_REFERENCE.md)** - Command reference ## ๐ŸŽฅ Tutorials diff --git a/data/metadata/adaptive_stt.code-workspace b/data/metadata/adaptive_stt.code-workspace new file mode 100644 index 0000000..0844037 --- /dev/null +++ b/data/metadata/adaptive_stt.code-workspace @@ -0,0 +1,7 @@ +{ + "folders": [ + { + "path": "../.." + } + ] +} \ No newline at end of file diff --git a/docs/CONTROL_PANEL.md b/docs/CONTROL_PANEL.md new file mode 100644 index 0000000..d043fc0 --- /dev/null +++ b/docs/CONTROL_PANEL.md @@ -0,0 +1,78 @@ +# Control Panel User Guide + +Web-based interface for managing the Adaptive Self-Learning Agentic AI System. + +## Quick Start + +```bash +./start_control_panel.sh +# Or: uvicorn src.control_panel_api:app --reload --port 8000 +``` + +Open **http://localhost:8000/app** (API docs: http://localhost:8000/docs) + +## Dashboard Overview + +- **System Health**: Baseline model, agent status, LLM availability +- **Agent Statistics**: Errors detected, corrections made, feedback count +- **Data Statistics**: Failed cases, correction rate, average error score +- **Model Information**: Active STT model, parameters, device + +## Tabs + +### Transcription +- Upload audio (drag-and-drop or click) +- **Baseline (Fast)**: Simple transcription +- **Agent (Recommended)**: Error detection + auto-correction +- Options: Enable auto-correction, record errors for learning + +### Data Management +- Browse failed cases with search/filter +- Add corrections, view case details +- Prepare fine-tuning datasets (min score, max samples, balance types) + +### Fine-Tuning +- Orchestrator status and readiness +- Trigger fine-tuning (force option for testing) +- View job history + +### Models +- Current model info +- Deployed model +- Version history + +### Monitoring +- Performance metrics (inferences, latency, error rates) +- Trends (WER/CER over 7/30/90 days) + +## API Endpoints + +| Category | Endpoints | +|----------|-----------| +| System | `GET /api/health`, `/api/system/stats` | +| Transcription | `POST /api/transcribe/baseline`, `/api/transcribe/agent` | +| Agent | `POST /api/agent/feedback`, `GET /api/agent/stats` | +| Data | `GET /api/data/failed-cases`, `POST /api/data/correction`, `POST /api/data/prepare-dataset` | +| Fine-Tuning | `GET /api/finetuning/status`, `POST /api/finetuning/trigger` | +| Models | `GET /api/models/info`, `/versions`, `/deployed` | +| Monitoring | `GET /api/metadata/performance`, `/trends` | + +## Configuration + +```bash +export USE_GCS=true +export GCS_BUCKET=your-bucket +export GCP_PROJECT=your-project +``` + +Edit `src/control_panel_api.py` for backend config; `frontend/app.js` for `API_BASE_URL`, `PAGE_SIZE`. + +## Troubleshooting + +- **Port in use**: `kill -9 $(lsof -ti:8000)` +- **Frontend not loading**: Verify API at `curl http://localhost:8000/api/health` +- **Fine-tuning unavailable**: Normal if coordinator init fails; check logs + +## Additional Resources + +- [Setup](SETUP_INSTRUCTIONS.md) | [Data Management](DATA_MANAGEMENT_SYSTEM.md) | [Fine-Tuning](FINETUNING.md) diff --git a/docs/CONTROL_PANEL_GUIDE.md b/docs/CONTROL_PANEL_GUIDE.md deleted file mode 100644 index a31f2e8..0000000 --- a/docs/CONTROL_PANEL_GUIDE.md +++ /dev/null @@ -1,451 +0,0 @@ -# Control Panel User Guide - -## ๐ŸŽฏ Overview - -The STT Control Panel is a comprehensive web-based interface for managing and controlling all aspects of the Adaptive Self-Learning Agentic AI System. It provides an intuitive dashboard to: - -- Transcribe audio files with baseline or agent models -- Monitor system health and performance -- Manage failed cases and corrections -- Prepare fine-tuning datasets -- Trigger and monitor fine-tuning jobs -- View model versions and deployments -- Track performance metrics and trends - ---- - -## ๐Ÿš€ Quick Start - -### 1. Start the Control Panel API - -```bash -# Navigate to project directory -cd Adaptive-Self-Learning-Agentic-AI-System - -# Activate virtual environment (if using one) -source venv/bin/activate # On Windows: venv\Scripts\activate - -# Start the API server -uvicorn src.control_panel_api:app --reload --port 8000 -``` - -### 2. Access the Web Interface - -Open your web browser and navigate to: - -``` -http://localhost:8000/app -``` - -Or access the API documentation: - -``` -http://localhost:8000/docs -``` - ---- - -## ๐Ÿ“Š Dashboard Overview - -### System Health Card - -Displays real-time system status: -- **Baseline Model**: Current STT model and device -- **Agent Status**: Agent operational status -- **LLM Available**: Whether Gemma LLM is available for corrections -- **Last Check**: Last health check timestamp - -### Agent Statistics Card - -Shows agent performance metrics: -- **Error Threshold**: Configured error detection threshold -- **Total Errors Detected**: Cumulative errors found -- **Corrections Made**: Number of corrections applied -- **Feedback Count**: User feedback submissions - -### Data Statistics Card - -Displays data management metrics: -- **Total Failed Cases**: Number of transcription failures recorded -- **Corrected Cases**: Cases with corrections added -- **Correction Rate**: Percentage of cases corrected -- **Average Error Score**: Mean error confidence score - -### Model Information Card - -Shows current model details: -- **Model Name**: Active STT model -- **Parameters**: Total model parameters -- **Device**: CPU or CUDA GPU -- **Trainable Params**: Number of trainable parameters - ---- - -## ๐ŸŽค Transcription Tab - -### Upload Audio - -1. **Click** the upload area or drag and drop an audio file -2. **Select** transcription mode: - - **Baseline (Fast)**: Simple transcription without error detection - - **Agent (Recommended)**: With error detection and auto-correction - -### Agent Options - -When using Agent mode, you can: -- **Enable Auto-Correction**: Automatically correct detected errors -- **Record Errors Automatically**: Save failed cases for learning - -### View Results - -After transcription, you'll see: -- **Transcript**: Final transcribed text -- **Error Detection**: Detected errors and confidence scores -- **Corrections Applied**: Original vs. corrected text -- **Performance**: Inference time - ---- - -## ๐Ÿ’พ Data Management Tab - -### Failed Cases - -Browse and manage transcription failures: -- **Search**: Filter cases by text -- **Filter**: Show all, corrected, or uncorrected cases -- **View Details**: Click on a case to see full details -- **Add Corrections**: Submit manual corrections for cases - -### Dataset Preparation - -Prepare fine-tuning datasets: -1. **Minimum Error Score**: Set threshold (0.0 - 1.0) -2. **Max Samples**: Limit dataset size -3. **Balance Error Types**: Ensure diverse error representation -4. **Create Version**: Enable dataset versioning -5. **Click** "Prepare Dataset" to generate - -### Available Datasets - -View all prepared datasets ready for fine-tuning. - ---- - -## ๐Ÿ”ง Fine-Tuning Tab - -### Orchestrator Status - -Monitor fine-tuning system: -- **Status**: System operational status -- **Ready for Fine-tuning**: Whether conditions are met -- **Total Jobs**: Number of fine-tuning jobs run - -### Trigger Fine-Tuning - -Manually start a fine-tuning job: -1. **Check** "Force Trigger" to bypass readiness checks (optional) -2. **Click** "Trigger Fine-Tuning" -3. **Confirm** the action - -### Fine-Tuning Jobs - -View all fine-tuning jobs: -- **Job ID**: Unique identifier -- **Status**: Current job status -- **Created At**: Job creation timestamp -- **Dataset ID**: Associated dataset - ---- - -## ๐ŸงŠ Models Tab - -### Current Model - -View active model information: -- Model name and parameters -- Device (CPU/GPU) -- Trainable parameters - -### Deployed Model - -See currently deployed production model: -- Version ID -- Model name -- Deployment timestamp - -### Model Versions - -Browse all registered model versions: -- Version ID -- Status (deployed, registered, etc.) -- Creation date - ---- - -## ๐Ÿ“ˆ Monitoring Tab - -### Performance Metrics - -Track system performance: -- **Total Inferences**: Total transcriptions processed -- **Average Inference Time**: Mean processing time -- **Error Detection Rate**: Percentage of errors detected -- **Correction Rate**: Percentage of errors corrected - -### Performance Trends - -Visualize metrics over time: -1. **Select Metric**: WER or CER -2. **Select Time Window**: 7, 30, or 90 days -3. View trend data - ---- - -## ๐ŸŽจ Features - -### Real-Time Updates - -The dashboard automatically updates: -- System health every 30 seconds -- Manual refresh buttons on each card -- Instant feedback on actions - -### Responsive Design - -Works on: -- Desktop computers -- Tablets -- Mobile devices (responsive layout) - -### Toast Notifications - -Receive instant feedback: -- โœ… Success messages (green) -- โš ๏ธ Warning messages (yellow) -- โŒ Error messages (red) -- โ„น๏ธ Info messages (blue) - -### Modal Dialogs - -Detailed views for: -- Case details with full transcripts -- Correction submissions -- Extended information - ---- - -## ๐Ÿ”Œ API Integration - -### API Base URL - -By default, the frontend connects to: -``` -http://localhost:8000 -``` - -### Available Endpoints - -#### System -- `GET /` - System overview -- `GET /api/health` - Health check -- `GET /api/system/stats` - System statistics - -#### Transcription -- `POST /api/transcribe/baseline` - Baseline transcription -- `POST /api/transcribe/agent` - Agent transcription - -#### Agent -- `POST /api/agent/feedback` - Submit feedback -- `GET /api/agent/stats` - Agent statistics -- `GET /api/agent/learning-data` - Learning data - -#### Data Management -- `GET /api/data/failed-cases` - List failed cases -- `GET /api/data/case/{case_id}` - Case details -- `POST /api/data/correction` - Add correction -- `GET /api/data/statistics` - Data statistics -- `POST /api/data/prepare-dataset` - Prepare dataset -- `GET /api/data/datasets` - List datasets - -#### Fine-Tuning -- `GET /api/finetuning/status` - Orchestrator status -- `POST /api/finetuning/trigger` - Trigger job -- `GET /api/finetuning/jobs` - List jobs - -#### Models -- `GET /api/models/info` - Model information -- `GET /api/models/versions` - Model versions -- `GET /api/models/deployed` - Deployed model - -#### Monitoring -- `GET /api/metadata/performance` - Performance metrics -- `GET /api/metadata/trends` - Performance trends - ---- - -## ๐Ÿ› ๏ธ Configuration - -### Environment Variables - -Set these before starting: - -```bash -# Optional: Enable GCS integration -export USE_GCS=true -export GCS_BUCKET=your-bucket-name -export GCP_PROJECT=your-project-id - -# Optional: Custom port -export PORT=8000 -``` - -### Backend Configuration - -Edit `src/control_panel_api.py` to customize: -- Base directory for data storage -- GCS integration -- Model selection -- Error thresholds - -### Frontend Configuration - -Edit `frontend/app.js` to modify: -- `API_BASE_URL`: Backend API URL -- `PAGE_SIZE`: Cases per page -- Auto-refresh intervals - ---- - -## ๐Ÿ” Troubleshooting - -### Control Panel Won't Start - -**Problem**: API server fails to start - -**Solution**: -```bash -# Check if port is in use -lsof -ti:8000 - -# Kill existing process -kill -9 $(lsof -ti:8000) - -# Restart API -uvicorn src.control_panel_api:app --reload --port 8000 -``` - -### Frontend Not Loading - -**Problem**: Blank page or 404 error - -**Solution**: -1. Ensure API is running: `curl http://localhost:8000/api/health` -2. Check browser console for errors -3. Verify frontend files exist in `frontend/` directory -4. Clear browser cache and reload - -### System Shows Offline - -**Problem**: Red "System Offline" status - -**Solution**: -1. Check API is running -2. Check network connection -3. Verify no firewall blocking localhost:8000 -4. Check API logs for errors - -### Fine-Tuning Not Available - -**Problem**: "Fine-tuning coordinator not available" message - -**Solution**: -- This is normal if fine-tuning components aren't initialized -- Check API startup logs for initialization errors -- Ensure all dependencies are installed -- Verify data directories exist - ---- - -## ๐Ÿ’ก Tips & Best Practices - -### For Optimal Performance - -1. **Use Agent Mode**: Enable error detection for better quality -2. **Record Errors**: Auto-record helps build training data -3. **Add Corrections**: Manual corrections improve fine-tuning quality -4. **Monitor Regularly**: Check dashboard for system health -5. **Prepare Datasets**: Aim for 500+ corrected cases before fine-tuning - -### For Production Use - -1. **Enable GCS**: Use cloud storage for data persistence -2. **Set Up Monitoring**: Track performance metrics regularly -3. **Regular Backups**: Export and backup data periodically -4. **Version Control**: Always create versions for datasets -5. **Test Before Deploy**: Validate models before deployment - -### For Development - -1. **Use Baseline for Testing**: Faster for quick tests -2. **Force Trigger**: Use force flag for testing fine-tuning -3. **Check Logs**: Monitor console logs for debugging -4. **API Docs**: Use `/docs` endpoint for API reference - ---- - -## ๐Ÿ“š Additional Resources - -- **Main README**: `README.md` - Project overview -- **API Docs**: `http://localhost:8000/docs` - Interactive API documentation -- **Quick Reference**: `QUICK_REFERENCE.md` - Command reference -- **Data Management**: `docs/DATA_MANAGEMENT_SYSTEM.md` - Detailed data guide -- **Fine-Tuning**: `docs/FINETUNING_ORCHESTRATION.md` - Fine-tuning guide - ---- - -## ๐Ÿ› Known Issues - -1. **Trend Visualization**: Currently shows text-based data. Integrate Chart.js for visual charts. -2. **Large Files**: Upload size limited by server configuration -3. **Long Transcriptions**: May timeout on very long audio files -4. **Mobile UX**: Some features optimized for desktop use - ---- - -## ๐Ÿ”„ Future Enhancements - -- [ ] Real-time audio recording -- [ ] Batch transcription upload -- [ ] Advanced visualization with Chart.js -- [ ] User authentication and roles -- [ ] WebSocket for real-time updates -- [ ] Export/import functionality -- [ ] Custom error type configuration -- [ ] A/B testing interface - ---- - -## ๐Ÿ“ž Support - -For issues or questions: -1. Check this guide -2. Review API documentation at `/docs` -3. Check system logs -4. Refer to main project documentation - ---- - -## ๐ŸŽ‰ Quick Tips - -- **Keyboard Shortcuts**: Tab to navigate, Enter to submit -- **Refresh**: Use refresh buttons to update data -- **Search**: Filter cases for quick access -- **Pagination**: Browse large datasets easily -- **Notifications**: Watch for toast messages in top-right - ---- - -**Version**: 1.0.0 -**Last Updated**: December 2024 -**Status**: Production Ready โœ… - diff --git a/docs/CONTROL_PANEL_SUMMARY.md b/docs/CONTROL_PANEL_SUMMARY.md deleted file mode 100644 index f166e00..0000000 --- a/docs/CONTROL_PANEL_SUMMARY.md +++ /dev/null @@ -1,494 +0,0 @@ -# Control Panel Implementation Summary - -## ๐ŸŽฏ Overview - -A comprehensive web-based control panel has been created to manage and control all aspects of the Adaptive Self-Learning Agentic AI System. The control panel provides a unified interface for transcription, data management, fine-tuning orchestration, model management, and performance monitoring. - ---- - -## โœ… What Was Created - -### 1. **Unified Backend API** (`src/control_panel_api.py`) - -A comprehensive FastAPI server that integrates all system components: - -**Key Features:** -- โœ… RESTful API with 30+ endpoints -- โœ… CORS enabled for frontend access -- โœ… Integration with baseline model, agent, data management, and fine-tuning systems -- โœ… Automatic component initialization -- โœ… Comprehensive error handling -- โœ… Interactive API documentation (OpenAPI/Swagger) - -**Endpoint Categories:** -- **System Status**: Health checks, statistics, system overview -- **Transcription**: Baseline and agent-based transcription -- **Agent Management**: Feedback, stats, learning data -- **Data Management**: Failed cases, corrections, dataset preparation -- **Fine-Tuning**: Job orchestration, triggers, status monitoring -- **Model Management**: Version tracking, deployment status -- **Monitoring**: Performance metrics, trends, analytics - -### 2. **Modern Web Frontend** (`frontend/`) - -A responsive, feature-rich web interface: - -**Files Created:** -- `index.html` (416 lines) - Complete HTML structure -- `styles.css` (797 lines) - Professional styling with CSS variables -- `app.js` (800+ lines) - Full application logic and API integration -- `README.md` - Frontend documentation - -**Features:** -- โœ… 6 main tabs: Dashboard, Transcribe, Data, Fine-Tuning, Models, Monitoring -- โœ… Real-time system health monitoring -- โœ… Drag-and-drop audio file upload -- โœ… Interactive data tables with pagination -- โœ… Modal dialogs for detailed views -- โœ… Toast notifications for user feedback -- โœ… Responsive design (desktop, tablet, mobile) -- โœ… Auto-refresh functionality -- โœ… Search and filter capabilities - -### 3. **Startup Script** (`start_control_panel.sh`) - -Automated startup script with conda environment support: - -**Features:** -- โœ… Conda environment activation (stt-genai) -- โœ… Dependency checking -- โœ… Port availability verification -- โœ… Directory structure creation -- โœ… Clear startup messages -- โœ… Error handling - -### 4. **Comprehensive Documentation** - -**Files Created:** -- `docs/CONTROL_PANEL_GUIDE.md` (452 lines) - Complete user guide -- `CONTROL_PANEL_SUMMARY.md` (This file) - Implementation summary -- `frontend/README.md` - Frontend-specific documentation - -**Documentation Includes:** -- Quick start guide -- Feature descriptions -- API reference -- Configuration options -- Troubleshooting guide -- Best practices - ---- - -## ๐ŸŽจ Frontend Features in Detail - -### Dashboard Tab -- **System Health Card**: Real-time status of all components -- **Agent Statistics**: Error detection and correction metrics -- **Data Statistics**: Failed cases and correction rates -- **Model Information**: Current model details and parameters -- **Recent Activity**: Timeline of system events - -### Transcription Tab -- **File Upload**: Drag-and-drop or click to upload -- **Mode Selection**: Choose baseline or agent transcription -- **Agent Options**: Auto-correction and error recording -- **Results Display**: Transcript, errors, corrections, performance -- **Case Recording**: Automatic or manual error case logging - -### Data Management Tab -- **Failed Cases List**: Paginated view of all error cases -- **Search & Filter**: Find cases by text or status -- **Case Details Modal**: Full view with correction submission -- **Dataset Preparation**: Configure and generate fine-tuning datasets -- **Available Datasets**: List of prepared datasets - -### Fine-Tuning Tab -- **Orchestrator Status**: System readiness and configuration -- **Manual Trigger**: Start fine-tuning with force option -- **Jobs History**: Track all fine-tuning jobs -- **Job Details**: Status, timestamps, dataset info - -### Models Tab -- **Current Model**: Active model information -- **Deployed Model**: Production model details -- **Version History**: All registered model versions -- **Deployment Status**: Track model deployments - -### Monitoring Tab -- **Performance Metrics**: Inference times, error rates -- **Trend Analysis**: WER/CER over time -- **System Analytics**: Comprehensive statistics -- **Time Windows**: 7, 30, or 90-day views - ---- - -## ๐Ÿ”Œ API Endpoints - -### System Endpoints -``` -GET / # System overview -GET /api/health # Health check -GET /api/system/stats # System statistics -``` - -### Transcription Endpoints -``` -POST /api/transcribe/baseline # Baseline transcription -POST /api/transcribe/agent # Agent transcription with error detection -``` - -### Agent Endpoints -``` -POST /api/agent/feedback # Submit user feedback -GET /api/agent/stats # Agent statistics -GET /api/agent/learning-data # Get learning data -``` - -### Data Management Endpoints -``` -GET /api/data/failed-cases # List failed cases (paginated) -GET /api/data/case/{case_id} # Get case details -POST /api/data/correction # Add correction to case -GET /api/data/statistics # Data statistics -POST /api/data/prepare-dataset # Prepare fine-tuning dataset -GET /api/data/datasets # List available datasets -GET /api/data/report # Generate comprehensive report -``` - -### Fine-Tuning Endpoints -``` -GET /api/finetuning/status # Orchestrator status -POST /api/finetuning/trigger # Trigger fine-tuning job -GET /api/finetuning/jobs # List all jobs -GET /api/finetuning/job/{id} # Get job details -``` - -### Model Management Endpoints -``` -GET /api/models/info # Current model information -GET /api/models/versions # List model versions -GET /api/models/deployed # Get deployed model -``` - -### Monitoring Endpoints -``` -GET /api/metadata/performance # Performance metrics -GET /api/metadata/trends # Performance trends -``` - ---- - -## ๐Ÿš€ Quick Start - -### 1. Using the Startup Script (Recommended) - -```bash -# Make script executable (first time only) -chmod +x start_control_panel.sh - -# Run the script -./start_control_panel.sh -``` - -The script will: -- Activate the conda environment (stt-genai) -- Check dependencies -- Verify port availability -- Create necessary directories -- Start the API server - -### 2. Manual Start - -```bash -# Activate conda environment -conda activate stt-genai - -# Start the API server -uvicorn src.control_panel_api:app --reload --port 8000 -``` - -### 3. Access the Control Panel - -Open your browser and navigate to: -- **Control Panel**: http://localhost:8000/app -- **API Docs**: http://localhost:8000/docs -- **Health Check**: http://localhost:8000/api/health - ---- - -## ๐ŸŽฏ Key Capabilities - -### For Users -- โœ… Intuitive web interface for all operations -- โœ… No command-line knowledge required -- โœ… Real-time feedback and notifications -- โœ… Visual representation of data -- โœ… Easy audio file processing - -### For Developers -- โœ… RESTful API for programmatic access -- โœ… Interactive API documentation -- โœ… Modular and extensible design -- โœ… Clear separation of concerns -- โœ… Easy to customize and extend - -### For Operations -- โœ… Centralized monitoring dashboard -- โœ… System health at a glance -- โœ… Performance metrics tracking -- โœ… Automated workflows -- โœ… Error case management - ---- - -## ๐Ÿ“Š Architecture - -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Web Browser โ”‚ -โ”‚ (Control Panel Interface) โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ HTTP/REST API - โ–ผ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Control Panel API Server โ”‚ -โ”‚ (src/control_panel_api.py) โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ - โ–ผ โ–ผ โ–ผ โ–ผ โ–ผ - โ”Œโ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ” - โ”‚Baseโ”‚ โ”‚Agntโ”‚ โ”‚Dataโ”‚ โ”‚Fineโ”‚ โ”‚Metaโ”‚ - โ”‚lineโ”‚ โ”‚ โ”‚ โ”‚Mgmtโ”‚ โ”‚Tuneโ”‚ โ”‚dataโ”‚ - โ”‚Modelโ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ - โ””โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”˜ -``` - ---- - -## ๐ŸŽจ Design Highlights - -### Visual Design -- **Modern UI**: Clean, professional interface -- **Color Scheme**: Primary blue (#4f46e5) with semantic colors -- **Typography**: System fonts for optimal readability -- **Icons**: Font Awesome for consistent iconography -- **Responsive**: Works on all screen sizes - -### User Experience -- **Intuitive Navigation**: Tab-based interface -- **Immediate Feedback**: Toast notifications -- **Loading States**: Clear indication of processing -- **Error Handling**: Graceful degradation -- **Auto-Refresh**: Keep data current - -### Technical Design -- **No Build Step**: Pure HTML/CSS/JS -- **Fast Loading**: Minimal dependencies -- **API-First**: Backend decoupled from frontend -- **Extensible**: Easy to add new features -- **Documented**: Comprehensive inline comments - ---- - -## ๐Ÿ”ง Configuration - -### Backend Configuration - -Edit `src/control_panel_api.py`: -```python -# Enable/disable GCS integration -data_system = IntegratedDataManagementSystem( - base_dir="data/production", - use_gcs=False # Set to True for cloud storage -) - -# Configure coordinator -coordinator = FinetuningCoordinator( - data_manager=data_system.data_manager, - use_gcs=False -) -``` - -### Frontend Configuration - -Edit `frontend/app.js`: -```javascript -// Change API URL if needed -const API_BASE_URL = window.location.origin; - -// Adjust pagination -const PAGE_SIZE = 20; - -// Modify auto-refresh interval (in milliseconds) -setInterval(checkSystemHealth, 30000); // 30 seconds -``` - ---- - -## ๐Ÿ“ˆ Performance - -### Backend -- **Startup Time**: ~2-5 seconds (depending on model loading) -- **Response Time**: <100ms for most endpoints -- **Concurrent Requests**: Handled by uvicorn/FastAPI -- **Memory Usage**: ~1-2GB (with models loaded) - -### Frontend -- **Load Time**: <1 second -- **API Calls**: Optimized with caching -- **Responsiveness**: 60fps animations -- **Bundle Size**: ~100KB total (HTML+CSS+JS) - ---- - -## ๐Ÿ› ๏ธ Troubleshooting - -### Common Issues - -**1. Port 8000 already in use** -```bash -# Kill existing process -kill -9 $(lsof -ti:8000) -``` - -**2. Conda environment not found** -```bash -# Create environment -conda create -n stt-genai python=3.8 -conda activate stt-genai -pip install -r requirements.txt -``` - -**3. Frontend not loading** -- Clear browser cache -- Check browser console for errors -- Verify API is running: `curl http://localhost:8000/api/health` - -**4. Fine-tuning not available** -- This is normal if coordinator initialization fails -- Check API logs for errors -- Verify all dependencies installed - ---- - -## ๐Ÿ”ฎ Future Enhancements - -### Planned Features -- [ ] **Chart.js Integration**: Visual trend charts -- [ ] **WebSocket Support**: Real-time updates without refresh -- [ ] **User Authentication**: Login system and user roles -- [ ] **Batch Upload**: Process multiple audio files -- [ ] **Export Functionality**: Download data as CSV/JSON -- [ ] **Dark Mode**: Theme switcher -- [ ] **Custom Dashboards**: Configurable widgets -- [ ] **Advanced Analytics**: ML-powered insights - -### Technical Improvements -- [ ] **Unit Tests**: Frontend and backend testing -- [ ] **CI/CD Pipeline**: Automated deployment -- [ ] **Docker Support**: Containerized deployment -- [ ] **Load Balancing**: Multiple API instances -- [ ] **Caching Layer**: Redis integration -- [ ] **Rate Limiting**: API throttling - ---- - -## ๐Ÿ“š Documentation Files - -1. **docs/CONTROL_PANEL_GUIDE.md** (452 lines) - - Comprehensive user guide - - Feature descriptions - - API reference - - Troubleshooting - -2. **CONTROL_PANEL_SUMMARY.md** (This file) - - Implementation overview - - Architecture details - - Quick reference - -3. **frontend/README.md** - - Frontend-specific documentation - - Technical details - - Development guide - -4. **API Docs** (http://localhost:8000/docs) - - Interactive Swagger UI - - Try endpoints directly - - Request/response schemas - ---- - -## ๐ŸŽ‰ Success Metrics - -### Implementation Completeness -- โœ… **Backend API**: 100% (30+ endpoints) -- โœ… **Frontend UI**: 100% (6 tabs, all features) -- โœ… **Documentation**: 100% (3 comprehensive guides) -- โœ… **Startup Script**: 100% (conda integration) -- โœ… **Testing**: Ready for manual testing - -### Code Statistics -- **Backend**: ~570 lines (control_panel_api.py) -- **Frontend HTML**: ~416 lines -- **Frontend CSS**: ~797 lines -- **Frontend JS**: ~800+ lines -- **Documentation**: ~1000+ lines -- **Total**: ~3500+ lines of production code - -### Features Delivered -- โœ… System monitoring dashboard -- โœ… Audio transcription interface -- โœ… Data management system -- โœ… Fine-tuning orchestration -- โœ… Model management -- โœ… Performance monitoring -- โœ… Real-time updates -- โœ… Responsive design - ---- - -## ๐Ÿš€ Getting Started Checklist - -- [ ] Activate conda environment: `conda activate stt-genai` -- [ ] Install dependencies: `pip install -r requirements.txt` -- [ ] Run startup script: `./start_control_panel.sh` -- [ ] Open browser: http://localhost:8000/app -- [ ] Check system health on dashboard -- [ ] Try transcribing a test audio file -- [ ] Explore other tabs and features -- [ ] Read docs/CONTROL_PANEL_GUIDE.md for detailed usage - ---- - -## ๐Ÿ“ž Support - -For help with the control panel: -1. **User Guide**: docs/CONTROL_PANEL_GUIDE.md -2. **API Docs**: http://localhost:8000/docs -3. **Frontend Docs**: frontend/README.md -4. **Main Project**: README.md - ---- - -## โœจ Summary - -The Control Panel provides a **production-ready, comprehensive web interface** for managing the entire Adaptive Self-Learning Agentic AI System. With **30+ API endpoints**, a **modern responsive UI**, and **extensive documentation**, it enables users to: - -- ๐ŸŽค Transcribe audio with or without agent features -- ๐Ÿ’พ Manage failed cases and corrections -- ๐Ÿ“Š Monitor system health and performance -- ๐Ÿ”ง Orchestrate fine-tuning workflows -- ๐ŸงŠ Track model versions and deployments -- ๐Ÿ“ˆ Analyze performance trends - -**Status**: โœ… Production Ready -**Total Implementation**: ~3500+ lines of code -**Documentation**: Complete -**Testing**: Ready for use - ---- - -**Last Updated**: December 2024 -**Version**: 1.0.0 -**Implemented By**: AI Assistant - diff --git a/docs/DATA_MANAGEMENT_SYSTEM.md b/docs/DATA_MANAGEMENT_SYSTEM.md index f8b4eda..e0481ce 100644 --- a/docs/DATA_MANAGEMENT_SYSTEM.md +++ b/docs/DATA_MANAGEMENT_SYSTEM.md @@ -1,5 +1,29 @@ # Data Management System Documentation +## Quick Start (5 Minutes) + +```bash +pip install -r requirements.txt +python experiments/test_data_management.py +``` + +```python +from src.data.integration import IntegratedDataManagementSystem + +system = IntegratedDataManagementSystem(base_dir="data/quickstart", use_gcs=False) +case_id = system.record_failed_transcription( + audio_path="audio/sample.wav", + original_transcript="THIS IS ALL CAPS", + corrected_transcript="This is proper text", + error_types=["all_caps"], + error_score=0.8, + inference_time=0.5 +) +stats = system.get_system_statistics() +``` + +--- + ## Overview The Data Management System is a comprehensive solution for managing failed transcription cases, tracking performance metrics, preparing fine-tuning datasets, and maintaining data quality with version control. It is designed to integrate seamlessly with Google Cloud Storage for scalable, production-ready deployments. diff --git a/experiments/evaluation_outputs/EVALUATION_SUMMARY.md b/docs/EVALUATION_SUMMARY.md similarity index 98% rename from experiments/evaluation_outputs/EVALUATION_SUMMARY.md rename to docs/EVALUATION_SUMMARY.md index a298a36..e2b7dbd 100644 --- a/experiments/evaluation_outputs/EVALUATION_SUMMARY.md +++ b/docs/EVALUATION_SUMMARY.md @@ -105,7 +105,7 @@ experiments/evaluation_outputs/ โ”‚ โ”œโ”€โ”€ wer_cer_comparison.png # WER/CER comparison chart โ”‚ โ”œโ”€โ”€ error_distribution.png # Error distribution histogram โ”‚ โ””โ”€โ”€ evaluation_dashboard.png # Comprehensive dashboard -โ””โ”€โ”€ EVALUATION_SUMMARY.md # This summary document +โ””โ”€โ”€ (see docs/EVALUATION_SUMMARY.md for this summary) ``` ## ๐Ÿš€ Next Steps diff --git a/docs/FINETUNING.md b/docs/FINETUNING.md new file mode 100644 index 0000000..8c04d4f --- /dev/null +++ b/docs/FINETUNING.md @@ -0,0 +1,184 @@ +# Fine-Tuning Guide + +Complete guide covering Wav2Vec2 script fine-tuning and the orchestration system for automated model improvement. + +--- + +## Quick Start + +```bash +# Run comprehensive demo +python experiments/demo_finetuning_orchestration.py +``` + +```python +from src.data.data_manager import DataManager +from src.data.finetuning_coordinator import FinetuningCoordinator + +data_manager = DataManager(use_gcs=False) +coordinator = FinetuningCoordinator(data_manager=data_manager, use_gcs=False) +coordinator.print_status() + +trigger_result = coordinator.orchestrator.check_trigger_conditions() +if trigger_result['should_trigger']: + workflow = coordinator.run_complete_workflow(force_trigger=True, auto_deploy=False) +``` + +--- + +## Part 1: Wav2Vec2 Script Fine-Tuning + +Fine-tune the Wav2Vec2 STT model using LLM-generated gold standard transcripts. + +### Overview + +1. **Evaluation Phase**: Process 200 audio files (100 clean, 100 noisy), get STT transcripts, use LLM to generate gold standard, calculate baseline WER/CER +2. **Fine-Tuning Phase**: Fine-tune only on samples where STT made errors +3. **Re-evaluation Phase**: Evaluate fine-tuned model and show improvements + +### Prerequisites + +- Python 3.8+ +- Audio files (200 total: 100 clean, 100 noisy) +- LLM (Mistral) connection working + +### Run Fine-Tuning + +```bash +# Test LLM connection first +python scripts/test_llm_connection.py + +# Basic usage (uses LoRA by default) +python scripts/finetune_wav2vec2.py --audio_dir data/finetuning_audio + +# Advanced options +python scripts/finetune_wav2vec2.py \ + --audio_dir data/finetuning_audio \ + --output_dir models/finetuned_wav2vec2 \ + --num_epochs 5 \ + --batch_size 8 \ + --learning_rate 3e-5 \ + --lora_rank 8 \ + --lora_alpha 16 + +# Full fine-tuning (no LoRA) +python scripts/finetune_wav2vec2.py --audio_dir data/finetuning_audio --no_lora +``` + +### LoRA vs Full Fine-Tuning + +**LoRA (default)**: 3-5x faster, 3-5x less GPU memory, comparable accuracy (within 0.3-0.5% of full). Use for limited resources or fast iteration. + +**Full Fine-Tuning**: Maximum accuracy potential. Use when maximum accuracy is critical and you have abundant compute. + +### Audio File Structure + +``` +data/finetuning_audio/ +โ”œโ”€โ”€ clean/ +โ”‚ โ””โ”€โ”€ audio_001.wav ... (100 files) +โ””โ”€โ”€ noisy/ + โ””โ”€โ”€ audio_101.wav ... (100 files) +``` + +--- + +## Part 2: Orchestration System + +Automated pipeline for model fine-tuning, validation, deployment, and monitoring. + +### System Architecture + +``` +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ Fine-Tuning Coordinator โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” + โ–ผ โ–ผ โ–ผ โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ Fine-Tuning โ”‚ โ”‚ Model โ”‚ โ”‚ Model โ”‚ โ”‚ Regression โ”‚ +โ”‚ Orchestrator โ”‚ โ”‚ Validator โ”‚ โ”‚ Deployer โ”‚ โ”‚ Tester โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +### Components + +1. **FinetuningOrchestrator** (`src/data/finetuning_orchestrator.py`) - Monitors errors, triggers fine-tuning +2. **ModelValidator** (`src/data/model_validator.py`) - Validates against baseline, statistical significance +3. **ModelDeployer** (`src/data/model_deployer.py`) - Version management, deployment, rollback +4. **RegressionTester** (`src/data/regression_tester.py`) - Prevents degradation +5. **FinetuningCoordinator** (`src/data/finetuning_coordinator.py`) - Central orchestration + +### Configuration + +```python +from src.data.finetuning_orchestrator import FinetuningConfig + +# Development +config = FinetuningConfig( + min_error_cases=10, + auto_approve_finetuning=True +) + +# Production +config = FinetuningConfig( + min_error_cases=100, + min_corrected_cases=50, + trigger_on_error_rate=True, + error_rate_threshold=0.15, + auto_approve_finetuning=False +) +``` + +### Complete Workflow + +```python +from src.data.finetuning_coordinator import FinetuningCoordinator +from src.data.data_manager import DataManager + +data_manager = DataManager(use_gcs=True) +coordinator = FinetuningCoordinator( + data_manager=data_manager, + use_gcs=True, + project_id="your-project" +) + +# Set callbacks +coordinator.set_training_callback(custom_training_function) +coordinator.set_baseline_transcribe_function(baseline_transcribe) + +# Run workflow +workflow = coordinator.run_complete_workflow(force_trigger=True, auto_deploy=False) +``` + +### GCP Deployment + +```bash +python scripts/deploy_finetuning_to_gcp.py \ + --create-vm --prepare-dataset --run-training \ + --dataset-id your_dataset_id +``` + +### Troubleshooting + +- **Trigger not working**: `orchestrator.check_trigger_conditions()` โ†’ check `reasons` and `metrics` +- **Validation failing**: Review `result.failure_reason` and validation config +- **Deployment issues**: `deployer.print_status()`, `deployer.rollback()` if needed + +--- + +## File Structure + +``` +src/data/ +โ”œโ”€โ”€ finetuning_orchestrator.py +โ”œโ”€โ”€ model_validator.py +โ”œโ”€โ”€ model_deployer.py +โ”œโ”€โ”€ regression_tester.py +โ”œโ”€โ”€ finetuning_coordinator.py +โ””โ”€โ”€ finetuning_pipeline.py + +scripts/ +โ”œโ”€โ”€ finetune_wav2vec2.py +โ””โ”€โ”€ deploy_finetuning_to_gcp.py +``` diff --git a/docs/FINETUNING_GUIDE.md b/docs/FINETUNING_GUIDE.md deleted file mode 100644 index 8d1da0f..0000000 --- a/docs/FINETUNING_GUIDE.md +++ /dev/null @@ -1,297 +0,0 @@ -# Wav2Vec2 Fine-tuning Guide - -This guide explains how to fine-tune the Wav2Vec2 STT model using LLM-generated gold standard transcripts. - -## Overview - -The fine-tuning process: -1. **Evaluation Phase**: Processes 200 audio files (100 clean, 100 noisy), gets STT transcripts, uses LLM to generate gold standard transcripts, and calculates baseline WER/CER -2. **Fine-tuning Phase**: Fine-tunes the model only on samples where STT made errors -3. **Re-evaluation Phase**: Evaluates the fine-tuned model and shows improvements - -## Prerequisites - -- Python 3.8+ -- Audio files (200 total: 100 clean, 100 noisy) -- LLM (Mistral) connection working - -## Setup - -1. **Install dependencies** (if not already installed): -```bash -pip install torch transformers librosa jiwer datasets peft bitsandbytes -``` - -Optional (for faster LLM inference): -```bash -pip install flash-attn # Requires CUDA and proper compilation -``` - -2. **Organize your audio files**: -``` -data/finetuning_audio/ -โ”œโ”€โ”€ clean/ -โ”‚ โ”œโ”€โ”€ audio_001.wav -โ”‚ โ”œโ”€โ”€ audio_002.wav -โ”‚ โ””โ”€โ”€ ... (100 files) -โ””โ”€โ”€ noisy/ - โ”œโ”€โ”€ audio_101.wav - โ”œโ”€โ”€ audio_102.wav - โ””โ”€โ”€ ... (100 files) -``` - -Alternatively, if you put all files in one directory, the script will automatically split them in half. - -## Test LLM Connection - -Before fine-tuning, test that the LLM is working: - -```bash -python scripts/test_llm_connection.py -``` - -Expected output: -``` -============================================================ -LLM Connection Test -============================================================ - -1. Initializing Mistral LLM... - Loading LLM: mistralai/Mistral-7B-Instruct-v0.3 on cuda (fast_mode=True) - Using 4-bit quantization for fast inference - Warming up model... - Model warm-up complete - โœ“ LLM corrector initialized - -2. Checking LLM availability... - โœ“ LLM is available and loaded - -3. Testing transcript correction... - Input: HIS LATRPAR AS USUALLY FORE - Output: [LLM corrected output] - โœ“ LLM successfully corrected the transcript - -4. Testing transcript improvement... - ... -``` - -### LLM Optimization Features - -The LLM corrector now includes several optimizations for faster inference: - -1. **4-bit Quantization** (when CUDA available): - - Reduces memory usage by ~75% - - Significantly speeds up inference - - Minimal accuracy loss - -2. **Fast Mode** (enabled by default): - - Reduced max tokens (128 vs 512) - - Greedy decoding (faster, deterministic) - - KV cache optimization - - Model warm-up on initialization - -3. **Flash Attention 2** (optional): - - Automatically used if installed - - Faster attention computation - - Requires CUDA and proper compilation - -These optimizations target **<1 second per transcript** inference time while maintaining quality. - -## Run Fine-tuning - -### Basic Usage - -```bash -python scripts/finetune_wav2vec2.py --audio_dir data/finetuning_audio -``` - -By default, the script uses **LoRA** (Low-Rank Adaptation) for efficient fine-tuning, which is 3-5x faster and uses 3-5x less memory than full fine-tuning while maintaining comparable accuracy (within 0.3-0.5%). - -### Advanced Options - -```bash -python scripts/finetune_wav2vec2.py \ - --audio_dir data/finetuning_audio \ - --output_dir models/finetuned_wav2vec2 \ - --num_epochs 5 \ - --batch_size 8 \ - --learning_rate 3e-5 \ - --lora_rank 8 \ - --lora_alpha 16 -``` - -### Arguments - -- `--audio_dir`: Directory containing audio files (required) - - Should have `clean/` and `noisy/` subdirectories, OR - - All files in root directory (will be split in half) -- `--output_dir`: Output directory for fine-tuned model (default: `models/finetuned_wav2vec2`) -- `--num_epochs`: Number of training epochs (default: 3) -- `--batch_size`: Training batch size (default: 4) -- `--learning_rate`: Learning rate (default: 3e-5) -- `--use_lora`: Enable LoRA fine-tuning (default: True) -- `--no_lora`: Disable LoRA and use full fine-tuning -- `--lora_rank`: LoRA rank - controls number of trainable parameters (default: 8) - - Higher rank = more parameters, potentially better accuracy, but slower - - Recommended range: 4-16 -- `--lora_alpha`: LoRA alpha scaling factor (default: 16) - - Typically set to 2ร— rank for good performance - -## Output - -The script will: - -1. **Display baseline metrics**: - ``` - Baseline Metrics: - WER: 0.3620 (36.20%) - CER: 0.1300 (13.00%) - Error Samples: 150/200 - Error Rate: 0.7500 (75.00%) - ``` - -2. **Estimate training time**: - ``` - Estimated training time: ~X.X minutes - ``` - -3. **Run fine-tuning** and show progress - -4. **Display fine-tuned metrics**: - ``` - Fine-tuned Metrics: - WER: 0.3200 (32.00%) - CER: 0.1100 (11.00%) - Error Samples: 140/200 - ``` - -5. **Show summary with improvements**: - ``` - SUMMARY - ============================================================ - - Baseline WER: 0.3620 (36.20%) - Fine-tuned WER: 0.3200 (32.00%) - WER Improvement: 0.0420 (4.20 percentage points) - - Baseline CER: 0.1300 (13.00%) - Fine-tuned CER: 0.1100 (11.00%) - CER Improvement: 0.0200 (2.00 percentage points) - ``` - -6. **Save results** to `{output_dir}/evaluation_results.json` - -## LoRA vs Full Fine-Tuning - -### LoRA (Low-Rank Adaptation) - Default - -**Benefits:** -- **3-5x faster** training time -- **3-5x less GPU memory** usage -- Only ~0.8% of parameters are trainable -- Comparable accuracy (typically within 0.3-0.5% of full fine-tuning) -- Smaller saved models (only adapters, not full model) - -**When to use:** -- Limited computational resources -- Fast iteration and experimentation -- When slight accuracy trade-off is acceptable - -**Model saving:** -- LoRA adapters are saved to `{output_dir}/lora_adapters/` -- To use: Load base model + adapters, or merge adapters for standalone use - -### Full Fine-Tuning - -**Benefits:** -- Maximum accuracy potential -- All model parameters updated -- Better for complex domain-specific tasks - -**When to use:** -- When maximum accuracy is critical -- When you have abundant computational resources -- For complex tasks requiring comprehensive model updates - -**To use full fine-tuning:** -```bash -python scripts/finetune_wav2vec2.py --audio_dir data/finetuning_audio --no_lora -``` - -## Training Time Estimation - -The script estimates training time based on: -- Number of error samples -- Number of epochs -- LoRA vs Full fine-tuning - -**LoRA**: ~7.5 seconds per sample per epoch (3-5x faster) -**Full Fine-tuning**: ~30 seconds per sample per epoch - -**Examples**: -- **LoRA**: 150 error samples ร— 3 epochs ร— 7.5 seconds = ~56 minutes -- **Full**: 150 error samples ร— 3 epochs ร— 30 seconds = ~3.75 hours - -**Actual time** may vary based on: -- Hardware (CPU vs GPU) -- Audio file lengths -- Batch size -- LoRA rank (higher rank = slightly slower) - -## Using the Fine-tuned Model - -After fine-tuning, the model will be saved to the output directory. To use it in the system: - -1. Update `src/baseline_model.py` to load from the fine-tuned path for "wav2vec2-finetuned" -2. Or load directly: -```python -from src.baseline_model import BaselineSTTModel - -model = BaselineSTTModel(model_name="path/to/finetuned/model") -result = model.transcribe("audio_file.wav") -``` - -## Troubleshooting - -### LLM Not Available -If you see warnings about LLM not being available: -- Run `python scripts/test_llm_connection.py` to diagnose -- Check that Mistral model can be loaded -- The script will continue using STT transcripts as gold standard (not ideal) - -### Out of Memory -- Reduce `--batch_size` (try 2 or 1) -- Process fewer samples -- Use a smaller model - -### Slow Processing -- Ensure you're using GPU if available -- Reduce number of epochs -- Process files in batches - -## Performance Benchmarks - -### LoRA vs Full Fine-Tuning - -Typical performance on STT tasks: -- **LoRA**: WER/CER within 0.3-0.5% of full fine-tuning -- **Training time**: 3-5x faster with LoRA -- **Memory usage**: 3-5x less with LoRA -- **Model size**: LoRA adapters ~10-50MB vs full model ~300MB+ - -### LLM Inference Speed - -With optimizations enabled (fast_mode=True, 4-bit quantization): -- **Target**: <1 second per transcript -- **Typical**: 0.5-2 seconds depending on transcript length and hardware -- **Without optimizations**: 3-10+ seconds per transcript - -## Notes - -- The script only fine-tunes on **error cases** (samples where STT transcript != LLM gold standard) -- WER/CER are calculated using `jiwer` library -- With LoRA: Only adapters are saved (much smaller files) -- With Full Fine-tuning: Complete model is saved -- Training history and logs are saved to `{output_dir}/logs/` -- LoRA adapters can be merged into base model for standalone inference if needed - diff --git a/docs/FINETUNING_ORCHESTRATION.md b/docs/FINETUNING_ORCHESTRATION.md deleted file mode 100644 index 836d00d..0000000 --- a/docs/FINETUNING_ORCHESTRATION.md +++ /dev/null @@ -1,655 +0,0 @@ -# Fine-Tuning Orchestration System - -**Complete Guide to Automated Model Fine-Tuning, Validation, and Deployment** - -## Overview - -The Fine-Tuning Orchestration System provides a comprehensive, automated pipeline for improving speech-to-text models through continuous learning from error cases. The system handles the complete lifecycle: - -1. **Automated Triggering** - Monitors error accumulation and triggers fine-tuning -2. **Model Validation** - Validates models against baseline with statistical testing -3. **Version Management** - Manages model versions with deployment and rollback -4. **Regression Testing** - Prevents performance degradation - ---- - -## System Architecture - -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Fine-Tuning Coordinator โ”‚ -โ”‚ (Central Orchestration) โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ - โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” - โ”‚ โ”‚ โ”‚ โ”‚ - โ–ผ โ–ผ โ–ผ โ–ผ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Fine-Tuning โ”‚ โ”‚ Model โ”‚ โ”‚ Model โ”‚ โ”‚ Regression โ”‚ -โ”‚ Orchestrator โ”‚ โ”‚ Validator โ”‚ โ”‚ Deployer โ”‚ โ”‚ Tester โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ โ”‚ โ”‚ โ”‚ - โ–ผ โ–ผ โ–ผ โ–ผ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Data โ”‚ โ”‚ Evaluation โ”‚ โ”‚ Version โ”‚ โ”‚ Test Suites โ”‚ -โ”‚ Manager โ”‚ โ”‚ Metrics โ”‚ โ”‚ Control โ”‚ โ”‚ โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ โ”‚ โ”‚ โ”‚ - โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ - โ–ผ - โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” - โ”‚ Google Cloud โ”‚ - โ”‚ Storage โ”‚ - โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - ---- - -## Components - -### 1. Fine-Tuning Orchestrator - -**Location:** `src/data/finetuning_orchestrator.py` - -**Purpose:** Monitors error cases and automatically triggers fine-tuning when thresholds are met. - -**Key Features:** -- Automatic trigger based on configurable thresholds -- Dataset preparation from failed cases -- Job management and tracking -- Integration with version control -- Monitoring loop for continuous operation - -**Configuration:** -```python -from src.data.finetuning_orchestrator import FinetuningConfig - -config = FinetuningConfig( - min_error_cases=100, # Minimum cases before triggering - min_corrected_cases=50, # Minimum corrected cases - trigger_on_error_rate=True, # Trigger on error rate - error_rate_threshold=0.15, # 15% error rate threshold - auto_approve_finetuning=False # Require manual approval -) -``` - -**Usage:** -```python -from src.data.finetuning_orchestrator import FinetuningOrchestrator -from src.data.data_manager import DataManager - -# Initialize -data_manager = DataManager(use_gcs=True) -orchestrator = FinetuningOrchestrator( - data_manager=data_manager, - config=config -) - -# Check trigger conditions -trigger_result = orchestrator.check_trigger_conditions() - -if trigger_result['should_trigger']: - # Trigger fine-tuning - job = orchestrator.trigger_finetuning(force=True) - print(f"Job created: {job.job_id}") -``` - ---- - -### 2. Model Validator - -**Location:** `src/data/model_validator.py` - -**Purpose:** Validates fine-tuned models against baseline using standardized evaluation sets. - -**Key Features:** -- Compare model performance vs baseline -- Statistical significance testing -- Multi-metric evaluation (WER, CER) -- Per-sample analysis -- Degradation detection - -**Configuration:** -```python -from src.data.model_validator import ValidationConfig - -config = ValidationConfig( - min_wer_improvement=0.0, # Minimum improvement required - min_cer_improvement=0.0, - require_significance=True, # Require statistical significance - significance_alpha=0.05, # P-value threshold - max_wer_degradation_rate=0.1, # Max 10% samples can degrade - require_no_major_degradation=True # No >50% degradation per sample -) -``` - -**Usage:** -```python -from src.data.model_validator import ModelValidator - -validator = ModelValidator(config=config, use_gcs=True) - -# Validate model -result = validator.validate_model( - model_id="finetuned_v1", - model_transcribe_fn=fine_tuned_transcribe, - baseline_id="baseline_v1", - baseline_transcribe_fn=baseline_transcribe, - evaluation_set_path="data/evaluation/test_set.jsonl" -) - -print(f"Validation {'PASSED' if result.passed else 'FAILED'}") -print(f"WER Improvement: {result.wer_improvement:+.4f}") -``` - ---- - -### 3. Model Deployer - -**Location:** `src/data/model_deployer.py` - -**Purpose:** Manages model versioning, deployment, and rollback. - -**Key Features:** -- Model version registry -- Deployment with backup -- Rollback capability -- Version history tracking -- GCS synchronization - -**Configuration:** -```python -from src.data.model_deployer import DeploymentConfig - -config = DeploymentConfig( - deployment_strategy="replace", # 'replace', 'canary', 'blue_green' - keep_previous_versions=5, # Keep 5 previous versions - auto_backup_before_deploy=True, - enable_auto_rollback=True, - rollback_on_error_threshold=0.5 -) -``` - -**Usage:** -```python -from src.data.model_deployer import ModelDeployer - -deployer = ModelDeployer(config=config, use_gcs=True) - -# Register model -version_id = deployer.register_model( - model_name="fine-tuned-stt", - model_path="/path/to/model", - validation_result=validation_result.to_dict() -) - -# Deploy model -deployer.deploy_model(version_id) - -# Rollback if needed -deployer.rollback() # Rollback to previous version -``` - ---- - -### 4. Regression Tester - -**Location:** `src/data/regression_tester.py` - -**Purpose:** Prevents model degradation through continuous testing. - -**Key Features:** -- Register regression test suites -- Track baseline performance -- Detect degradation -- Per-test and aggregate metrics -- Test history tracking - -**Configuration:** -```python -from src.data.regression_tester import RegressionConfig - -config = RegressionConfig( - run_on_deploy=True, - fail_on_critical_degradation=True, - critical_degradation_threshold=0.1, # 10% degradation is critical - max_failed_samples_rate=0.05, # 5% samples can fail - sample_degradation_threshold=0.2 # 20% per-sample threshold -) -``` - -**Usage:** -```python -from src.data.regression_tester import RegressionTester - -tester = RegressionTester(config=config, use_gcs=True) - -# Register test -test_id = tester.register_test( - test_name="Critical Benchmark", - test_type="benchmark", - test_data_path="data/evaluation/benchmark.jsonl", - baseline_wer=0.15, - baseline_cer=0.08, - baseline_version="baseline_v1", - max_wer_degradation=0.05 -) - -# Run test -result = tester.run_test( - test_id=test_id, - model_version="finetuned_v1", - model_transcribe_fn=model_transcribe -) - -# Run full test suite -suite_results = tester.run_test_suite( - model_version="finetuned_v1", - model_transcribe_fn=model_transcribe -) -``` - ---- - -### 5. Fine-Tuning Coordinator - -**Location:** `src/data/finetuning_coordinator.py` - -**Purpose:** Central coordinator that orchestrates the complete workflow. - -**Key Features:** -- Complete workflow automation -- Component integration -- Callback management -- Status monitoring -- Workflow tracking - -**Usage:** -```python -from src.data.finetuning_coordinator import FinetuningCoordinator -from src.data.data_manager import DataManager - -# Initialize -data_manager = DataManager(use_gcs=True) -coordinator = FinetuningCoordinator( - data_manager=data_manager, - use_gcs=True -) - -# Set callbacks -coordinator.set_training_callback(custom_training_function) -coordinator.set_baseline_transcribe_function(baseline_transcribe) -coordinator.set_model_transcribe_function_factory(model_factory) - -# Run complete workflow -workflow = coordinator.run_complete_workflow( - force_trigger=True, - auto_deploy=True -) - -# Check status -coordinator.print_status() -``` - ---- - -## Complete Workflow Example - -### Step 1: Setup - -```python -from src.data.data_manager import DataManager -from src.data.finetuning_coordinator import FinetuningCoordinator -from src.baseline_model import BaselineSTTModel - -# Initialize components -data_manager = DataManager(use_gcs=True) -baseline_model = BaselineSTTModel() - -# Initialize coordinator -coordinator = FinetuningCoordinator( - data_manager=data_manager, - use_gcs=True, - project_id="your-gcp-project" -) -``` - -### Step 2: Accumulate Error Cases - -```python -# Store failed cases as they occur -case_id = data_manager.store_failed_case( - audio_path="audio/sample.wav", - original_transcript="incorrect transcription", - corrected_transcript="correct transcription", - error_types=["word_substitution"], - error_score=0.85, - metadata={"source": "production"} -) -``` - -### Step 3: Monitor and Trigger - -```python -# Check if ready to trigger -trigger_result = coordinator.orchestrator.check_trigger_conditions() - -if trigger_result['should_trigger']: - print("Ready to trigger fine-tuning!") - - # Trigger workflow - workflow = coordinator.run_complete_workflow( - force_trigger=True, - auto_deploy=False # Manual deployment for safety - ) -``` - -### Step 4: Validate and Deploy - -```python -# After training completes, deploy the model -coordinator.deploy_job_model( - job_id=workflow['stages']['trigger']['job_id'], - model_path="/path/to/trained/model", - run_validation=True, - run_regression=True -) -``` - ---- - -## Google Cloud Platform Integration - -### Prerequisites - -1. **GCP Setup:** - ```bash - # Install Google Cloud SDK - curl https://sdk.cloud.google.com | bash - - # Authenticate - gcloud auth login - gcloud config set project your-project-id - ``` - -2. **Create Storage Buckets:** - ```bash - gsutil mb gs://your-project-datasets - gsutil mb gs://your-project-models - ``` - -3. **Set Permissions:** - ```bash - gcloud projects add-iam-policy-binding your-project-id \ - --member="serviceAccount:your-service-account@your-project.iam.gserviceaccount.com" \ - --role="roles/storage.objectAdmin" - ``` - -### Deploy Fine-Tuning to GCP - -```bash -# Create GPU VM -python scripts/deploy_finetuning_to_gcp.py \ - --create-vm \ - --machine-type n1-standard-8 - -# Upload code and prepare -python scripts/deploy_finetuning_to_gcp.py \ - --prepare-dataset - -# Run training -python scripts/deploy_finetuning_to_gcp.py \ - --run-training \ - --dataset-id finetuning_dataset_20231201_120000 \ - --epochs 5 - -# Download trained model -python scripts/deploy_finetuning_to_gcp.py \ - --download-model ~/stt-project/models/finetuned_model \ - --local-dest ./models - -# Clean up -python scripts/deploy_finetuning_to_gcp.py --stop-vm -``` - ---- - -## Configuration Best Practices - -### Development Environment - -```python -# Use local storage, low thresholds -config = FinetuningConfig( - min_error_cases=10, - auto_approve_finetuning=True -) - -coordinator = FinetuningCoordinator( - data_manager=data_manager, - use_gcs=False # Local storage -) -``` - -### Production Environment - -```python -# Use GCS, higher thresholds, manual approval -config = FinetuningConfig( - min_error_cases=100, - min_corrected_cases=50, - trigger_on_error_rate=True, - error_rate_threshold=0.15, - auto_approve_finetuning=False # Require approval -) - -coordinator = FinetuningCoordinator( - data_manager=data_manager, - use_gcs=True, - project_id="production-project-id" -) -``` - ---- - -## Monitoring and Alerting - -### Get System Status - -```python -# Get comprehensive status -status = coordinator.get_system_status() - -print(f"Active model: {status['deployer']['active_version']}") -print(f"Error cases: {status['orchestrator']['trigger_conditions']['metrics']['total_error_cases']}") -print(f"Validation pass rate: {status['validation']['pass_rate']}") -``` - -### Track Metrics - -```python -from src.data.metadata_tracker import MetadataTracker - -tracker = MetadataTracker(use_gcs=True) - -# Get performance trends -wer_trend = tracker.get_performance_trend('wer', time_window_days=30) -print(f"WER improvement: {wer_trend['improvement']:.4f}") - -# Get inference statistics -stats = tracker.get_inference_statistics(time_window_hours=24) -print(f"Error detection rate: {stats['error_detection_rate']:.2%}") -``` - ---- - -## Troubleshooting - -### Common Issues - -1. **Trigger Not Working:** - ```python - # Check trigger conditions - result = orchestrator.check_trigger_conditions() - print(result['reasons']) - print(result['metrics']) - ``` - -2. **Validation Failing:** - ```python - # Check validation configuration - print(validator.config.to_dict()) - - # Review validation results - result = validator.validate_model(...) - print(result.failure_reason) - ``` - -3. **Deployment Issues:** - ```python - # Check deployment status - deployer.print_status() - - # Review deployment history - history = deployer.get_deployment_history() - for deployment in history: - print(f"{deployment['version_id']}: {deployment['status']}") - ``` - -### Rollback Procedure - -```python -# If model performs poorly in production -deployer = ModelDeployer(use_gcs=True) - -# Rollback to previous version -success = deployer.rollback() - -if success: - print("Rolled back to previous version") -else: - # Manual rollback to specific version - deployer.rollback(target_version_id="model_v123") -``` - ---- - -## Testing - -### Run Demo - -```bash -# Run comprehensive demo -python experiments/demo_finetuning_orchestration.py -``` - -### Unit Tests - -```bash -# Run specific component tests -pytest tests/test_finetuning_orchestrator.py -pytest tests/test_model_validator.py -pytest tests/test_model_deployer.py -pytest tests/test_regression_tester.py -``` - ---- - -## API Reference - -### Quick Reference - -```python -# Data Manager -data_manager.store_failed_case(...) -data_manager.get_statistics() - -# Fine-Tuning Orchestrator -orchestrator.check_trigger_conditions() -orchestrator.trigger_finetuning(force=True) -orchestrator.get_job_info(job_id) - -# Model Validator -validator.validate_model(model_id, ...) -validator.get_best_model(metric='wer') - -# Model Deployer -deployer.register_model(...) -deployer.deploy_model(version_id) -deployer.rollback() - -# Regression Tester -tester.register_test(...) -tester.run_test_suite(...) - -# Coordinator -coordinator.run_complete_workflow(...) -coordinator.get_system_status() -coordinator.print_status() -``` - ---- - -## Performance Considerations - -### Scalability - -- **Dataset Size:** System handles 1000s of error cases efficiently -- **Concurrent Jobs:** Supports multiple fine-tuning jobs -- **GCS Integration:** Offloads storage to cloud for scalability - -### Optimization Tips - -1. **Batch Operations:** Accumulate cases before triggering -2. **Parallel Processing:** Use GCP VMs for parallel training -3. **Caching:** Version control caches metadata locally -4. **Incremental Updates:** Only sync changed data to GCS - ---- - -## Security - -### Best Practices - -1. **Authentication:** Use service accounts for GCS access -2. **Permissions:** Follow principle of least privilege -3. **Data Privacy:** Encrypt sensitive audio/transcript data -4. **Audit Logging:** All operations are logged with timestamps - -### Configuration - -```python -# Use service account key -os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = '/path/to/key.json' - -coordinator = FinetuningCoordinator( - data_manager=data_manager, - use_gcs=True, - project_id="your-project" -) -``` - ---- - -## Support and Contributing - -### Documentation -- Full API docs: `docs/API_REFERENCE.md` -- Setup guide: `docs/SETUP_INSTRUCTIONS.md` -- GCP guide: `docs/GCP_SETUP_GUIDE.md` - -### Examples -- Demo script: `experiments/demo_finetuning_orchestration.py` -- Test cases: `tests/` - -### Questions -For questions or issues, see the project repository. - ---- - -## License - -This system is part of the Adaptive Self-Learning Agentic AI System project. - - diff --git a/docs/FINETUNING_QUICK_START.md b/docs/FINETUNING_QUICK_START.md deleted file mode 100644 index 8950834..0000000 --- a/docs/FINETUNING_QUICK_START.md +++ /dev/null @@ -1,421 +0,0 @@ -# Fine-Tuning Orchestration - Quick Start Guide - -Get started with the automated fine-tuning orchestration system in 5 minutes. - -## ๐Ÿš€ Quick Start - -### 1. Run the Demo - -```bash -# Run comprehensive demo -python experiments/demo_finetuning_orchestration.py -``` - -This demonstrates all components: -- โœ… Data Manager (error tracking) -- โœ… Fine-Tuning Orchestrator (automated triggering) -- โœ… Model Validator (baseline comparison) -- โœ… Model Deployer (version management) -- โœ… Regression Tester (degradation prevention) -- โœ… Complete Workflow Coordinator - -### 2. Basic Usage - -```python -from src.data.data_manager import DataManager -from src.data.finetuning_coordinator import FinetuningCoordinator - -# Initialize -data_manager = DataManager(use_gcs=False) # Local for testing -coordinator = FinetuningCoordinator( - data_manager=data_manager, - use_gcs=False -) - -# Check status -coordinator.print_status() - -# Check if ready to trigger -trigger_result = coordinator.orchestrator.check_trigger_conditions() -print(f"Should trigger: {trigger_result['should_trigger']}") - -# Trigger fine-tuning (when ready) -if trigger_result['should_trigger']: - workflow = coordinator.run_complete_workflow( - force_trigger=True, - auto_deploy=False # Manual approval for safety - ) -``` - ---- - -## ๐Ÿ“‹ 4 Core Components - -### 1. Automated Fine-Tuning Orchestrator - -**What it does:** Monitors error cases and triggers fine-tuning automatically - -```python -from src.data.finetuning_orchestrator import FinetuningOrchestrator, FinetuningConfig - -config = FinetuningConfig( - min_error_cases=100, # Trigger after 100 errors - auto_approve_finetuning=False # Require approval -) - -orchestrator = FinetuningOrchestrator( - data_manager=data_manager, - config=config -) - -# Trigger fine-tuning -job = orchestrator.trigger_finetuning(force=True) -``` - -**Key Features:** -- โœ… Automatic trigger based on error thresholds -- โœ… Dataset preparation from failed cases -- โœ… Job management and tracking -- โœ… Integration with GCS - -### 2. Model Validation System - -**What it does:** Validates fine-tuned models against baseline - -```python -from src.data.model_validator import ModelValidator, ValidationConfig - -config = ValidationConfig( - min_wer_improvement=0.0, - require_significance=True -) - -validator = ModelValidator(config=config) - -# Validate model -result = validator.validate_model( - model_id="finetuned_v1", - model_transcribe_fn=model_transcribe, - baseline_id="baseline_v1", - baseline_transcribe_fn=baseline_transcribe -) - -print(f"Passed: {result.passed}") -print(f"WER Improvement: {result.wer_improvement:+.4f}") -``` - -**Key Features:** -- โœ… Baseline comparison -- โœ… Statistical significance testing -- โœ… Multi-metric evaluation (WER, CER) -- โœ… Per-sample analysis - -### 3. Model Versioning & Deployment - -**What it does:** Manages model versions and deployment - -```python -from src.data.model_deployer import ModelDeployer, DeploymentConfig - -config = DeploymentConfig( - keep_previous_versions=5, - auto_backup_before_deploy=True -) - -deployer = ModelDeployer(config=config) - -# Register and deploy -version_id = deployer.register_model( - model_name="fine-tuned-stt", - model_path="/path/to/model", - validation_result=validation_result.to_dict() -) - -deployer.deploy_model(version_id) - -# Rollback if needed -deployer.rollback() -``` - -**Key Features:** -- โœ… Version registry and history -- โœ… Automated backup before deployment -- โœ… One-click rollback -- โœ… GCS synchronization - -### 4. Regression Testing - -**What it does:** Prevents model degradation - -```python -from src.data.regression_tester import RegressionTester, RegressionConfig - -config = RegressionConfig( - fail_on_critical_degradation=True, - critical_degradation_threshold=0.1 -) - -tester = RegressionTester(config=config) - -# Register test -test_id = tester.register_test( - test_name="Critical Benchmark", - test_type="benchmark", - test_data_path="data/evaluation/test.jsonl", - baseline_wer=0.15, - baseline_cer=0.08, - baseline_version="baseline_v1" -) - -# Run tests -results = tester.run_test_suite( - model_version="finetuned_v1", - model_transcribe_fn=model_transcribe -) -``` - -**Key Features:** -- โœ… Multiple test suites (benchmark, critical, edge cases) -- โœ… Baseline tracking -- โœ… Automated degradation detection -- โœ… Test history and trends - ---- - -## ๐Ÿ”„ Complete Workflow - -The system automates the entire fine-tuning lifecycle: - -``` -1. Monitor Error Cases - โ””โ”€> Accumulate failed transcriptions - โ””โ”€> Track corrections - -2. Trigger Fine-Tuning (automatic when threshold met) - โ””โ”€> Prepare dataset from error cases - โ””โ”€> Create data version - โ””โ”€> Launch training job - -3. Validate Model - โ””โ”€> Compare against baseline - โ””โ”€> Statistical significance test - โ””โ”€> Check quality metrics - -4. Run Regression Tests - โ””โ”€> Test critical samples - โ””โ”€> Check for degradation - โ””โ”€> Verify edge cases - -5. Deploy Model (if validation passes) - โ””โ”€> Register new version - โ””โ”€> Backup current model - โ””โ”€> Deploy new version - โ””โ”€> Update active pointer - -6. Continuous Monitoring - โ””โ”€> Track performance metrics - โ””โ”€> Alert on degradation - โ””โ”€> Enable rollback -``` - ---- - -## โ˜๏ธ Google Cloud Integration - -### Setup GCP - -```bash -# Authenticate -gcloud auth login -gcloud config set project your-project-id - -# Create buckets -gsutil mb gs://your-project-datasets -gsutil mb gs://your-project-models -``` - -### Enable GCS in Code - -```python -# Enable GCS for all components -coordinator = FinetuningCoordinator( - data_manager=data_manager, - use_gcs=True, # โ† Enable GCS - project_id="your-project-id" -) -``` - -### Deploy to GCP for Training - -```bash -# Create GPU VM and run fine-tuning -python scripts/deploy_finetuning_to_gcp.py \ - --create-vm \ - --prepare-dataset \ - --run-training \ - --dataset-id your_dataset_id -``` - ---- - -## ๐Ÿ“Š Monitoring - -### Check System Status - -```python -# Print comprehensive status -coordinator.print_status() - -# Get detailed status -status = coordinator.get_system_status() - -# Check specific components -trigger_result = coordinator.orchestrator.check_trigger_conditions() -validation_report = coordinator.validator.generate_validation_report() -deployment_report = coordinator.deployer.generate_deployment_report() -regression_report = coordinator.regression_tester.generate_regression_report() -``` - -### Track Metrics - -```python -from src.data.metadata_tracker import MetadataTracker - -tracker = MetadataTracker(use_gcs=True) - -# Performance trends -wer_trend = tracker.get_performance_trend('wer', time_window_days=30) -print(f"WER improvement over 30 days: {wer_trend['improvement']:.4f}") - -# Recent inference stats -stats = tracker.get_inference_statistics(time_window_hours=24) -print(f"Error detection rate: {stats['error_detection_rate']:.2%}") -``` - ---- - -## ๐Ÿ› ๏ธ Configuration - -### Development (Local Testing) - -```python -from src.data.finetuning_orchestrator import FinetuningConfig - -config = FinetuningConfig( - min_error_cases=10, # Low threshold - auto_approve_finetuning=True # Auto-approve -) - -coordinator = FinetuningCoordinator( - data_manager=data_manager, - finetuning_config=config, - use_gcs=False # Local storage -) -``` - -### Production - -```python -config = FinetuningConfig( - min_error_cases=100, # Higher threshold - min_corrected_cases=50, - trigger_on_error_rate=True, - error_rate_threshold=0.15, - auto_approve_finetuning=False # Manual approval -) - -coordinator = FinetuningCoordinator( - data_manager=data_manager, - finetuning_config=config, - use_gcs=True, # Cloud storage - project_id="production-project" -) -``` - ---- - -## ๐Ÿšจ Troubleshooting - -### Issue: Fine-tuning not triggering - -```python -# Check trigger conditions -result = orchestrator.check_trigger_conditions() -print(f"Should trigger: {result['should_trigger']}") -print(f"Reasons: {result['reasons']}") -print(f"Current error cases: {result['metrics']['total_error_cases']}") -print(f"Required: {orchestrator.config.min_error_cases}") -``` - -### Issue: Validation failing - -```python -# Check validation configuration -print(validator.config.to_dict()) - -# Review detailed results -result = validator.validate_model(...) -if not result.passed: - print(f"Failure reason: {result.failure_reason}") - print(f"Failed samples: {len(result.failed_samples)}") -``` - -### Issue: Deployment problems - -```python -# Check deployment status -deployer.print_status() - -# Review version history -for version in deployer.list_versions(limit=5): - print(f"{version.version_id}: {version.status}") - -# Rollback if needed -deployer.rollback() -``` - ---- - -## ๐Ÿ“š Further Reading - -- **Full Documentation:** `docs/FINETUNING_ORCHESTRATION.md` -- **API Reference:** `docs/API_REFERENCE.md` (if exists) -- **GCP Setup:** `docs/GCP_SETUP_GUIDE.md` -- **Testing Guide:** `docs/TESTING_GUIDE.md` - ---- - -## ๐Ÿ’ก Tips - -1. **Start Local:** Test with `use_gcs=False` first -2. **Low Thresholds:** Use low `min_error_cases` for testing -3. **Manual Approval:** Keep `auto_approve_finetuning=False` in production -4. **Monitor Metrics:** Regularly check system status -5. **Test Rollback:** Practice rollback procedure before production - ---- - -## ๐ŸŽฏ Next Steps - -1. โœ… Run demo: `python experiments/demo_finetuning_orchestration.py` -2. โœ… Configure for your use case -3. โœ… Set up GCP (optional but recommended) -4. โœ… Integrate with your training pipeline -5. โœ… Set up monitoring and alerts -6. โœ… Test rollback procedure - ---- - -## ๐Ÿค Support - -- Check `docs/` for detailed documentation -- Review `examples/` for more use cases -- See `tests/` for test examples - -**Ready to automate your fine-tuning? Start with the demo!** - -```bash -python experiments/demo_finetuning_orchestration.py -``` - - diff --git a/docs/FINETUNING_SYSTEM_SUMMARY.md b/docs/FINETUNING_SYSTEM_SUMMARY.md deleted file mode 100644 index 61cc3d8..0000000 --- a/docs/FINETUNING_SYSTEM_SUMMARY.md +++ /dev/null @@ -1,527 +0,0 @@ -# Fine-Tuning Orchestration System - Implementation Summary - -## ๐ŸŽฏ Overview - -This document summarizes the complete Fine-Tuning Orchestration System built for the Adaptive Self-Learning Agentic AI System project. The system provides end-to-end automation of model fine-tuning, validation, deployment, and monitoring. - ---- - -## โœ… Implemented Components - -### 1. Automated Fine-Tuning Pipeline โœ… - -**File:** `src/data/finetuning_orchestrator.py` - -**Features Implemented:** -- โœ… Automatic monitoring of error case accumulation -- โœ… Configurable trigger thresholds (error count, correction rate, error rate) -- โœ… Automated dataset preparation from failed cases -- โœ… Job management and tracking -- โœ… Integration with data manager and version control -- โœ… Continuous monitoring loop -- โœ… Manual and automatic approval workflows -- โœ… GCS integration for cloud storage - -**Key Classes:** -- `FinetuningConfig` - Configuration for trigger conditions -- `FinetuningJob` - Job state tracking -- `FinetuningOrchestrator` - Main orchestration logic - -**Usage Example:** -```python -orchestrator = FinetuningOrchestrator( - data_manager=data_manager, - config=FinetuningConfig(min_error_cases=100) -) -job = orchestrator.trigger_finetuning(force=True) -``` - ---- - -### 2. Model Validation System โœ… - -**File:** `src/data/model_validator.py` - -**Features Implemented:** -- โœ… Baseline comparison with standardized evaluation sets -- โœ… Statistical significance testing (paired t-test) -- โœ… Multi-metric evaluation (WER, CER) -- โœ… Per-sample analysis and degradation detection -- โœ… Configurable quality gates and thresholds -- โœ… Validation result tracking and history -- โœ… Best model selection -- โœ… Comprehensive reporting - -**Key Classes:** -- `ValidationConfig` - Validation criteria configuration -- `ValidationResult` - Validation outcome with metrics -- `ModelValidator` - Validation orchestration - -**Usage Example:** -```python -validator = ModelValidator(config=ValidationConfig()) -result = validator.validate_model( - model_id="finetuned_v1", - model_transcribe_fn=model_fn, - baseline_id="baseline_v1", - baseline_transcribe_fn=baseline_fn -) -``` - ---- - -### 3. Model Versioning & Deployment System โœ… - -**File:** `src/data/model_deployer.py` - -**Features Implemented:** -- โœ… Model version registry with metadata -- โœ… Deployment with automatic backup -- โœ… Rollback to previous versions -- โœ… Version history tracking -- โœ… Multiple deployment strategies support -- โœ… Automatic cleanup of old versions -- โœ… GCS synchronization -- โœ… Deployment status monitoring - -**Key Classes:** -- `DeploymentConfig` - Deployment settings -- `ModelVersion` - Version metadata -- `ModelDeployer` - Deployment orchestration - -**Usage Example:** -```python -deployer = ModelDeployer(config=DeploymentConfig()) -version_id = deployer.register_model( - model_name="fine-tuned-stt", - model_path="/path/to/model" -) -deployer.deploy_model(version_id) -``` - ---- - -### 4. Regression Testing Framework โœ… - -**File:** `src/data/regression_tester.py` - -**Features Implemented:** -- โœ… Regression test suite management -- โœ… Baseline performance tracking -- โœ… Automated degradation detection -- โœ… Per-sample and aggregate metrics -- โœ… Multiple test types (benchmark, critical, edge cases) -- โœ… Configurable degradation thresholds -- โœ… Test history and trends -- โœ… Comprehensive reporting - -**Key Classes:** -- `RegressionConfig` - Testing configuration -- `RegressionTest` - Test definition -- `RegressionTestResult` - Test outcome -- `RegressionTester` - Test orchestration - -**Usage Example:** -```python -tester = RegressionTester(config=RegressionConfig()) -test_id = tester.register_test( - test_name="Critical Benchmark", - test_data_path="data/test.jsonl", - baseline_wer=0.15 -) -results = tester.run_test_suite( - model_version="v1", - model_transcribe_fn=model_fn -) -``` - ---- - -### 5. Central Coordination System โœ… - -**File:** `src/data/finetuning_coordinator.py` - -**Features Implemented:** -- โœ… Complete workflow orchestration -- โœ… Integration of all components -- โœ… Callback management for custom training -- โœ… Workflow state tracking -- โœ… Comprehensive status monitoring -- โœ… End-to-end automation -- โœ… Error handling and recovery - -**Key Class:** -- `FinetuningCoordinator` - Central orchestration - -**Usage Example:** -```python -coordinator = FinetuningCoordinator( - data_manager=data_manager, - use_gcs=True -) -workflow = coordinator.run_complete_workflow( - force_trigger=True, - auto_deploy=True -) -``` - ---- - -## ๐Ÿš€ Google Cloud Platform Integration - -### GCP Deployment Script โœ… - -**File:** `scripts/deploy_finetuning_to_gcp.py` - -**Features Implemented:** -- โœ… Automated VM creation with GPU support -- โœ… Code and dependency deployment -- โœ… Dataset preparation on GCP -- โœ… Training job execution -- โœ… Model download from GCP -- โœ… VM lifecycle management (stop/delete) -- โœ… Cost optimization features - -**Usage Example:** -```bash -python scripts/deploy_finetuning_to_gcp.py \ - --create-vm \ - --prepare-dataset \ - --run-training \ - --dataset-id dataset_123 -``` - ---- - -## ๐Ÿ“š Documentation - -### Comprehensive Documentation โœ… - -**Files Created:** -1. **`docs/FINETUNING_ORCHESTRATION.md`** (Main documentation) - - Complete system architecture - - Component details - - Configuration guide - - API reference - - Troubleshooting guide - - Best practices - -2. **`docs/FINETUNING_QUICK_START.md`** (Quick start guide) - - 5-minute setup - - Basic usage examples - - Configuration templates - - Common patterns - -3. **`FINETUNING_SYSTEM_SUMMARY.md`** (This file) - - Implementation overview - - Component summary - - File structure - ---- - -## ๐Ÿงช Demo and Testing - -### Comprehensive Demo โœ… - -**File:** `experiments/demo_finetuning_orchestration.py` - -**Features:** -- โœ… Data Manager demonstration -- โœ… Orchestrator trigger demo -- โœ… Validation demo -- โœ… Deployment demo -- โœ… Regression testing demo -- โœ… Complete workflow simulation -- โœ… Status monitoring examples - -**Run Demo:** -```bash -python experiments/demo_finetuning_orchestration.py -``` - ---- - -## ๐Ÿ“ File Structure - -``` -src/data/ -โ”œโ”€โ”€ finetuning_orchestrator.py # Automated triggering -โ”œโ”€โ”€ model_validator.py # Validation against baseline -โ”œโ”€โ”€ model_deployer.py # Version management & deployment -โ”œโ”€โ”€ regression_tester.py # Regression testing -โ”œโ”€โ”€ finetuning_coordinator.py # Central coordination -โ”œโ”€โ”€ data_manager.py # (Already existed) Error tracking -โ”œโ”€โ”€ finetuning_pipeline.py # (Already existed) Dataset prep -โ”œโ”€โ”€ version_control.py # (Already existed) Data versioning -โ””โ”€โ”€ metadata_tracker.py # (Already existed) Performance tracking - -scripts/ -โ””โ”€โ”€ deploy_finetuning_to_gcp.py # GCP deployment automation - -experiments/ -โ””โ”€โ”€ demo_finetuning_orchestration.py # Comprehensive demo - -docs/ -โ””โ”€โ”€ FINETUNING_ORCHESTRATION.md # Complete documentation -โ”œโ”€โ”€ FINETUNING_QUICK_START.md # Quick start guide -โ””โ”€โ”€ FINETUNING_SYSTEM_SUMMARY.md # This file -``` - ---- - -## ๐Ÿ”„ Complete Workflow - -The system implements a complete automated workflow: - -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ STEP 1: MONITOR & TRIGGER โ”‚ -โ”‚ โ€ข Accumulate error cases via DataManager โ”‚ -โ”‚ โ€ข Monitor thresholds (FinetuningOrchestrator) โ”‚ -โ”‚ โ€ข Auto-trigger when conditions met โ”‚ -โ”‚ โ€ข Prepare dataset (FinetuningDatasetPipeline) โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ - โ–ผ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ STEP 2: TRAIN MODEL โ”‚ -โ”‚ โ€ข Use prepared dataset โ”‚ -โ”‚ โ€ข Train on GCP GPU VM (optional) โ”‚ -โ”‚ โ€ข Save model artifacts โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ - โ–ผ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ STEP 3: VALIDATE MODEL โ”‚ -โ”‚ โ€ข Compare against baseline (ModelValidator) โ”‚ -โ”‚ โ€ข Calculate WER/CER improvements โ”‚ -โ”‚ โ€ข Statistical significance testing โ”‚ -โ”‚ โ€ข Check quality gates โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ - โ–ผ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ STEP 4: REGRESSION TESTS โ”‚ -โ”‚ โ€ข Run test suites (RegressionTester) โ”‚ -โ”‚ โ€ข Check for degradation โ”‚ -โ”‚ โ€ข Test critical samples โ”‚ -โ”‚ โ€ข Verify edge cases โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ - โ–ผ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ STEP 5: DEPLOY MODEL โ”‚ -โ”‚ โ€ข Register version (ModelDeployer) โ”‚ -โ”‚ โ€ข Backup current model โ”‚ -โ”‚ โ€ข Deploy new version โ”‚ -โ”‚ โ€ข Update active pointer โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ - โ–ผ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ STEP 6: MONITOR โ”‚ -โ”‚ โ€ข Track performance (MetadataTracker) โ”‚ -โ”‚ โ€ข Monitor for degradation โ”‚ -โ”‚ โ€ข Alert on issues โ”‚ -โ”‚ โ€ข Enable rollback if needed โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - ---- - -## ๐ŸŽ“ Key Design Principles - -### 1. Modularity -- Each component can be used independently -- Clear interfaces between components -- Easy to extend and customize - -### 2. Automation -- Minimal manual intervention required -- Configurable thresholds and triggers -- Self-monitoring and self-healing - -### 3. Safety -- Manual approval option for critical operations -- Automatic backups before deployment -- One-click rollback capability -- Regression testing to prevent degradation - -### 4. Scalability -- GCS integration for cloud storage -- Support for large datasets -- Parallel training on GCP -- Efficient caching and versioning - -### 5. Observability -- Comprehensive logging -- Metrics tracking -- Status monitoring -- Performance history - ---- - -## ๐Ÿ“Š Metrics and Monitoring - -The system tracks and reports: - -### Performance Metrics -- Word Error Rate (WER) -- Character Error Rate (CER) -- Error detection rate -- Correction rate -- Inference time - -### System Metrics -- Error case count -- Correction rate -- Fine-tuning job status -- Validation pass/fail rates -- Deployment history -- Test suite results - -### Monitoring Tools -```python -# Get comprehensive status -coordinator.print_status() - -# Get detailed metrics -status = coordinator.get_system_status() - -# Track trends -tracker = MetadataTracker() -trend = tracker.get_performance_trend('wer', time_window_days=30) -``` - ---- - -## ๐Ÿ”ง Configuration Options - -### Fine-Tuning Triggers -- `min_error_cases`: Minimum error cases to trigger -- `min_corrected_cases`: Minimum corrected cases -- `error_rate_threshold`: Error rate threshold -- `auto_approve_finetuning`: Auto-approval setting - -### Validation Criteria -- `min_wer_improvement`: Minimum WER improvement -- `require_significance`: Require statistical significance -- `max_wer_degradation_rate`: Max degradation rate allowed - -### Deployment Settings -- `deployment_strategy`: Deployment strategy -- `keep_previous_versions`: Number of versions to keep -- `auto_backup_before_deploy`: Auto-backup setting -- `enable_auto_rollback`: Auto-rollback on errors - -### Regression Testing -- `fail_on_critical_degradation`: Fail on critical degradation -- `critical_degradation_threshold`: Threshold for critical -- `max_failed_samples_rate`: Max failed samples rate - ---- - -## ๐Ÿšฆ Getting Started - -### 1. Quick Test (Local) -```bash -python experiments/demo_finetuning_orchestration.py -``` - -### 2. Production Setup -```python -from src.data.finetuning_coordinator import FinetuningCoordinator -from src.data.data_manager import DataManager - -# Initialize with GCS -data_manager = DataManager(use_gcs=True, project_id="your-project") -coordinator = FinetuningCoordinator( - data_manager=data_manager, - use_gcs=True, - project_id="your-project" -) - -# Configure callbacks -coordinator.set_training_callback(your_training_function) -coordinator.set_baseline_transcribe_function(baseline_fn) -coordinator.set_model_transcribe_function_factory(model_factory) - -# Monitor and trigger -coordinator.orchestrator.run_monitoring_loop( - check_interval_seconds=3600 # Check every hour -) -``` - -### 3. Deploy to GCP -```bash -# Setup and run fine-tuning on GCP -python scripts/deploy_finetuning_to_gcp.py \ - --create-vm \ - --prepare-dataset \ - --run-training \ - --dataset-id your_dataset_id -``` - ---- - -## ๐Ÿ“ˆ Benefits - -### For Development -- โœ… Faster iteration cycles -- โœ… Automated testing -- โœ… Easy rollback -- โœ… Clear metrics - -### For Operations -- โœ… Reduced manual intervention -- โœ… Consistent deployment process -- โœ… Audit trail -- โœ… Cost optimization (GCP lifecycle management) - -### For Quality -- โœ… Automated validation -- โœ… Regression prevention -- โœ… Performance tracking -- โœ… Data quality checks - ---- - -## ๐ŸŽฏ Next Steps - -1. **Testing:** Run the demo to understand the system -2. **Configuration:** Customize configs for your use case -3. **Integration:** Set up training callbacks -4. **Production:** Enable GCS and deploy to GCP -5. **Monitoring:** Set up alerts and dashboards - ---- - -## ๐Ÿ“ž Support - -- **Full Documentation:** `docs/FINETUNING_ORCHESTRATION.md` -- **Quick Start:** `docs/FINETUNING_QUICK_START.md` -- **Demo:** `experiments/demo_finetuning_orchestration.py` -- **API Reference:** See inline documentation in source files - ---- - -## โœจ Summary - -The Fine-Tuning Orchestration System provides a **production-ready, automated solution** for: -- โœ… Monitoring error cases -- โœ… Triggering fine-tuning automatically -- โœ… Validating models against baselines -- โœ… Managing versions and deployment -- โœ… Preventing regression -- โœ… Integrating with Google Cloud - -**Total Implementation:** -- 5 Core Components -- 1 GCP Deployment Script -- 1 Comprehensive Demo -- 3 Documentation Files -- ~2,500+ lines of production-ready code - -**Ready to use with minimal setup!** ๐Ÿš€ - - diff --git a/docs/GCP.md b/docs/GCP.md new file mode 100644 index 0000000..24f5699 --- /dev/null +++ b/docs/GCP.md @@ -0,0 +1,95 @@ +# Google Cloud Platform Guide + +Setup and deployment for GCP resources. + +## Install gcloud CLI + +### macOS (Homebrew) + +```bash +brew install --cask google-cloud-sdk +gcloud init +``` + +### Direct Download + +```bash +curl https://sdk.cloud.google.com | bash +exec -l $SHELL +gcloud init +``` + +### After Installation + +```bash +gcloud auth login +gcloud config set project your-project-id +gcloud services enable compute.googleapis.com storage-api.googleapis.com +``` + +--- + +## GPU VM for Training/Evaluation + +### Quick Start + +```bash +chmod +x scripts/setup_gcp_gpu.sh +bash scripts/setup_gcp_gpu.sh + +python scripts/deploy_to_gcp.py +python scripts/monitor_gcp_costs.py +``` + +### Manual VM Creation + +```bash +gcloud compute instances create stt-gpu-vm \ + --zone=us-central1-a \ + --machine-type=n1-standard-4 \ + --accelerator=type=nvidia-tesla-t4,count=1 \ + --image-family=pytorch-latest-gpu \ + --image-project=deeplearning-platform-release \ + --boot-disk-size=100GB +``` + +**Cost**: ~$0.54/hour (T4 GPU). Stop when not in use: `gcloud compute instances stop stt-gpu-vm --zone=us-central1-a` + +--- + +## Deployment Options + +- **Cloud Run**: Production API hosting +- **GPU VM**: Training and fine-tuning (see above) +- **App Engine**: Simple hosting +- **Cloud Storage**: Datasets and models + +### Create Storage Buckets + +```bash +gsutil mb gs://your-project-datasets +gsutil mb gs://your-project-models +``` + +### Fine-Tuning on GCP + +```bash +python scripts/deploy_finetuning_to_gcp.py \ + --create-vm --prepare-dataset --run-training \ + --dataset-id your_dataset_id +``` + +--- + +## Cost Management + +- **T4 + n1-standard-4**: ~$0.54/hour +- **Preemptible**: Add `--preemptible` for 60-80% savings +- **Stop VMs**: Always stop when not in use +- **Alerts**: GCP Console โ†’ Billing โ†’ Budgets & Alerts + +## Troubleshooting + +- **GPU not detected**: `nvidia-smi` on VM +- **Port/auth issues**: `gcloud auth login` +- **Quota**: `gcloud compute project-info describe` diff --git a/docs/GCP_DEPLOYMENT_GUIDE.md b/docs/GCP_DEPLOYMENT_GUIDE.md deleted file mode 100644 index 6ad874f..0000000 --- a/docs/GCP_DEPLOYMENT_GUIDE.md +++ /dev/null @@ -1,1000 +0,0 @@ -# Comprehensive Google Cloud Platform Deployment Guide - -## ๐Ÿ“‹ Table of Contents - -1. [Overview](#overview) -2. [Prerequisites](#prerequisites) -3. [Architecture Overview](#architecture-overview) -4. [Step-by-Step Deployment](#step-by-step-deployment) -5. [Deployment Options](#deployment-options) -6. [Cost Optimization](#cost-optimization) -7. [Monitoring & Maintenance](#monitoring--maintenance) -8. [Troubleshooting](#troubleshooting) -9. [Security Best Practices](#security-best-practices) - ---- - -## ๐ŸŽฏ Overview - -This guide provides comprehensive instructions for deploying the Adaptive Self-Learning Agentic AI System on Google Cloud Platform. The system can be deployed in multiple configurations depending on your needs: - -- **Option A**: Complete production deployment with Cloud Run + Cloud Storage -- **Option B**: GPU VM for model training and fine-tuning -- **Option C**: App Engine deployment for simple hosting -- **Option D**: Development/testing deployment - ---- - -## ๐Ÿ“‹ Prerequisites - -### Required Accounts & Tools - -1. **Google Cloud Platform Account** - - Active GCP account with billing enabled - - At least $50 in credits (recommended for testing) - - Project created (or use default) - -2. **Local Development Tools** - ```bash - # Install gcloud CLI (macOS) - brew install --cask google-cloud-sdk - - # Or download from: - # https://cloud.google.com/sdk/docs/install - - # Verify installation - gcloud --version - ``` - -3. **Required Software** - - Python 3.8+ (`python --version`) - - Docker Desktop (for containerized deployment) - - Git - - curl - -4. **Optional Tools** - - kubectl (for Kubernetes deployments) - - Terraform (for infrastructure as code) - -### Initial GCP Setup - -```bash -# 1. Authenticate with GCP -gcloud auth login - -# 2. Set your project ID (create one if needed) -export PROJECT_ID="your-project-id" -gcloud config set project $PROJECT_ID - -# 3. Enable required APIs -gcloud services enable compute.googleapis.com -gcloud services enable storage.googleapis.com -gcloud services enable run.googleapis.com -gcloud services enable cloudbuild.googleapis.com -gcloud services enable containerregistry.googleapis.com -gcloud services enable appengine.googleapis.com - -# 4. Set default region/zone -export REGION="us-central1" -export ZONE="us-central1-a" -gcloud config set compute/region $REGION -gcloud config set compute/zone $ZONE - -# 5. Verify setup -gcloud config list -``` - ---- - -## ๐Ÿ—๏ธ Architecture Overview - -### Component Architecture - -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Google Cloud Platform โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ โ”‚ -โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ -โ”‚ โ”‚ Cloud Run โ”‚ โ”‚ Cloud โ”‚ โ”‚ -โ”‚ โ”‚ (API) โ”‚โ—„โ”€โ”€โ”€โ”ค Storage โ”‚ โ”‚ -โ”‚ โ”‚ โ”‚ โ”‚ (GCS) โ”‚ โ”‚ -โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ -โ”‚ โ”‚ โ”‚ โ”‚ -โ”‚ โ”‚ โ”‚ โ”‚ -โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ–ผโ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ–ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ -โ”‚ โ”‚ GPU VM โ”‚ โ”‚ Artifact โ”‚ โ”‚ -โ”‚ โ”‚ (Training) โ”‚ โ”‚ Registry โ”‚ โ”‚ -โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ -โ”‚ โ”‚ -โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ -โ”‚ โ”‚ Cloud โ”‚ โ”‚ Cloud โ”‚ โ”‚ -โ”‚ โ”‚ Monitoring โ”‚ โ”‚ Logging โ”‚ โ”‚ -โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -### Storage Structure - -``` -GCS Buckets: -โ”œโ”€โ”€ stt-project-datasets/ -โ”‚ โ”œโ”€โ”€ raw/ # Raw audio files -โ”‚ โ”œโ”€โ”€ processed/ # Processed audio -โ”‚ โ””โ”€โ”€ finetuning/ # Training datasets -โ”œโ”€โ”€ stt-project-models/ -โ”‚ โ”œโ”€โ”€ baseline/ # Base models -โ”‚ โ”œโ”€โ”€ finetuned/ # Fine-tuned models -โ”‚ โ””โ”€โ”€ deployed/ # Active models -โ””โ”€โ”€ stt-project-logs/ - โ”œโ”€โ”€ training/ # Training logs - โ””โ”€โ”€ inference/ # Inference logs -``` - ---- - -## ๐Ÿš€ Step-by-Step Deployment - -### STEP 1: Clone and Setup Repository - -```bash -# 1. Clone repository -git clone -cd Adaptive-Self-Learning-Agentic-AI-System - -# 2. Create virtual environment -python3 -m venv venv -source venv/bin/activate # On Windows: venv\Scripts\activate - -# 3. Install dependencies -pip install -r requirements.txt - -# 4. Verify local setup -python scripts/verify_setup.py -``` - -### STEP 2: Configure GCP Storage - -```bash -# 1. Set bucket names (must be globally unique) -export PROJECT_ID="your-project-id" -export DATASETS_BUCKET="${PROJECT_ID}-stt-datasets" -export MODELS_BUCKET="${PROJECT_ID}-stt-models" -export LOGS_BUCKET="${PROJECT_ID}-stt-logs" - -# 2. Create GCS buckets -gsutil mb -p $PROJECT_ID -c STANDARD -l $REGION gs://$DATASETS_BUCKET -gsutil mb -p $PROJECT_ID -c STANDARD -l $REGION gs://$MODELS_BUCKET -gsutil mb -p $PROJECT_ID -c STANDARD -l $REGION gs://$LOGS_BUCKET - -# 3. Set lifecycle policies (optional - save costs) -cat > lifecycle.json << EOF -{ - "lifecycle": { - "rule": [ - { - "action": {"type": "Delete"}, - "condition": { - "age": 90, - "matchesPrefix": ["logs/"] - } - } - ] - } -} -EOF - -gsutil lifecycle set lifecycle.json gs://$LOGS_BUCKET - -# 4. Verify buckets -gsutil ls -p $PROJECT_ID - -# 5. Set bucket permissions (if needed for specific service accounts) -gsutil iam ch allUsers:objectViewer gs://$MODELS_BUCKET # Only if models should be public -``` - -### STEP 3: Setup Service Account - -```bash -# 1. Create service account -gcloud iam service-accounts create stt-service-account \ - --display-name="STT System Service Account" \ - --description="Service account for STT system operations" - -# 2. Grant necessary roles -gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member="serviceAccount:stt-service-account@${PROJECT_ID}.iam.gserviceaccount.com" \ - --role="roles/storage.objectAdmin" - -gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member="serviceAccount:stt-service-account@${PROJECT_ID}.iam.gserviceaccount.com" \ - --role="roles/logging.logWriter" - -gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member="serviceAccount:stt-service-account@${PROJECT_ID}.iam.gserviceaccount.com" \ - --role="roles/monitoring.metricWriter" - -# 3. Download service account key -gcloud iam service-accounts keys create ~/stt-service-account-key.json \ - --iam-account=stt-service-account@${PROJECT_ID}.iam.gserviceaccount.com - -# 4. Set environment variable -export GOOGLE_APPLICATION_CREDENTIALS=~/stt-service-account-key.json - -# 5. Verify service account -gcloud iam service-accounts list -``` - -### STEP 4: Upload Initial Data - -```bash -# 1. Upload test audio files -gsutil -m cp -r data/test_audio gs://$DATASETS_BUCKET/test_audio/ - -# 2. Upload any existing processed data -gsutil -m cp -r data/processed gs://$DATASETS_BUCKET/processed/ - -# 3. Verify upload -gsutil ls -r gs://$DATASETS_BUCKET -``` - -### STEP 5: Deploy Backend API (Cloud Run - Recommended) - -#### Option A: Deploy with Cloud Build - -```bash -# 1. Create Dockerfile (already provided below) -# See the Dockerfile in the next section - -# 2. Build and deploy with Cloud Build -gcloud builds submit --tag gcr.io/$PROJECT_ID/stt-api - -# 3. Deploy to Cloud Run -gcloud run deploy stt-api \ - --image gcr.io/$PROJECT_ID/stt-api \ - --platform managed \ - --region $REGION \ - --allow-unauthenticated \ - --memory 4Gi \ - --cpu 2 \ - --timeout 300 \ - --max-instances 10 \ - --set-env-vars="GCS_DATASETS_BUCKET=$DATASETS_BUCKET,GCS_MODELS_BUCKET=$MODELS_BUCKET,GCS_LOGS_BUCKET=$LOGS_BUCKET,USE_GCS=true" \ - --service-account=stt-service-account@${PROJECT_ID}.iam.gserviceaccount.com - -# 4. Get service URL -export SERVICE_URL=$(gcloud run services describe stt-api --region=$REGION --format='value(status.url)') -echo "Service URL: $SERVICE_URL" - -# 5. Test deployment -curl $SERVICE_URL/api/health -``` - -#### Option B: Deploy from Container Registry - -```bash -# 1. Build Docker image locally -docker build -t gcr.io/$PROJECT_ID/stt-api:latest . - -# 2. Configure Docker to use gcloud -gcloud auth configure-docker - -# 3. Push to Container Registry -docker push gcr.io/$PROJECT_ID/stt-api:latest - -# 4. Deploy to Cloud Run (same as above) -gcloud run deploy stt-api \ - --image gcr.io/$PROJECT_ID/stt-api:latest \ - --platform managed \ - --region $REGION \ - --allow-unauthenticated \ - --memory 4Gi \ - --cpu 2 \ - --timeout 300 \ - --max-instances 10 \ - --set-env-vars="GCS_DATASETS_BUCKET=$DATASETS_BUCKET,GCS_MODELS_BUCKET=$MODELS_BUCKET,GCS_LOGS_BUCKET=$LOGS_BUCKET,USE_GCS=true" -``` - -### STEP 6: Setup GPU VM for Training - -```bash -# 1. Use the optimized script -bash scripts/setup_gcp_gpu.sh - -# Or create manually: -gcloud compute instances create stt-training-vm \ - --zone=$ZONE \ - --machine-type=n1-standard-8 \ - --accelerator=type=nvidia-tesla-t4,count=1 \ - --image-family=pytorch-latest-gpu \ - --image-project=deeplearning-platform-release \ - --boot-disk-size=200GB \ - --boot-disk-type=pd-ssd \ - --maintenance-policy=TERMINATE \ - --metadata="install-nvidia-driver=True" \ - --scopes=cloud-platform \ - --service-account=stt-service-account@${PROJECT_ID}.iam.gserviceaccount.com - -# 2. Wait for VM to be ready (2-3 minutes) -sleep 180 - -# 3. SSH into VM -gcloud compute ssh stt-training-vm --zone=$ZONE - -# 4. On VM: Setup environment -# (Inside VM) -cd ~ -git clone -cd Adaptive-Self-Learning-Agentic-AI-System -pip install -r requirements.txt - -# 5. Verify GPU -python -c "import torch; print(f'CUDA: {torch.cuda.is_available()}')" - -# 6. Exit VM -exit - -# 7. Stop VM to save costs (start when needed) -gcloud compute instances stop stt-training-vm --zone=$ZONE -``` - -### STEP 7: Deploy Frontend - -The frontend is automatically served by the Cloud Run API at `/app`. - -```bash -# Test frontend access -curl $SERVICE_URL/app - -# Or open in browser -open $SERVICE_URL/app -``` - -For separate frontend hosting (optional): - -```bash -# Deploy to Firebase Hosting (alternative) -# 1. Install Firebase CLI -npm install -g firebase-tools - -# 2. Initialize Firebase -firebase init hosting - -# 3. Deploy frontend -firebase deploy --only hosting - -# Or deploy to Cloud Storage as static site -gsutil cp -r frontend/* gs://${PROJECT_ID}-frontend/ -gsutil web set -m index.html gs://${PROJECT_ID}-frontend -``` - -### STEP 8: Configure Environment Variables - -```bash -# Update your local .env file -cat > .env << EOF -# GCP Configuration -PROJECT_ID=$PROJECT_ID -REGION=$REGION -ZONE=$ZONE - -# Storage Buckets -GCS_DATASETS_BUCKET=$DATASETS_BUCKET -GCS_MODELS_BUCKET=$MODELS_BUCKET -GCS_LOGS_BUCKET=$LOGS_BUCKET - -# API Configuration -API_URL=$SERVICE_URL -USE_GCS=true - -# Service Account -GOOGLE_APPLICATION_CREDENTIALS=~/stt-service-account-key.json - -# Model Configuration -MODEL_NAME=whisper -DEVICE=cpu # Use 'cuda' on GPU VMs -EOF - -# Load environment variables -source .env -``` - -### STEP 9: Initial System Verification - -```bash -# 1. Check API health -curl $SERVICE_URL/api/health | jq - -# 2. Test baseline transcription -curl -X POST $SERVICE_URL/api/transcribe/baseline \ - -F "file=@data/test_audio/test_1.wav" | jq - -# 3. Test agent transcription -curl -X POST "$SERVICE_URL/api/transcribe/agent?auto_correction=true" \ - -F "file=@data/test_audio/test_1.wav" | jq - -# 4. Check system stats -curl $SERVICE_URL/api/system/stats | jq - -# 5. View API documentation -open $SERVICE_URL/docs -``` - -### STEP 10: Setup Monitoring & Alerts - -```bash -# 1. Create notification channel (email) -gcloud alpha monitoring channels create \ - --display-name="STT Alerts" \ - --type=email \ - --channel-labels=email_address=your-email@example.com - -# Get channel ID -export CHANNEL_ID=$(gcloud alpha monitoring channels list --format="value(name)") - -# 2. Create uptime check -gcloud monitoring uptime create http stt-api-uptime \ - --display-name="STT API Uptime Check" \ - --resource-type=uptime-url \ - --host=$SERVICE_URL \ - --path=/api/health \ - --check-interval=5m - -# 3. Create alert policy for errors -cat > alert-policy.json << EOF -{ - "displayName": "STT API High Error Rate", - "conditions": [ - { - "displayName": "Error rate > 5%", - "conditionThreshold": { - "filter": "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"stt-api\" AND metric.type=\"run.googleapis.com/request_count\" AND metric.labels.response_code_class=\"5xx\"", - "comparison": "COMPARISON_GT", - "thresholdValue": 0.05, - "duration": "300s" - } - } - ], - "notificationChannels": ["$CHANNEL_ID"], - "enabled": true -} -EOF - -gcloud alpha monitoring policies create --policy-from-file=alert-policy.json - -# 4. View logs -gcloud logging read "resource.type=cloud_run_revision AND resource.labels.service_name=stt-api" \ - --limit 50 \ - --format json -``` - ---- - -## ๐ŸŽ›๏ธ Deployment Options - -### Option A: Production Deployment (Cloud Run + GCS) - -**Best for**: Production environments, scalable applications, cost-effective for variable traffic - -**Pros**: -- Auto-scaling (0 to N instances) -- Pay per request -- Managed infrastructure -- HTTPS out of the box -- Easy rollbacks - -**Cons**: -- Cold start latency (~2-5s) -- No GPU support -- 300s timeout limit - -**Setup**: Follow STEP 5 Option A above - -**Estimated Cost**: $0 (free tier) to $50/month (moderate traffic) - -### Option B: GPU VM for Training - -**Best for**: Model training, fine-tuning, GPU-intensive workloads - -**Pros**: -- GPU acceleration (3-7x faster) -- Full control over environment -- No timeout limits -- Persistent storage - -**Cons**: -- Higher cost when running -- Manual scaling -- Requires management - -**Setup**: Follow STEP 6 above - -**Estimated Cost**: $0.54/hour (~$400/month if running 24/7) - -**๐Ÿ’ก Cost Tip**: Stop VM when not training! - -```bash -# Start VM for training -gcloud compute instances start stt-training-vm --zone=$ZONE - -# Train models -# ... - -# Stop VM when done -gcloud compute instances stop stt-training-vm --zone=$ZONE -``` - -### Option C: App Engine Deployment - -**Best for**: Simple deployment, no containerization needed - -**Pros**: -- Easy deployment (`gcloud app deploy`) -- Auto-scaling -- Integrated with GCP services - -**Cons**: -- Less flexible than Cloud Run -- Higher minimum cost -- Longer deployment times - -**Setup**: - -```bash -# 1. Create app.yaml -cat > app.yaml << EOF -runtime: python39 -instance_class: F4 -automatic_scaling: - min_instances: 0 - max_instances: 10 - target_cpu_utilization: 0.65 - -entrypoint: uvicorn src.control_panel_api:app --host 0.0.0.0 --port \$PORT - -env_variables: - USE_GCS: "true" - GCS_DATASETS_BUCKET: "$DATASETS_BUCKET" - GCS_MODELS_BUCKET: "$MODELS_BUCKET" - GCS_LOGS_BUCKET: "$LOGS_BUCKET" -EOF - -# 2. Deploy -gcloud app deploy - -# 3. Open application -gcloud app browse -``` - -### Option D: Kubernetes Deployment (Advanced) - -**Best for**: Complex deployments, microservices, advanced orchestration - -See `docs/KUBERNETES_DEPLOYMENT.md` for detailed instructions. - ---- - -## ๐Ÿ’ฐ Cost Optimization - -### Estimated Monthly Costs - -| Component | Configuration | Monthly Cost | -|-----------|--------------|--------------| -| Cloud Run API | Light traffic (<100k requests) | $0-10 | -| Cloud Run API | Medium traffic (1M requests) | $50-100 | -| Cloud Storage | 100GB storage | $2-3 | -| GPU VM (T4) | Running 24/7 | ~$400 | -| GPU VM (T4) | 8 hours/day | ~$130 | -| Monitoring | Standard | Free-$5 | -| **Total (Development)** | Cloud Run + Storage | **$5-15/month** | -| **Total (Production)** | All services | **$150-500/month** | - -### Cost-Saving Strategies - -#### 1. Use Preemptible VMs for Training - -```bash -# Create preemptible VM (60-80% cheaper) -gcloud compute instances create stt-training-vm-preemptible \ - --zone=$ZONE \ - --machine-type=n1-standard-8 \ - --accelerator=type=nvidia-tesla-t4,count=1 \ - --preemptible \ - --image-family=pytorch-latest-gpu \ - --image-project=deeplearning-platform-release \ - --boot-disk-size=200GB - -# Note: Preemptible VMs can be terminated anytime -# Use checkpointing in your training code! -``` - -#### 2. Set Budget Alerts - -```bash -# Create budget -gcloud billing budgets create \ - --billing-account=YOUR-BILLING-ACCOUNT-ID \ - --display-name="STT Project Budget" \ - --budget-amount=100USD \ - --threshold-rule=percent=50 \ - --threshold-rule=percent=90 \ - --threshold-rule=percent=100 -``` - -#### 3. Lifecycle Policies for Storage - -```bash -# Delete old training logs after 30 days -cat > lifecycle-logs.json << EOF -{ - "lifecycle": { - "rule": [ - { - "action": {"type": "Delete"}, - "condition": {"age": 30} - } - ] - } -} -EOF - -gsutil lifecycle set lifecycle-logs.json gs://$LOGS_BUCKET -``` - -#### 4. Cloud Run Optimization - -```bash -# Deploy with minimum instances = 0 (cold starts but cheaper) -gcloud run services update stt-api \ - --region=$REGION \ - --min-instances=0 \ - --max-instances=5 - -# For production with consistent traffic, use min-instances=1 -gcloud run services update stt-api \ - --region=$REGION \ - --min-instances=1 \ - --max-instances=10 -``` - -#### 5. Monitor Costs Continuously - -```bash -# Use provided monitoring script -python scripts/monitor_gcp_costs.py - -# Or check in console -open https://console.cloud.google.com/billing -``` - ---- - -## ๐Ÿ“Š Monitoring & Maintenance - -### Daily Monitoring - -```bash -# 1. Check service health -curl $SERVICE_URL/api/health - -# 2. View recent logs -gcloud logging read "resource.type=cloud_run_revision" \ - --limit 20 \ - --format="table(timestamp,jsonPayload.message)" - -# 3. Check costs -python scripts/monitor_gcp_costs.py - -# 4. View system stats -curl $SERVICE_URL/api/system/stats | jq '.data_management' -``` - -### Weekly Tasks - -```bash -# 1. Review failed cases -curl $SERVICE_URL/api/data/failed-cases?limit=100 | jq - -# 2. Check if ready for fine-tuning -curl $SERVICE_URL/api/data/statistics | jq - -# 3. Generate performance report -curl $SERVICE_URL/api/data/report -o weekly-report.json - -# 4. Review storage usage -gsutil du -sh gs://$DATASETS_BUCKET -gsutil du -sh gs://$MODELS_BUCKET -gsutil du -sh gs://$LOGS_BUCKET -``` - -### Monthly Maintenance - -```bash -# 1. Update dependencies -pip install --upgrade -r requirements.txt - -# 2. Rebuild and redeploy -gcloud builds submit --tag gcr.io/$PROJECT_ID/stt-api -gcloud run deploy stt-api --image gcr.io/$PROJECT_ID/stt-api --region=$REGION - -# 3. Backup critical data -gsutil -m cp -r gs://$MODELS_BUCKET gs://${PROJECT_ID}-backup/models-$(date +%Y%m%d) - -# 4. Clean up old artifacts -gcloud container images list-tags gcr.io/$PROJECT_ID/stt-api \ - --filter='-tags:*' --format='get(digest)' | \ - xargs -I {} gcloud container images delete gcr.io/$PROJECT_ID/stt-api@{} --quiet - -# 5. Review and optimize costs -python scripts/monitor_gcp_costs.py -``` - -### Performance Monitoring - -```bash -# 1. View Cloud Run metrics -gcloud monitoring dashboards list - -# 2. Check request latency -gcloud logging read "resource.type=cloud_run_revision AND jsonPayload.latency>1000" \ - --limit 50 - -# 3. Monitor error rates -gcloud logging read "resource.type=cloud_run_revision AND severity=ERROR" \ - --limit 50 -``` - ---- - -## ๐Ÿ”ง Troubleshooting - -### Issue 1: Cloud Run Deployment Fails - -**Symptoms**: Build fails or service doesn't deploy - -**Solutions**: - -```bash -# Check build logs -gcloud builds list --limit=5 -gcloud builds log BUILD_ID - -# Verify Docker image builds locally -docker build -t test-stt-api . -docker run -p 8000:8000 test-stt-api - -# Check Cloud Run logs -gcloud run services describe stt-api --region=$REGION -gcloud logging read "resource.type=cloud_run_revision" --limit=50 -``` - -### Issue 2: GPU VM Not Detecting GPU - -**Symptoms**: `torch.cuda.is_available()` returns `False` - -**Solutions**: - -```bash -# SSH into VM -gcloud compute ssh stt-training-vm --zone=$ZONE - -# Check NVIDIA drivers -nvidia-smi - -# If not found, install drivers -sudo /opt/deeplearning/install-driver.sh - -# Reinstall PyTorch with CUDA -pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 - -# Reboot if needed -sudo reboot -``` - -### Issue 3: Permission Denied on GCS - -**Symptoms**: `403 Forbidden` when accessing buckets - -**Solutions**: - -```bash -# Check service account permissions -gcloud projects get-iam-policy $PROJECT_ID \ - --flatten="bindings[].members" \ - --filter="bindings.members:serviceAccount:stt-service-account@${PROJECT_ID}.iam.gserviceaccount.com" - -# Add missing permissions -gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member="serviceAccount:stt-service-account@${PROJECT_ID}.iam.gserviceaccount.com" \ - --role="roles/storage.objectAdmin" - -# Verify bucket IAM -gsutil iam get gs://$DATASETS_BUCKET -``` - -### Issue 4: High Costs - -**Symptoms**: Unexpected high billing - -**Solutions**: - -```bash -# Check running instances -gcloud compute instances list -gcloud run services list - -# Stop unused VMs -gcloud compute instances stop INSTANCE_NAME --zone=$ZONE - -# Check storage usage -gsutil du -sh gs://$DATASETS_BUCKET - -# Delete old data -gsutil -m rm -r gs://$LOGS_BUCKET/old-logs/* - -# Review detailed costs -open https://console.cloud.google.com/billing -``` - -### Issue 5: API Timeout Errors - -**Symptoms**: 504 Gateway Timeout on long transcriptions - -**Solutions**: - -```bash -# Increase Cloud Run timeout (max 3600s for 2nd gen) -gcloud run services update stt-api \ - --region=$REGION \ - --timeout=900 \ - --execution-environment=gen2 - -# Add request-timeout header -curl -X POST $SERVICE_URL/api/transcribe/agent \ - -H "X-Cloud-Trace-Context: TRACE_ID" \ - -F "file=@audio.wav" \ - --max-time 300 - -# For very long files, use GPU VM instead -``` - -### Issue 6: Out of Memory Errors - -**Symptoms**: Container crashes with OOM error - -**Solutions**: - -```bash -# Increase Cloud Run memory -gcloud run services update stt-api \ - --region=$REGION \ - --memory=8Gi \ - --cpu=4 - -# Check memory usage in logs -gcloud logging read "resource.type=cloud_run_revision AND jsonPayload.message:memory" \ - --limit=50 -``` - ---- - -## ๐Ÿ”’ Security Best Practices - -### 1. Service Account Security - -```bash -# Use separate service accounts for different components -gcloud iam service-accounts create stt-api-sa --display-name="API Service Account" -gcloud iam service-accounts create stt-training-sa --display-name="Training Service Account" - -# Grant minimal required permissions -gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member="serviceAccount:stt-api-sa@${PROJECT_ID}.iam.gserviceaccount.com" \ - --role="roles/storage.objectViewer" # Read-only for API - -# Rotate keys regularly -gcloud iam service-accounts keys list \ - --iam-account=stt-service-account@${PROJECT_ID}.iam.gserviceaccount.com -``` - -### 2. API Authentication - -```bash -# For production, require authentication -gcloud run services update stt-api \ - --region=$REGION \ - --no-allow-unauthenticated - -# Access with token -gcloud auth print-identity-token > token.txt -curl -H "Authorization: Bearer $(cat token.txt)" $SERVICE_URL/api/health -``` - -### 3. Network Security - -```bash -# Use VPC for VM isolation -gcloud compute networks create stt-network --subnet-mode=custom - -gcloud compute networks subnets create stt-subnet \ - --network=stt-network \ - --region=$REGION \ - --range=10.0.0.0/24 - -# Create firewall rules -gcloud compute firewall-rules create allow-ssh \ - --network=stt-network \ - --allow=tcp:22 \ - --source-ranges=YOUR_IP/32 -``` - -### 4. Data Encryption - -```bash -# Use customer-managed encryption keys (CMEK) -gcloud kms keyrings create stt-keyring --location=$REGION - -gcloud kms keys create stt-key \ - --location=$REGION \ - --keyring=stt-keyring \ - --purpose=encryption - -# Apply to bucket -gsutil kms encryption gs://$DATASETS_BUCKET \ - projects/$PROJECT_ID/locations/$REGION/keyRings/stt-keyring/cryptoKeys/stt-key -``` - -### 5. Audit Logging - -```bash -# Enable Cloud Audit Logs -gcloud logging sinks create stt-audit-sink \ - gs://${PROJECT_ID}-audit-logs \ - --log-filter='protoPayload.serviceName="storage.googleapis.com"' -``` - ---- - -## ๐Ÿ“ Post-Deployment Checklist - -- [ ] All APIs enabled -- [ ] Service account created with proper permissions -- [ ] GCS buckets created and configured -- [ ] Cloud Run service deployed and healthy -- [ ] GPU VM created (if needed) and tested -- [ ] Frontend accessible -- [ ] Environment variables configured -- [ ] Monitoring and alerts setup -- [ ] Budget alerts configured -- [ ] Cost monitoring script tested -- [ ] API authentication configured (if production) -- [ ] Backup strategy implemented -- [ ] Documentation updated with actual URLs and IDs -- [ ] Team trained on deployment process - ---- - -## ๐ŸŽ“ Next Steps - -1. **Fine-tune your first model**: Follow `docs/FINETUNING_QUICK_START.md` -2. **Integrate W&B**: Follow `docs/WANDB_SWEEPS_GUIDE.md` -3. **Setup CI/CD**: Automate deployments with Cloud Build triggers -4. **Scale up**: Configure auto-scaling based on traffic -5. **Optimize costs**: Review and implement cost-saving strategies - ---- - -## ๐Ÿ“ž Support & Resources - -- **GCP Documentation**: https://cloud.google.com/docs -- **Cloud Run Docs**: https://cloud.google.com/run/docs -- **Cost Calculator**: https://cloud.google.com/products/calculator -- **Project Documentation**: See `docs/` directory -- **API Reference**: `http://YOUR-SERVICE-URL/docs` - ---- - -**Last Updated**: December 2024 -**Version**: 1.0 -**Tested on**: Google Cloud Platform - diff --git a/docs/GCP_SETUP_GUIDE.md b/docs/GCP_SETUP_GUIDE.md deleted file mode 100644 index ce528d7..0000000 --- a/docs/GCP_SETUP_GUIDE.md +++ /dev/null @@ -1,232 +0,0 @@ -# GCP GPU Setup Guide - -Complete guide for setting up and using Google Cloud Platform GPU resources for the STT project. - -## ๐ŸŽฏ Overview - -This guide helps you: -- Create GPU-enabled VMs on GCP -- Deploy and run your evaluation framework on GPU -- Monitor costs and optimize spending -- Leverage GCP credits effectively - -## ๐Ÿ“‹ Prerequisites - -1. **GCP Account** with credits -2. **gcloud CLI** installed: [Install Guide](https://cloud.google.com/sdk/install) -3. **Project ID**: `stt-agentic-ai-2025` (or update in scripts) - -## ๐Ÿš€ Quick Start - -### Step 1: Create GPU VM - -```bash -# Make script executable -chmod +x scripts/setup_gcp_gpu.sh - -# Run setup script -bash scripts/setup_gcp_gpu.sh -``` - -This will: -- Create a VM with NVIDIA T4 GPU -- Install PyTorch + CUDA -- Configure GPU drivers -- Set up project access - -**Estimated Cost**: ~$0.54/hour (~$12.96/day if running 24/7) - -### Step 2: Deploy Code to VM - -```bash -# Deploy code and run evaluation -python scripts/deploy_to_gcp.py -``` - -This will: -- Upload your code to the VM -- Install dependencies -- Verify GPU access -- Run evaluation framework -- Download results - -### Step 3: Monitor Costs - -```bash -# Check VM status and costs -python scripts/monitor_gcp_costs.py -``` - -## ๐Ÿ’ป Manual Setup (Alternative) - -If you prefer manual setup: - -### 1. Create VM Manually - -```bash -gcloud compute instances create stt-gpu-vm \ - --zone=us-central1-a \ - --machine-type=n1-standard-4 \ - --accelerator=type=nvidia-tesla-t4,count=1 \ - --image-family=pytorch-latest-gpu \ - --image-project=deeplearning-platform-release \ - --boot-disk-size=100GB \ - --maintenance-policy=TERMINATE \ - --scopes=https://www.googleapis.com/auth/cloud-platform -``` - -### 2. SSH into VM - -```bash -gcloud compute ssh stt-gpu-vm --zone=us-central1-a -``` - -### 3. On VM: Clone and Setup - -```bash -# Clone your repository -git clone -cd Adaptive-Self-Learning-Agentic-AI-System - -# Install dependencies -pip install -r requirements.txt - -# Verify GPU -python -c "import torch; print(f'CUDA: {torch.cuda.is_available()}')" -``` - -### 4. Run Evaluation - -```bash -# Run evaluation framework (will automatically use GPU) -python experiments/kavya_evaluation_framework.py -``` - -## ๐Ÿ“Š GPU vs CPU Performance - -Based on your benchmarks: - -| Metric | CPU | GPU (T4) | Speedup | -|--------|-----|----------|---------| -| Latency (per sample) | 0.72s | ~0.1-0.2s | **3-7x faster** | -| Throughput | 2.97 samples/s | ~10-20 samples/s | **3-7x faster** | -| 100 samples | 72s | 10-20s | **3-7x faster** | - -## ๐Ÿ’ฐ Cost Management - -### VM Costs - -- **T4 GPU + n1-standard-4**: ~$0.54/hour -- **Per day (24h)**: ~$12.96 -- **Per month (730h)**: ~$394.20 - -### Cost-Saving Tips - -1. **Stop VMs when not in use**: - ```bash - gcloud compute instances stop stt-gpu-vm --zone=us-central1-a - gcloud compute instances start stt-gpu-vm --zone=us-central1-a - ``` - -2. **Use Preemptible Instances** (60-80% cheaper): - ```bash - # Add --preemptible flag when creating VM - ``` - -3. **Use Smaller GPUs for Development**: - - T4: Good for development (~$0.35/hour GPU) - - V100: For training (~$2.50/hour GPU) - - A100: For heavy training (~$3.00/hour GPU) - -4. **Set Billing Alerts**: - - Go to GCP Console โ†’ Billing โ†’ Budgets & Alerts - - Set up alerts at 50%, 90%, 100% of budget - -### Storage Costs - -- **GCS Storage**: ~$0.02/GB/month -- **Current buckets**: - - `stt-project-datasets`: Store datasets - - `stt-project-models`: Store model checkpoints - - `stt-project-logs`: Store training logs - -## ๐Ÿ”ง GPU Optimization - -The code automatically optimizes for GPU when available: - -- **TensorFloat-32 (TF32)**: Enabled for Ampere+ GPUs -- **Beam Search**: Better quality with GPU -- **KV Cache**: Faster generation on GPU -- **Half Precision**: Can be enabled for 2x speedup (with minor quality loss) - -## ๐Ÿ“ File Structure - -``` -scripts/ -โ”œโ”€โ”€ setup_gcp_gpu.sh # Create GPU VM -โ”œโ”€โ”€ deploy_to_gcp.py # Deploy and run on VM -โ””โ”€โ”€ monitor_gcp_costs.py # Monitor costs - -docs/ -โ””โ”€โ”€ GCP_SETUP_GUIDE.md # This guide -``` - -## ๐Ÿ› Troubleshooting - -### GPU Not Detected - -```bash -# On VM, check NVIDIA drivers -nvidia-smi - -# If not available, install drivers -sudo /opt/deeplearning/install-driver.sh -``` - -### Out of Memory - -- Use smaller batch sizes -- Use gradient checkpointing -- Use smaller models (whisper-tiny instead of whisper-base) - -### VM Won't Start - -- Check GPU quota: `gcloud compute project-info describe` -- Request quota increase if needed -- Try different zone - -### High Costs - -- Stop VM immediately: `gcloud compute instances stop stt-gpu-vm` -- Check running VMs: `gcloud compute instances list` -- Review billing: https://console.cloud.google.com/billing - -## ๐Ÿ“š Additional Resources - -- [GCP GPU Documentation](https://cloud.google.com/compute/docs/gpus) -- [PyTorch on GCP](https://cloud.google.com/ai-platform/training/docs/getting-started-pytorch) -- [Cost Calculator](https://cloud.google.com/products/calculator) -- [Preemptible VMs](https://cloud.google.com/compute/docs/instances/preemptible) - -## โœ… Checklist - -- [ ] GCP account with credits -- [ ] gcloud CLI installed and authenticated -- [ ] GPU VM created -- [ ] Code deployed to VM -- [ ] GPU verified working -- [ ] Evaluation framework runs on GPU -- [ ] Billing alerts configured -- [ ] VM stopped when not in use - -## ๐ŸŽ“ Next Steps - -1. **Run full evaluation on GPU** - See 3-7x speedup -2. **Train/fine-tune models** - Use GPU for training -3. **Scale up evaluation** - Run on larger datasets -4. **Deploy API** - Use Cloud Run for inference API - ---- - -**Need Help?** Check the scripts or run with `--help` flag for options. - diff --git a/docs/LLAMA_INTEGRATION_SUMMARY.md b/docs/LLAMA_INTEGRATION_SUMMARY.md deleted file mode 100644 index 53349c7..0000000 --- a/docs/LLAMA_INTEGRATION_SUMMARY.md +++ /dev/null @@ -1,254 +0,0 @@ -# Gemma LLM Integration Summary - -## โœ… Integration Complete - -Gemma LLM has been successfully integrated into the agent system for intelligent error correction. - ---- - -## ๐Ÿ“ Files Modified/Created - -### New Files Created: -1. **`src/agent/llm_corrector.py`** โญ NEW - - Gemma LLM-based error corrector - - Intelligent transcript correction using Google's Gemma model - - Supports quantization for memory efficiency - - Falls back gracefully if LLM unavailable - -### Files Modified: -1. **`src/agent/agent.py`** - - Added Gemma LLM integration - - Uses LLM for intelligent correction when available - - Falls back to rule-based correction if LLM unavailable - - Added `use_llm_correction`, `llm_model_name`, `use_quantization` parameters - -2. **`src/agent/__init__.py`** - - Exported `LlamaLLMCorrector` class - -3. **`src/agent_api.py`** - - Updated to initialize agent with LLM support - - Added LLM status to startup logs - - Added LLM status to health endpoint - -4. **`requirements.txt`** - - Added `bitsandbytes>=0.41.0` for optional quantization support - ---- - -## ๐ŸŽฏ How It Works - -### Architecture Flow: -``` -STT Transcription โ†’ Error Detection โ†’ Gemma LLM Correction โ†’ Final Transcript - โ†“ (if unavailable) - Rule-based Correction -``` - -### Integration Points: - -1. **Agent Initialization** (`src/agent/agent.py`): - ```python - agent = STTAgent( - baseline_model=baseline_model, - use_llm_correction=True, # Enable Gemma LLM - llm_model_name="google/gemma-2b-it", # Default model - use_quantization=False # Set True to save memory - ) - ``` - -2. **Error Correction Process**: - - When errors are detected, the agent first tries LLM-based correction - - Gemma receives the transcript + error context - - LLM generates an improved/corrected version - - Falls back to rule-based correction if LLM fails or unavailable - -3. **LLM Corrector** (`src/agent/llm_corrector.py`): - - Builds intelligent prompts with error context - - Uses Gemma to generate corrections - - Handles errors gracefully with fallback - ---- - -## ๐Ÿš€ Usage - -### Basic Usage (with LLM): -```python -from src.baseline_model import BaselineSTTModel -from src.agent import STTAgent - -# Initialize with LLM support -baseline_model = BaselineSTTModel(model_name="whisper") -agent = STTAgent( - baseline_model=baseline_model, - use_llm_correction=True # Enable Gemma LLM -) - -# Transcribe with intelligent LLM correction -result = agent.transcribe_with_agent("audio.wav", enable_auto_correction=True) - -print(f"Original: {result['original_transcript']}") -print(f"Corrected: {result['transcript']}") -print(f"Method: {result['agent_metadata']['correction_method']}") -print(f"LLM Available: {result['agent_metadata']['llm_available']}") -``` - -### API Usage: -```bash -# Start API (LLM enabled by default) -uvicorn src.agent_api:app --reload --port 8000 - -# Transcribe with LLM correction -curl -X POST "http://localhost:8000/agent/transcribe?auto_correction=true" \ - -F "file=@audio.wav" - -# Check LLM status -curl "http://localhost:8000/agent/stats" -``` - ---- - -## ๐Ÿ“Š Features - -### LLM-Based Correction: -- โœ… Intelligent error correction using Gemma 2B model -- โœ… Context-aware corrections (uses error detection results) -- โœ… Natural language understanding for better fixes -- โœ… Handles complex error patterns beyond rule-based heuristics -- โœ… Graceful fallback to rule-based correction if LLM unavailable - -### Memory Optimization: -- โœ… Optional 8-bit quantization support (requires `bitsandbytes`) -- โœ… Automatic device selection (CUDA/CPU) -- โœ… Efficient model loading - -### Error Handling: -- โœ… Graceful degradation if LLM fails to load -- โœ… Fallback to rule-based correction -- โœ… Comprehensive logging - ---- - -## ๐Ÿ”ง Configuration - -### Model Selection: -- **Default**: `google/gemma-2b-it` (2B parameter instruction-tuned model) -- **Alternative**: Can use `google/gemma-7b-it` for better quality (requires more memory) - -### Quantization (Memory Saving): -```python -agent = STTAgent( - baseline_model=baseline_model, - use_llm_correction=True, - use_quantization=True # Reduces memory usage by ~50% -) -``` - -**Note**: Requires `bitsandbytes` package and CUDA GPU - -### Disable LLM (Use Rule-Based Only): -```python -agent = STTAgent( - baseline_model=baseline_model, - use_llm_correction=False # Use rule-based correction only -) -``` - ---- - -## ๐Ÿ“ˆ Performance Considerations - -### LLM Correction: -- **Time Overhead**: ~1-3 seconds per correction (depending on GPU) -- **Memory**: ~4-8GB GPU memory (2B model), ~2-4GB with quantization -- **Quality**: Significantly better than rule-based for complex errors - -### Fallback Behavior: -- If LLM unavailable โ†’ Uses rule-based correction (no errors) -- If LLM fails โ†’ Falls back to rule-based correction (logged) - ---- - -## โœ… Week 2 Tasks Verification - -### All Week 2 Tasks Complete: - -- [x] **Error Detection Module** โœ… - - [x] Multi-heuristic error detection - - [x] Confidence scoring - - [x] Error summarization - -- [x] **Lightweight Self-Learning Component** โœ… - - [x] In-memory error pattern tracking - - [x] Correction history (in-memory) - - [x] User feedback collection (in-memory) - - [x] Learning statistics - - [x] Data export interface for external persistence - -- [x] **Agent Integration** โœ… - - [x] Agent wrapper class - - [x] Integration with baseline model - - [x] Automatic correction (rule-based + LLM) - - [x] Feedback interface - -- [x] **API Endpoints** โœ… - - [x] Agent transcription endpoint - - [x] Feedback endpoint - - [x] Statistics endpoint - - [x] Learning report endpoint - -- [x] **Testing** โœ… - - [x] Agent testing script - - [x] Component validation - - [x] Integration testing - -### **BONUS: LLM Integration** โœ… -- [x] Gemma LLM corrector module -- [x] LLM integration into agent -- [x] API support for LLM correction -- [x] Graceful fallback handling - ---- - -## ๐Ÿ“ฆ Files Summary - -### Core Implementation (5 files - was 4, now includes LLM): -1. `src/agent/__init__.py` - Module initialization -2. `src/agent/error_detector.py` - Error detection module -3. `src/agent/self_learner.py` - Self-learning component -4. `src/agent/agent.py` - Main agent class (now with LLM support) -5. `src/agent/llm_corrector.py` - **NEW** Gemma LLM corrector - -### API (1 file): -6. `src/agent_api.py` - Agent API endpoints (updated with LLM support) - -### Testing (1 file): -7. `experiments/test_agent.py` - Agent testing script - -### Documentation (2 files): -8. Documentation files are in the `docs/` directory - -**Total: 9 files** (was 7, added 2 new files) - ---- - -## ๐Ÿ”ฎ Next Steps / Future Enhancements - -1. **Fine-tuning**: Fine-tune Gemma on transcription correction tasks -2. **Larger Models**: Support for Gemma 7B for better quality -3. **Caching**: Cache LLM corrections for similar errors -4. **Batch Processing**: Batch corrections for efficiency -5. **Custom Prompts**: Allow custom prompt templates - ---- - -## ๐Ÿ“ Notes - -- Gemma model will be downloaded from HuggingFace on first use (~4GB for 2B model) -- Requires internet connection for initial model download -- GPU recommended for best performance (CPU works but slower) -- LLM integration is optional - system works without it using rule-based correction - ---- - -**Status**: โœ… Gemma LLM integration complete! All Week 2 tasks verified and complete. - diff --git a/docs/LLM_INTEGRATION.md b/docs/LLM_INTEGRATION.md new file mode 100644 index 0000000..796eda9 --- /dev/null +++ b/docs/LLM_INTEGRATION.md @@ -0,0 +1,34 @@ +# LLM Integration (Gemma) + +Gemma LLM integration for intelligent transcript error correction. + +## Usage + +```python +agent = STTAgent( + baseline_model=baseline_model, + use_llm_correction=True, + llm_model_name="google/gemma-2b-it", + use_quantization=False # True to save memory +) +result = agent.transcribe_with_agent("audio.wav", enable_auto_correction=True) +``` + +## Architecture + +``` +STT Transcription โ†’ Error Detection โ†’ Gemma LLM Correction โ†’ Final Transcript + โ†“ (if unavailable) + Rule-based Correction +``` + +## Configuration + +- **Default model**: `google/gemma-2b-it` +- **Quantization**: `use_quantization=True` reduces memory ~50% (requires bitsandbytes + CUDA) +- **Disable LLM**: `use_llm_correction=False` for rule-based only + +## Files + +- `src/agent/llm_corrector.py` - LLM corrector +- `src/agent/agent.py` - Integration diff --git a/docs/QUICK_REFERENCE.md b/docs/QUICK_REFERENCE.md new file mode 100644 index 0000000..e2a7c19 --- /dev/null +++ b/docs/QUICK_REFERENCE.md @@ -0,0 +1,99 @@ +# Quick Reference + +## Installation + +```bash +git clone +cd Adaptive-Self-Learning-Agentic-AI-System +python -m venv venv +source venv/bin/activate # Windows: venv\Scripts\activate +pip install -r requirements.txt +python scripts/verify_setup.py +``` + +## Running the System + +```bash +# Agent API (recommended) +uvicorn src.agent_api:app --reload --port 8000 + +# Baseline API +uvicorn src.inference_api:app --reload --port 8000 + +# Control Panel +./start_control_panel.sh +``` + +## API Endpoints + +```bash +curl -X POST "http://localhost:8000/transcribe" -F "file=@audio.wav" +curl -X POST "http://localhost:8000/agent/transcribe?auto_correction=true" -F "file=@audio.wav" +curl "http://localhost:8000/agent/stats" +``` + +## Testing + +```bash +python experiments/test_baseline.py +python experiments/test_agent.py +python experiments/test_data_management.py +python experiments/kavya_evaluation_framework.py +pytest tests/ +``` + +## Data Management + +```python +from src.data.integration import IntegratedDataManagementSystem + +system = IntegratedDataManagementSystem() +case_id = system.record_failed_transcription(...) +system.add_correction(case_id, "Corrected text") +dataset_info = system.prepare_finetuning_dataset(max_samples=1000, create_version=True) +stats = system.get_system_statistics() +``` + +## GCP + +```bash +gcloud auth login +gcloud config set project your-project-id +bash scripts/setup_gcp_gpu.sh +python scripts/deploy_to_gcp.py +python scripts/monitor_gcp_costs.py +``` + +## Adaptive Scheduling (Week 3) + +```python +agent = STTAgent( + baseline_model=baseline_model, + enable_adaptive_fine_tuning=True, + scheduler_history_path="data/processed/scheduler_history.json" +) +stats = agent.get_adaptive_scheduler_stats() +``` + +## Integration & Testing (Week 4) + +```python +from src.integration.unified_system import UnifiedSTTSystem +from src.integration.end_to_end_testing import EndToEndTester + +system = UnifiedSTTSystem( + model_name="whisper", + enable_error_detection=True, + enable_llm_correction=True, + enable_adaptive_fine_tuning=True +) +tester = EndToEndTester(system) +results = tester.run_full_test_suite(audio_files, reference_transcripts) +``` + +## Troubleshooting + +```bash +python scripts/verify_setup.py +kill -9 $(lsof -ti:8000) +``` diff --git a/docs/QUICK_START_DATA_MANAGEMENT.md b/docs/QUICK_START_DATA_MANAGEMENT.md deleted file mode 100644 index fc865a7..0000000 --- a/docs/QUICK_START_DATA_MANAGEMENT.md +++ /dev/null @@ -1,309 +0,0 @@ -# Quick Start Guide: Data Management System - -## 5-Minute Setup - -### 1. Installation - -Ensure you have the required dependencies: - -```bash -cd /path/to/Adaptive-Self-Learning-Agentic-AI-System -pip install -r requirements.txt -``` - -### 2. Basic Usage - -```python -from src.data.integration import IntegratedDataManagementSystem - -# Initialize (local-only for quick start) -system = IntegratedDataManagementSystem( - base_dir="data/quickstart", - use_gcs=False # Set to True for GCS integration -) - -# Record a failed transcription -case_id = system.record_failed_transcription( - audio_path="audio/sample.wav", - original_transcript="THIS IS ALL CAPS", - corrected_transcript="This is proper text", - error_types=["all_caps"], - error_score=0.8, - inference_time=0.5 -) - -print(f"Recorded case: {case_id}") - -# Get statistics -stats = system.get_system_statistics() -print(f"Total cases: {stats['data_management']['total_failed_cases']}") -``` - -### 3. Run Tests - -```bash -python experiments/test_data_management.py -``` - -## Common Workflows - -### Workflow 1: Collect Failed Cases During Production - -```python -from src.data.integration import IntegratedDataManagementSystem -from src.agent.agent import STTAgent -from src.baseline_model import BaselineSTTModel - -# Initialize -system = IntegratedDataManagementSystem(base_dir="data/production") -baseline_model = BaselineSTTModel() -agent = STTAgent(baseline_model) - -# Process audio -result = agent.transcribe_with_agent("audio/sample.wav") - -# If errors detected, record the case -if result['error_detection']['has_errors']: - case_id = system.record_failed_transcription( - audio_path="audio/sample.wav", - original_transcript=result['original_transcript'], - corrected_transcript=None, # Will add later - error_types=list(result['error_detection']['error_types'].keys()), - error_score=result['error_detection']['error_score'], - inference_time=result['inference_time_seconds'] - ) -``` - -### Workflow 2: Add User Corrections - -```python -# Get uncorrected cases -uncorrected = system.data_manager.get_uncorrected_cases() - -# Add corrections (from user feedback) -for case in uncorrected: - # User provides corrected text - corrected_text = get_user_correction(case.original_transcript) - - system.add_correction( - case_id=case.case_id, - corrected_transcript=corrected_text, - correction_method='user_feedback' - ) -``` - -### Workflow 3: Prepare Fine-tuning Dataset - -```python -# Prepare dataset when you have enough corrected cases -dataset_info = system.prepare_finetuning_dataset( - min_error_score=0.5, - train_ratio=0.8, - val_ratio=0.1, - test_ratio=0.1, - max_samples=1000, - balance_error_types=True, - create_version=True # Creates a versioned snapshot -) - -print(f"Dataset ready: {dataset_info['dataset_id']}") -print(f"Train: {dataset_info['split_sizes']['train']} samples") -print(f"Val: {dataset_info['split_sizes']['val']} samples") -print(f"Test: {dataset_info['split_sizes']['test']} samples") - -# Get path for training -dataset_path = dataset_info['local_path'] -``` - -### Workflow 4: Track Training Progress - -```python -# After training each model version -system.record_training_performance( - model_version="whisper_base_v1", - wer=0.12, - cer=0.06, - training_metadata={ - 'model_name': 'whisper-base', - 'training_data_size': 1000, - 'epochs': 10, - 'batch_size': 16, - 'learning_rate': 1e-5 - } -) - -# View performance trends -wer_trend = system.metadata_tracker.get_performance_trend('wer') -print(f"WER improved by {wer_trend['improvement_percent']:.1f}%") -``` - -### Workflow 5: Generate Reports - -```python -# Generate comprehensive report -report = system.generate_comprehensive_report( - output_path="reports/weekly_report.json" -) - -print(f"Data Quality: {report['data_quality']['quality_status']}") -print("\nRecommendations:") -for rec in report['recommendations']: - print(f" - {rec}") -``` - -## Google Cloud Setup (Optional) - -### 1. Install Google Cloud SDK - -```bash -# Download and install -curl https://sdk.cloud.google.com | bash -exec -l $SHELL - -# Authenticate -gcloud auth login -gcloud config set project stt-agentic-ai-2025 -``` - -### 2. Create Buckets - -```bash -gsutil mb gs://stt-project-datasets -``` - -### 3. Enable GCS in Code - -```python -system = IntegratedDataManagementSystem( - base_dir="data/production", - use_gcs=True, # Enable GCS - gcs_bucket_name="stt-project-datasets", - project_id="stt-agentic-ai-2025" -) -``` - -## Monitoring & Maintenance - -### Daily Checks - -```python -# Check system health -stats = system.get_system_statistics() -print(f"New cases today: {stats['data_management']['total_failed_cases']}") -print(f"Correction rate: {stats['data_management']['correction_rate']:.2%}") -``` - -### Weekly Tasks - -```python -# Generate weekly report -report = system.generate_comprehensive_report( - output_path=f"reports/weekly_{datetime.now().strftime('%Y%m%d')}.json" -) - -# Prepare new dataset if needed -if stats['data_management']['corrected_cases'] >= 500: - dataset_info = system.prepare_finetuning_dataset( - max_samples=1000, - create_version=True - ) -``` - -### Monthly Tasks - -```python -# Sync to GCS -system.sync_all_to_gcs() - -# Review version history -versions = system.version_control.list_versions() -print(f"Total versions: {len(versions)}") - -# Generate version report -version_report = system.version_control.generate_version_report() -``` - -## Troubleshooting - -### Issue: No failed cases being recorded - -**Solution:** -```python -# Check if errors are being detected -result = agent.transcribe_with_agent("audio/sample.wav") -print(result['error_detection']) - -# Lower error threshold if needed -agent = STTAgent(baseline_model, error_threshold=0.2) -``` - -### Issue: Dataset preparation fails - -**Solution:** -```python -# Check statistics -stats = system.data_manager.get_statistics() -print(f"Total cases: {stats['total_failed_cases']}") -print(f"Corrected: {stats['corrected_cases']}") - -# Need at least some corrected cases -if stats['corrected_cases'] < 10: - print("Need more corrected cases!") -``` - -### Issue: GCS sync errors - -**Solution:** -```python -# Test GCS connection -try: - system.sync_all_to_gcs() -except Exception as e: - print(f"GCS error: {e}") - # Fall back to local-only - system = IntegratedDataManagementSystem(use_gcs=False) -``` - -## Next Steps - -1. **Explore Examples**: Run `python experiments/example_usage.py` -2. **Read Full Documentation**: See `docs/DATA_MANAGEMENT_SYSTEM.md` -3. **Integrate with Agent**: Add data management to your STT agent workflow -4. **Set Up GCS**: Enable cloud storage for production use -5. **Monitor Performance**: Track metrics and generate regular reports - -## Cheat Sheet - -```python -# Initialize -system = IntegratedDataManagementSystem(base_dir="data", use_gcs=False) - -# Record failure -case_id = system.record_failed_transcription(...) - -# Add correction -system.add_correction(case_id, corrected_text) - -# Prepare dataset -dataset_info = system.prepare_finetuning_dataset(max_samples=1000, create_version=True) - -# Track training -system.record_training_performance(model_version, wer, cer, metadata) - -# Get statistics -stats = system.get_system_statistics() - -# Generate report -report = system.generate_comprehensive_report(output_path="report.json") - -# Sync GCS -system.sync_all_to_gcs() -``` - -## Help & Support - -- **Documentation**: `docs/DATA_MANAGEMENT_SYSTEM.md` -- **Examples**: `experiments/example_usage.py` -- **Tests**: `experiments/test_data_management.py` -- **Issues**: Check logs in `data/*/` directories - diff --git a/docs/SETUP_INSTRUCTIONS.md b/docs/SETUP_INSTRUCTIONS.md index fbebfe5..2d2b314 100644 --- a/docs/SETUP_INSTRUCTIONS.md +++ b/docs/SETUP_INSTRUCTIONS.md @@ -892,10 +892,11 @@ Once setup is complete: ## ๐Ÿ“š Additional Resources -- **[README.md](README.md)** - Project overview and features -- **[docs/DATA_MANAGEMENT_SYSTEM.md](docs/DATA_MANAGEMENT_SYSTEM.md)** - Complete data management guide -- **[docs/QUICK_START_DATA_MANAGEMENT.md](docs/QUICK_START_DATA_MANAGEMENT.md)** - Quick start for data management -- **[docs/GCP_SETUP_GUIDE.md](docs/GCP_SETUP_GUIDE.md)** - Detailed GCP guide +- **[README.md](../README.md)** - Project overview +- **[DATA_MANAGEMENT_SYSTEM.md](DATA_MANAGEMENT_SYSTEM.md)** - Data management +- **[GCP.md](GCP.md)** - GCP setup & deployment +- **[FINETUNING.md](FINETUNING.md)** - Fine-tuning guide +- **[QUICK_REFERENCE.md](QUICK_REFERENCE.md)** - Command reference --- diff --git a/docs/TESTING_GUIDE.md b/docs/TESTING_GUIDE.md deleted file mode 100644 index 032a998..0000000 --- a/docs/TESTING_GUIDE.md +++ /dev/null @@ -1,562 +0,0 @@ -# Testing Guide - Adaptive Self-Learning STT System - -Comprehensive guide to testing the STT system with unit tests, integration tests, and API tests. - -## ๐Ÿ“‹ Table of Contents -- [Test Organization](#test-organization) -- [Running Tests](#running-tests) -- [Test Coverage](#test-coverage) -- [Writing New Tests](#writing-new-tests) -- [Test Types](#test-types) -- [CI/CD Integration](#cicd-integration) - -## ๐Ÿ—‚๏ธ Test Organization - -### Directory Structure - -``` -tests/ -โ”œโ”€โ”€ __init__.py # Test package initialization -โ”œโ”€โ”€ conftest.py # pytest fixtures and configuration -โ”œโ”€โ”€ pytest.ini # pytest settings -โ”œโ”€โ”€ run_all_tests.py # Master test runner -โ”‚ -โ”œโ”€โ”€ test_metrics.py # Unit tests for WER/CER metrics -โ”œโ”€โ”€ test_error_detector.py # Unit tests for error detection -โ”œโ”€โ”€ test_benchmark.py # Unit tests for benchmarking -โ”œโ”€โ”€ test_integration.py # Integration tests -โ””โ”€โ”€ test_api_comprehensive.py # Comprehensive API tests - -experiments/ -โ”œโ”€โ”€ test_baseline.py # Baseline model tests -โ”œโ”€โ”€ test_agent.py # Agent system tests -โ”œโ”€โ”€ test_data_management.py # Data management tests -โ””โ”€โ”€ test_api.py # Basic API tests -``` - -### Test Categories - -1. **Unit Tests** (`tests/test_*.py`) - - Test individual components in isolation - - Fast execution - - No external dependencies - -2. **Integration Tests** (`tests/test_integration.py`) - - Test multiple components working together - - End-to-end workflows - - May require test data - -3. **API Tests** (`tests/test_api_comprehensive.py`) - - Test REST API endpoints - - Requires running server - - Performance and load testing - -4. **Functional Tests** (`experiments/test_*.py`) - - High-level feature testing - - User-facing functionality - - Real-world scenarios - -## ๐Ÿš€ Running Tests - -### Quick Start - -```bash -# Install pytest (if not already installed) -pip install pytest pytest-cov - -# Run all tests -pytest tests/ -v - -# Run specific test file -pytest tests/test_metrics.py -v - -# Run with specific markers -pytest tests/ -v -m unit -pytest tests/ -v -m "not slow" -``` - -### Using the Test Runner - -```bash -# Run all test suites -python tests/run_all_tests.py --suite all - -# Run only unit tests -python tests/run_all_tests.py --suite unit - -# Run integration tests -python tests/run_all_tests.py --suite integration - -# Run API tests (requires server running) -python tests/run_all_tests.py --suite api - -# Quick tests (unit tests only) -python tests/run_all_tests.py --suite quick - -# Save results to JSON -python tests/run_all_tests.py --suite all --save-results -``` - -### Running Legacy Tests - -```bash -# Baseline model tests -python experiments/test_baseline.py - -# Agent system tests -python experiments/test_agent.py - -# Data management tests -python experiments/test_data_management.py - -# API tests (requires server) -uvicorn src.agent_api:app --port 8000 & -python experiments/test_api.py -``` - -### API Testing Workflow - -```bash -# Terminal 1: Start API server -uvicorn src.agent_api:app --reload --port 8000 - -# Terminal 2: Run API tests -pytest tests/test_api_comprehensive.py -v - -# Or use the test runner -python tests/run_all_tests.py --suite api -``` - -## ๐Ÿ“Š Test Coverage - -### Current Coverage - -| Component | Test File | Coverage | -|-----------|-----------|----------| -| Metrics (WER/CER) | test_metrics.py | โœ… 95% | -| Error Detector | test_error_detector.py | โœ… 90% | -| Benchmark | test_benchmark.py | โœ… 85% | -| Baseline Model | test_baseline.py | โœ… 80% | -| Agent System | test_agent.py | โœ… 85% | -| Data Management | test_data_management.py | โœ… 90% | -| API Endpoints | test_api_comprehensive.py | โœ… 90% | -| Integration | test_integration.py | โœ… 85% | - -### Generating Coverage Report - -```bash -# Install coverage tool -pip install pytest-cov - -# Run tests with coverage -pytest tests/ --cov=src --cov-report=html --cov-report=term-missing - -# Open coverage report -open htmlcov/index.html # macOS -xdg-open htmlcov/index.html # Linux -``` - -### Coverage Goals - -- **Unit Tests**: 90%+ coverage -- **Integration Tests**: 80%+ coverage -- **API Tests**: 95%+ endpoint coverage -- **Overall**: 85%+ code coverage - -## โœ๏ธ Writing New Tests - -### Test File Template - -```python -""" -Description of what this test file covers -""" - -import sys -from pathlib import Path -sys.path.insert(0, str(Path(__file__).parent.parent)) - -from src.module_name import ClassName -import pytest - - -class TestClassName: - """Test cases for ClassName""" - - def setup_method(self): - """Setup before each test""" - self.instance = ClassName() - - def teardown_method(self): - """Cleanup after each test""" - pass - - def test_basic_functionality(self): - """Test basic functionality""" - result = self.instance.method() - assert result is not None - - def test_edge_case(self): - """Test edge case""" - with pytest.raises(ValueError): - self.instance.method(invalid_input) - - -def test_standalone_function(): - """Test standalone function""" - result = standalone_function() - assert result == expected_value - - -if __name__ == "__main__": - pytest.main([__file__, "-v"]) -``` - -### Best Practices - -1. **Test Naming** - - Use descriptive names: `test_calculate_wer_with_perfect_match` - - Start with `test_` prefix - - Group related tests in classes - -2. **Assertions** - - Use clear assertion messages - - Test one concept per test - - Use pytest's assertion helpers - -3. **Fixtures** - - Define reusable fixtures in `conftest.py` - - Use appropriate fixture scopes - - Clean up resources in teardown - -4. **Mocking** - - Mock external dependencies - - Use `pytest-mock` for complex mocking - - Mock at appropriate level - -5. **Test Data** - - Use fixtures for test data - - Keep test data small and focused - - Store large test data separately - -### Example: Testing Error Detection - -```python -def test_all_caps_detection(): - """Test detection of all caps text""" - detector = ErrorDetector() - errors = detector.detect_errors("HELLO WORLD") - - assert len(errors) > 0, "Should detect errors in all caps text" - assert any(e.error_type == "all_caps" for e in errors), \ - "Should specifically detect all_caps error" - - summary = detector.get_error_summary(errors) - assert summary['has_errors'] is True - assert summary['error_score'] > 0.0 -``` - -## ๐Ÿงช Test Types - -### 1. Unit Tests - -**Purpose**: Test individual functions/methods in isolation - -**Example**: -```python -def test_calculate_wer_perfect_match(): - """Test WER calculation with perfect match""" - evaluator = STTEvaluator() - wer_score = evaluator.calculate_wer("hello world", "hello world") - assert wer_score == 0.0 -``` - -**Characteristics**: -- Fast execution (< 1 second) -- No external dependencies -- Deterministic results -- High coverage - -### 2. Integration Tests - -**Purpose**: Test multiple components working together - -**Example**: -```python -def test_end_to_end_workflow(): - """Test complete workflow from transcription to data storage""" - # Initialize components - model = BaselineSTTModel() - agent = STTAgent(model) - data_system = IntegratedDataManagementSystem() - - # Transcribe - result = agent.transcribe_with_agent("audio.wav") - - # Record errors - if result['error_detection']['has_errors']: - case_id = data_system.record_failed_transcription(...) - assert case_id is not None -``` - -**Characteristics**: -- Slower execution (seconds to minutes) -- Tests component interactions -- May require test data -- Realistic scenarios - -### 3. API Tests - -**Purpose**: Test REST API endpoints - -**Example**: -```python -def test_agent_transcribe_endpoint(): - """Test /agent/transcribe endpoint""" - with open("data/test_audio/test_1.wav", "rb") as f: - files = {"file": f} - response = requests.post( - "http://localhost:8000/agent/transcribe", - files=files - ) - - assert response.status_code == 200 - data = response.json() - assert 'transcript' in data - assert 'error_detection' in data -``` - -**Characteristics**: -- Requires running server -- Tests HTTP interface -- Includes performance tests -- Validates API contracts - -### 4. Performance Tests - -**Purpose**: Verify performance characteristics - -**Example**: -```python -def test_transcription_latency(): - """Test transcription latency is acceptable""" - start_time = time.time() - result = model.transcribe("audio.wav") - latency = time.time() - start_time - - assert latency < 10.0, "Transcription should complete within 10s" -``` - -**Characteristics**: -- Measure timing -- Check resource usage -- Verify scalability -- Set performance baselines - -## ๐Ÿ”„ CI/CD Integration - -### GitHub Actions Example - -```yaml -name: Tests - -on: [push, pull_request] - -jobs: - test: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Install dependencies - run: | - pip install -r requirements.txt - pip install pytest pytest-cov - - - name: Run unit tests - run: | - pytest tests/ -v -m "not api" --cov=src - - - name: Upload coverage - uses: codecov/codecov-action@v2 -``` - -### Pre-commit Hook - -```bash -# .git/hooks/pre-commit -#!/bin/bash - -echo "Running tests before commit..." -pytest tests/ -v -m "not api and not slow" - -if [ $? -ne 0 ]; then - echo "Tests failed. Commit aborted." - exit 1 -fi - -echo "All tests passed!" -``` - -## ๐Ÿ“ Test Markers - -Use pytest markers to categorize tests: - -```python -@pytest.mark.unit -def test_unit_function(): - """Unit test""" - pass - -@pytest.mark.integration -def test_integration_workflow(): - """Integration test""" - pass - -@pytest.mark.api -def test_api_endpoint(): - """API test""" - pass - -@pytest.mark.slow -def test_slow_operation(): - """Slow test""" - pass -``` - -Run specific markers: -```bash -# Run only unit tests -pytest -m unit - -# Run everything except slow tests -pytest -m "not slow" - -# Run integration and API tests -pytest -m "integration or api" -``` - -## ๐Ÿ› Debugging Tests - -### Running Single Test - -```bash -# Run specific test function -pytest tests/test_metrics.py::test_perfect_match -v - -# Run specific test class -pytest tests/test_metrics.py::TestSTTEvaluator -v - -# Run specific test method -pytest tests/test_metrics.py::TestSTTEvaluator::test_perfect_match -v -``` - -### Verbose Output - -```bash -# Show print statements -pytest tests/ -v -s - -# Show full traceback -pytest tests/ -v --tb=long - -# Stop at first failure -pytest tests/ -v -x -``` - -### Debugging with pdb - -```python -def test_debug_example(): - """Test with debugging""" - import pdb; pdb.set_trace() - result = function_to_test() - assert result == expected -``` - -Run with: -```bash -pytest tests/test_file.py --pdb -``` - -## ๐Ÿ“ˆ Test Metrics - -### Key Metrics to Track - -1. **Code Coverage**: Percentage of code executed by tests -2. **Test Count**: Number of tests per component -3. **Pass Rate**: Percentage of tests passing -4. **Execution Time**: Time taken to run test suite -5. **Failure Rate**: Percentage of tests failing - -### Viewing Test Results - -```bash -# Generate HTML report -pytest tests/ --html=report.html --self-contained-html - -# Generate JUnit XML (for CI) -pytest tests/ --junit-xml=junit.xml -``` - -## ๐ŸŽฏ Testing Checklist - -Before submitting code, ensure: - -- [ ] All tests pass locally -- [ ] New features have tests -- [ ] Bug fixes have regression tests -- [ ] Code coverage >= 85% -- [ ] Tests are well-documented -- [ ] No skipped tests without reason -- [ ] API tests pass (if applicable) -- [ ] Integration tests pass -- [ ] Performance tests meet benchmarks - -## ๐Ÿ“š Additional Resources - -- [pytest Documentation](https://docs.pytest.org/) -- [Testing Best Practices](https://docs.python-guide.org/writing/tests/) -- [Coverage.py](https://coverage.readthedocs.io/) -- [pytest-cov](https://pytest-cov.readthedocs.io/) - -## ๐Ÿ†˜ Troubleshooting - -### Common Issues - -**Issue: Tests can't find src module** -```bash -# Solution: Add src to PYTHONPATH -export PYTHONPATH="${PYTHONPATH}:$(pwd)" -``` - -**Issue: API tests fail** -```bash -# Solution: Make sure API server is running -uvicorn src.agent_api:app --port 8000 -``` - -**Issue: Tests are slow** -```bash -# Solution: Run quick tests only -pytest tests/ -v -m "not slow" -``` - -**Issue: Import errors** -```bash -# Solution: Install test dependencies -pip install pytest pytest-cov pytest-mock requests -``` - ---- - -**Last Updated**: November 24, 2025 -**Version**: 1.0.0 -**Status**: Complete Testing Suite โœ… - diff --git a/docs/UI_TUTORIAL.md b/docs/UI_TUTORIAL.md deleted file mode 100644 index 4462d3e..0000000 --- a/docs/UI_TUTORIAL.md +++ /dev/null @@ -1,594 +0,0 @@ -# STT Control Panel - User Tutorial & Guide - -Welcome to the Adaptive Self-Learning Agentic AI System Control Panel! This guide will help you navigate the UI and understand how to use all the features. - -## Table of Contents - -1. [Getting Started](#getting-started) -2. [UI Overview](#ui-overview) -3. [Navigation Tabs](#navigation-tabs) -4. [Transcription Feature](#transcription-feature) -5. [Model Selection](#model-selection) -6. [Understanding Results](#understanding-results) -7. [Data Management](#data-management) -8. [Fine-Tuning](#fine-tuning) -9. [Troubleshooting](#troubleshooting) -10. [Important Notes](#important-notes) - ---- - -## Getting Started - -### Prerequisites - -- Python 3.8 or higher -- Virtual environment (recommended) -- Required dependencies installed (see `requirements.txt`) - -### Starting the Control Panel - -1. **Navigate to project directory:** - ```bash - cd Adaptive-Self-Learning-Agentic-AI-System - ``` - -2. **Activate virtual environment:** - ```bash - source venv/bin/activate # On macOS/Linux - # or - .venv\Scripts\activate # On Windows - ``` - -3. **Start the control panel:** - ```bash - ./start_control_panel.sh - ``` - -4. **Access the UI:** - - Open your browser and go to: `http://localhost:8000/app` - - API documentation: `http://localhost:8000/docs` - ---- - -## UI Overview - -The Control Panel has a modern, dark-themed interface with the following main sections: - -### Header -- **Logo & Title**: STT Control Panel -- **System Status Indicator**: Shows if the system is online/offline (green = online, red = offline) - -### Navigation Tabs -Six main tabs for different functionalities: -1. **Dashboard** - System overview and statistics -2. **Transcribe** - Audio transcription interface -3. **Data Management** - Failed cases and dataset preparation -4. **Fine-Tuning** - Fine-tuning orchestration -5. **Models** - Model version management -6. **Monitoring** - Performance metrics and trends - ---- - -## Navigation Tabs - -### 1. Dashboard Tab - -**Purpose**: Overview of system health and statistics - -**What you'll see:** -- **System Health Card**: Shows baseline model status, agent status, and LLM availability -- **Agent Statistics Card**: - - Error detection threshold - - Total errors detected - - Corrections made - - Feedback count -- **Data Statistics Card**: - - Total failed cases - - Corrected cases - - Correction rate percentage - - Average error score -- **Model Information Card**: Current model details (name, parameters, device) -- **Recent Activity**: Log of recent system activities - -**How to use:** -- Click the refresh icon (๐Ÿ”„) on any card to update statistics -- Monitor system health indicators -- Check if all components are operational - ---- - -### 2. Transcribe Tab โญ (Main Feature) - -**Purpose**: Upload audio files and get transcriptions with error detection and correction - -**Key Features:** -- Upload audio files (.wav, .mp3, .ogg) -- Select STT model version -- Choose transcription mode (Baseline or Agent) -- View side-by-side comparison of original vs. corrected transcripts - -#### Step-by-Step Transcription Process: - -1. **Select STT Model** (Dropdown): - - **Wav2Vec2 Base**: Baseline model (facebook/wav2vec2-base-960h) - - **Fine-tuned Wav2Vec2**: Improved model after fine-tuning - -2. **Choose Transcription Mode**: - - **Agent (Recommended)**: Full pipeline with error detection and LLM correction - - Processing time: 10-15 seconds (includes LLM processing) - - Shows both original STT transcript and LLM-refined transcript - - **Baseline (Fast)**: Simple transcription without error detection - - Processing time: 1-2 seconds - - No LLM correction - -3. **Agent Options** (only visible in Agent mode): - - **Enable Auto-Correction**: - - โœ… ON: LLM detects errors AND applies corrections - - โŒ OFF: LLM only detects errors but doesn't correct them - - **Record Errors Automatically**: - - โœ… ON: Failed cases are saved for future fine-tuning - - โŒ OFF: Errors detected but not saved - -4. **Upload Audio File**: - - Click the upload area or drag and drop - - Supported formats: WAV, MP3, OGG - - File info will display after selection - -5. **Click "Transcribe Audio"**: - - Button shows loading state during processing - - Results appear below when complete - -#### Understanding Transcription Results: - -**Side-by-Side Comparison:** -- **Left Column (Red border)**: STT Original Transcript - - Raw output from the selected STT model - - May contain errors, especially with base model -- **Right Column (Blue border)**: LLM Refined Transcript (Gold Standard) - - Corrected version after LLM analysis - - Shows what the transcript should be - -**Additional Information:** -- **Model Information**: Selected model and mode -- **Error Detection**: - - Has Errors: Yes/No badge - - Error Count: Number of errors found - - Error Score: Severity score (0-1) -- **Corrections Applied**: Number of corrections made -- **Case Recorded**: Case ID if errors were saved -- **Performance**: Inference time in seconds - ---- - -### 3. Data Management Tab - -**Purpose**: View and manage failed transcription cases - -**Features:** - -#### Failed Cases Section: -- **Search Bar**: Filter cases by keywords -- **Filter Dropdown**: - - All Cases - - Uncorrected (need attention) - - Corrected (already processed) -- **Case List**: Shows case cards with: - - Case ID - - Status badge (Corrected/Uncorrected) - - Transcript preview - - Timestamp - - Error score -- **Pagination**: Navigate through cases (Previous/Next) - -**Clicking a Case:** -- Opens a modal with full case details -- Shows original and corrected transcripts -- Displays error types -- Option to add manual corrections - -#### Dataset Preparation Section: -- **Minimum Error Score**: Filter cases by error severity (0.0-1.0) -- **Max Samples**: Limit number of samples in dataset -- **Balance Error Types**: Ensure diverse error types -- **Create Version**: Create a new dataset version -- **Prepare Dataset Button**: Generate fine-tuning dataset - -#### Available Datasets Section: -- Lists all prepared datasets -- Shows dataset IDs and status - ---- - -### 4. Fine-Tuning Tab - -**Purpose**: Manage automated fine-tuning pipeline - -**Features:** - -#### Orchestrator Status: -- **Status**: Operational/Unavailable -- **Ready for Fine-tuning**: Yes/No indicator -- **Total Jobs**: Number of fine-tuning jobs - -#### Trigger Fine-Tuning: -- **Force Trigger**: Bypass readiness checks -- **Trigger Fine-Tuning Button**: Manually start a fine-tuning job - -#### Fine-Tuning Jobs: -- List of all fine-tuning jobs -- Shows job ID, status, creation time, and dataset used -- Click to view job details - -**Note**: Fine-tuning requires sufficient failed cases and proper configuration. - ---- - -### 5. Models Tab - -**Purpose**: View and manage model versions - -**Features:** - -#### Current Model: -- Model name and parameters -- Device information -- Trainable parameters - -#### Deployed Model: -- Currently deployed model version -- Deployment timestamp -- Model metadata - -#### Model Versions: -- List of all model versions -- Status badges (deployed/available) -- Creation timestamps -- Click to view version details - ---- - -### 6. Monitoring Tab - -**Purpose**: Track system performance over time - -**Features:** - -#### Performance Metrics: -- Total inferences -- Average inference time -- Error detection rate -- Correction rate - -#### Performance Trends: -- Select metric (WER or CER) -- Select time window (7/30/90 days) -- View trend data (visualization can be added) - ---- - -## Model Selection Guide - -### Understanding Model Versions - -#### Wav2Vec2 Base (Baseline) -- **Model**: facebook/wav2vec2-base-960h -- **Framework**: PyTorch -- **Performance**: Baseline accuracy (~36% WER on real-world data) -- **Use Case**: Demonstrates baseline performance before fine-tuning -- **When to use**: Show the "before" state in your demo - -#### Fine-tuned Wav2Vec2 (Improved) -- **Model**: Fine-tuned Wav2Vec2 (trained on failed cases) -- **Framework**: PyTorch -- **Performance**: Improved accuracy after fine-tuning -- **Use Case**: Shows improvement after fine-tuning on domain-specific data -- **When to use**: Demonstrate improved performance after fine-tuning - -### Model Selection Strategy for Demo: - -1. **Start with Baseline**: Upload audio โ†’ See baseline transcription -2. **Show Error Detection**: Notice errors in original transcript -3. **Show LLM Correction**: See refined transcript in right column -4. **Explain Fine-tuning**: Mention that errors are saved for training -5. **Switch to Fine-tuned v2/v3**: Upload same audio โ†’ See better results - ---- - -## Understanding Results - -### Transcript Comparison - -**Original STT Transcript (Left):** -- Raw output from speech-to-text model -- May contain: - - Spelling errors - - Medical terminology mistakes - - Grammar issues - - Word substitutions - -**LLM Refined Transcript (Right):** -- Corrected by Llama LLM (via Ollama) -- Improvements: - - Fixed spelling errors - - Corrected medical terms - - Improved grammar - - Better context understanding - -### Error Detection Metrics - -- **Has Errors**: Boolean indicating if errors were found -- **Error Count**: Number of individual errors detected -- **Error Score**: Overall quality score (0.0 = perfect, 1.0 = many errors) -- **Error Types**: Categories of errors (medical terminology, spelling, grammar) - -### Case Recording - -When errors are detected and "Record Errors Automatically" is enabled: -- Case is saved to data management system -- Gets a unique Case ID -- Original and corrected transcripts are stored -- Used for future fine-tuning dataset preparation - ---- - -## Data Management - -### Failed Cases Workflow - -1. **Automatic Recording**: - - Errors detected during transcription - - Cases automatically saved if "Record Errors Automatically" is ON - -2. **Manual Review**: - - View cases in Data Management tab - - Filter by status (corrected/uncorrected) - - Click case to view details - -3. **Manual Correction**: - - Open case details - - Add correction if needed - - Save correction - -4. **Dataset Preparation**: - - Set filters (error score, max samples) - - Click "Prepare Dataset" - - Dataset created for fine-tuning - -### Dataset Preparation Tips - -- **Minimum Error Score**: - - Lower (0.3): Include more cases, diverse errors - - Higher (0.7): Only severe errors, focused training -- **Max Samples**: - - Start with 100-500 for testing - - Use 1000+ for production fine-tuning -- **Balance Error Types**: - - โœ… Recommended: Ensures diverse training data - - โŒ Off: May bias toward common error types - ---- - -## Fine-Tuning - -### When Fine-Tuning Triggers - -The system automatically triggers fine-tuning when: -- Sufficient failed cases accumulated (threshold: configurable) -- Error rate is high enough -- System is ready (no ongoing jobs) - -### Manual Trigger - -You can manually trigger fine-tuning: -1. Go to Fine-Tuning tab -2. Check "Force Trigger" if needed (bypasses checks) -3. Click "Trigger Fine-Tuning" -4. Monitor job status - -### Fine-Tuning Process - -1. **Dataset Preparation**: Failed cases converted to training format -2. **Model Training**: Fine-tune on prepared dataset -3. **Validation**: Test against baseline -4. **Deployment**: Deploy if improvements validated -5. **Versioning**: New model version created - ---- - -## Troubleshooting - -### Common Issues - -#### 1. "System Offline" Status -**Problem**: Red status indicator in header -**Solutions**: -- Check if server is running: `./start_control_panel.sh` -- Verify port 8000 is not in use -- Check server logs for errors - -#### 2. Transcription Fails -**Problem**: Error message when transcribing -**Solutions**: -- Check audio file format (WAV, MP3, OGG supported) -- Ensure file is not corrupted -- Check server logs for detailed error -- Verify model is loaded (check Dashboard) - -#### 3. "Fine-tuned model not found" -**Problem**: Fine-tuned model cannot be loaded -**Solutions**: -- Ensure fine-tuned model exists at `models/finetuned_wav2vec2/` -- Run fine-tuning script first if model doesn't exist -- Check server logs for detailed error messages - -#### 4. Slow Transcription -**Problem**: Transcription takes too long -**Solutions**: -- Agent mode takes 10-15 seconds (normal for LLM processing) -- Use Baseline mode for faster results (1-2 seconds) -- Check system resources (CPU/GPU) -- Reduce audio file size if very large - -#### 5. No Results Displayed -**Problem**: Transcription completes but no results shown -**Solutions**: -- Check browser console for JavaScript errors -- Refresh the page -- Check network tab for API errors -- Verify API is responding: `http://localhost:8000/api/health` - -#### 6. Model Not Loading -**Problem**: Model fails to load -**Solutions**: -- Check internet connection (models download from Hugging Face) -- Ensure sufficient disk space (~2-4GB per model) -- Check model name is correct -- Review server logs for specific error - -### Getting Help - -1. **Check Logs**: Server logs show detailed error messages -2. **API Documentation**: Visit `http://localhost:8000/docs` for API details -3. **Health Check**: Visit `http://localhost:8000/api/health` for system status -4. **Browser Console**: Press F12 to see frontend errors - ---- - -## Important Notes - -### System Architecture - -**Components:** -1. **STT Models**: Speech-to-text transcription (Wav2Vec2) -2. **LLM Corrector**: Llama LLM (via Ollama) for error detection and correction -3. **Error Detector**: Heuristic-based error detection -4. **Data Manager**: Stores failed cases and manages datasets -5. **Fine-tuning Coordinator**: Orchestrates model fine-tuning - -### Processing Flow - -1. **Audio Upload** โ†’ STT Model transcribes -2. **Error Detection** โ†’ Detects errors in transcript -3. **LLM Correction** โ†’ Llama LLM refines transcript -4. **Case Recording** โ†’ Saves errors if enabled -5. **Fine-tuning** โ†’ Uses cases to improve model - -### Best Practices - -1. **For Demos**: - - Start with Base v1 to show poor performance - - Use Agent mode to show full pipeline - - Enable both auto-correction and error recording - - Switch to Fine-tuned models to show improvement - -2. **For Production**: - - Use Fine-tuned v3 for best accuracy - - Monitor error rates in Monitoring tab - - Regularly review failed cases - - Prepare datasets when sufficient cases accumulated - -3. **Audio Files**: - - Use clear audio (minimize background noise) - - WAV format recommended for best quality - - Keep files under 10MB for faster processing - - Sample rate: 16kHz is optimal - -### Performance Expectations - -- **Base Model (Wav2Vec2 Base)**: - - Speed: ~1-2 seconds - - Accuracy: ~36% WER on real-world data (demonstrates need for fine-tuning) - -- **Fine-tuned Model (Fine-tuned Wav2Vec2)**: - - Speed: ~1-2 seconds - - Accuracy: Improved after fine-tuning on domain-specific data - -- **LLM Correction**: - - Processing time: <1 second (with Ollama) - - Improves transcript quality significantly - -### Security & Privacy - -- All processing happens locally (if using local models) -- Audio files are temporarily stored during processing -- Failed cases stored in `data/production/` directory -- No data sent to external services (unless using cloud APIs) - -### Limitations - -1. **Ollama LLM**: Requires Ollama server running locally with Llama models installed -2. **Model Loading**: First load takes time (downloads from Hugging Face) -3. **Memory**: Large models require sufficient RAM -4. **Audio Length**: Very long audio files may timeout - ---- - -## Quick Reference - -### Keyboard Shortcuts -- **F12**: Open browser developer console -- **Ctrl+R / Cmd+R**: Refresh page -- **Ctrl+Shift+R / Cmd+Shift+R**: Hard refresh (clear cache) - -### Important URLs -- **Control Panel**: `http://localhost:8000/app` -- **API Docs**: `http://localhost:8000/docs` -- **Health Check**: `http://localhost:8000/api/health` -- **API Root**: `http://localhost:8000/` - -### File Locations -- **Audio Files**: Upload via UI (temporary storage) -- **Failed Cases**: `data/production/failed_cases/` -- **Datasets**: `data/production/finetuning/` -- **Model Versions**: `data/production/versions/` - ---- - -## Demo Script Example - -Here's a suggested flow for demonstrating the system: - -1. **Introduction** (Dashboard Tab): - - Show system health - - Explain components - -2. **Base Model Demo** (Transcribe Tab): - - Select "Wav2Vec2 Base" - - Upload audio file - - Show baseline transcription in left column - - Explain errors - -3. **LLM Correction**: - - Show refined transcript in right column - - Highlight improvements - - Explain error detection and correction - -4. **Data Collection**: - - Show case was recorded - - Explain this feeds fine-tuning - -5. **Fine-tuned Model** (Transcribe Tab): - - Switch to "Fine-tuned Wav2Vec2" - - Upload same audio - - Show improved transcription - - Compare with base model results - -6. **System Overview**: - - Show Data Management tab (failed cases) - - Show Fine-tuning tab (jobs) - - Show Monitoring tab (metrics) - ---- - -## Support & Resources - -- **Project Documentation**: See `docs/` directory -- **API Documentation**: Built-in at `/docs` endpoint -- **Setup Guide**: See `docs/SETUP_INSTRUCTIONS.md` - ---- - -**Happy Transcribing! ๐ŸŽคโœจ** - -For questions or issues, check the troubleshooting section or review server logs. - diff --git a/docs/WANDB.md b/docs/WANDB.md new file mode 100644 index 0000000..50035ef --- /dev/null +++ b/docs/WANDB.md @@ -0,0 +1,81 @@ +# Weights & Biases Integration + +Experiment tracking and hyperparameter optimization for fine-tuning. + +## Quick Start + +```bash +pip install wandb +wandb login +``` + +```python +from src.data.finetuning_orchestrator import FinetuningConfig + +config = FinetuningConfig( + use_wandb=True, + wandb_project="my-project" +) + +coordinator = FinetuningCoordinator( + data_manager=data_manager, + finetuning_config=config +) +workflow = coordinator.run_complete_workflow() +# Check W&B dashboard for visualizations +``` + +## What Gets Tracked + +- **Training**: Loss per epoch, learning rate, duration +- **Validation**: WER/CER comparison, improvement metrics, pass/fail +- **Regression tests**: Test pass rates, degradation metrics +- **Dataset**: Split sizes, error type distribution +- **Model artifacts**: Trained models, metadata, version tracking + +## Hyperparameter Sweeps + +**Recommendation**: Use Random Search (20 trials) for initial optimization. + +```python +from src.data.wandb_sweeps import WandbSweepOrchestrator, SweepConfig + +sweep_orch = WandbSweepOrchestrator(project_name="stt-optimization") +config = SweepConfig.create_finetuning_sweep(method='random', num_trials=20) +sweep_id = sweep_orch.create_sweep(config) +sweep_orch.run_sweep_agent(your_train_fn, sweep_id, count=20) + +best = sweep_orch.get_best_run(sweep_id) +print(f"Optimal config: {best['hyperparameters']}") +``` + +### Search Strategies + +| Strategy | Best For | Trials | Expected Improvement | +|----------|----------|--------|---------------------| +| Random | Initial exploration | 20-50 | 20-30% WER | +| Bayesian | Refinement | 30-100 | 30-40% WER | +| Grid | Final validation | Limited | Exhaustive | + +### Recommended Approach + +1. **Phase 1**: Random search (20 trials) - find good config (~1-2 days) +2. **Phase 2**: Use best config for all future fine-tuning +3. **Phase 3**: Periodic mini-sweeps (5 trials) every 6 months as data evolves + +## Demo + +```bash +python experiments/demo_wandb_tracking.py +python experiments/demo_wandb_sweeps.py +``` + +## Configuration + +```python +config = FinetuningConfig( + use_wandb=True, + wandb_project="stt-finetuning", + wandb_entity="my-team" +) +``` diff --git a/docs/WANDB_INTEGRATION_SUMMARY.md b/docs/WANDB_INTEGRATION_SUMMARY.md deleted file mode 100644 index 0967e3e..0000000 --- a/docs/WANDB_INTEGRATION_SUMMARY.md +++ /dev/null @@ -1,393 +0,0 @@ -# Weights & Biases Integration - Summary - -## โœจ What's New - -Weights & Biases (W&B) is now integrated into the fine-tuning orchestration system, providing: -1. **Automatic experiment tracking** - Track all metrics automatically -2. **Beautiful visualizations** - Professional dashboards -3. **Hyperparameter optimization** - W&B Sweeps for finding optimal configs -4. **Model comparison** - Compare runs side-by-side - ---- - -## ๐Ÿš€ Quick Start - -### 1. Install W&B - -```bash -pip install wandb -wandb login -``` - -### 2. Enable in Code - -```python -from src.data.finetuning_orchestrator import FinetuningConfig - -config = FinetuningConfig( - use_wandb=True, # Enable W&B - wandb_project="my-project" -) -``` - -### 3. Run Fine-Tuning - -Everything is tracked automatically! - -```python -coordinator = FinetuningCoordinator( - data_manager=data_manager, - finetuning_config=config -) - -workflow = coordinator.run_complete_workflow() -# Check W&B dashboard for visualizations! -``` - ---- - -## ๐Ÿ“Š What Gets Tracked - -### Automatically Logged - -โœ… **Training Metrics** -- Loss per epoch -- Learning rate schedule -- Training duration - -โœ… **Validation Results** -- WER/CER comparison (model vs baseline) -- Improvement metrics -- Pass/fail status - -โœ… **Regression Tests** -- Test pass rates -- Degradation metrics -- Per-test results - -โœ… **Dataset Information** -- Split sizes (train/val/test) -- Error type distribution -- Data quality metrics - -โœ… **System Metrics** -- Error case accumulation -- Correction rates -- Models deployed over time - -โœ… **Model Artifacts** -- Trained models -- Metadata and configuration -- Version tracking - ---- - -## ๐Ÿ“ˆ Visualizations Created - -W&B automatically generates: - -- **Line Plots** - Training/validation loss curves -- **Bar Charts** - WER/CER comparisons -- **Pie Charts** - Test pass/fail rates -- **Scatter Plots** - Degradation analysis -- **Tables** - Detailed metrics -- **Custom Plots** - Error distributions - ---- - -## ๐Ÿ“ Files Added - -### Core Integration - -1. **`src/data/wandb_tracker.py`** (650+ lines) - - Main W&B integration class - - Logging methods for all metrics - - Visualization generation - - Artifact management - -2. **Updated: `src/data/finetuning_orchestrator.py`** - - W&B initialization - - Automatic run creation - - Metric logging at key points - -3. **Updated: `requirements.txt`** - - Added `wandb>=0.16.0` - -### Documentation - -4. **`docs/WANDB_INTEGRATION.md`** (500+ lines) - - Complete W&B guide - - Configuration options - - Usage examples - - Best practices - - Troubleshooting - -5. **`experiments/demo_wandb_tracking.py`** (450+ lines) - - Comprehensive demo - - 6 different tracking scenarios - - Working examples - -6. **Updated: `docs/FINETUNING_QUICK_START.md`** - - W&B section added - - Quick setup guide - ---- - -## ๐ŸŽฏ Features - -### Core Features - -- **Automatic Tracking** - No manual logging needed -- **Beautiful Dashboards** - Professional visualizations -- **Run Comparison** - Compare multiple experiments -- **Team Collaboration** - Share results easily -- **Model Versioning** - Track model lineage -- **Hyperparameter Logging** - Track all configurations - -### Advanced Features - -- **Audio Sample Logging** - Log audio with transcripts -- **Custom Plots** - Create custom visualizations -- **Performance History** - Track trends over time -- **Confusion Matrices** - Error type analysis -- **Alerts** - Get notified on issues -- **Model Registry** - Production model management - ---- - -## ๐Ÿ’ก Usage Examples - -### Example 1: Basic Usage - -```python -from src.data.wandb_tracker import WandbTracker - -tracker = WandbTracker(project_name="my-project") -tracker.start_run(run_name="experiment_1") - -# Log training -tracker.log_training_metrics( - epoch=10, - train_loss=0.15, - val_loss=0.18 -) - -tracker.finish_run() -``` - -### Example 2: Automatic with Orchestrator - -```python -# Just enable W&B in config -config = FinetuningConfig(use_wandb=True) -orchestrator = FinetuningOrchestrator(data_manager, config) - -# Everything tracked automatically! -job = orchestrator.trigger_finetuning(force=True) -``` - -### Example 3: Validation Results - -```python -validation_result = { - 'model_wer': 0.12, - 'baseline_wer': 0.20, - 'wer_improvement': 0.08, - 'passed': True -} - -tracker.log_validation_results(validation_result, "model_v1") -# W&B creates comparison charts automatically! -``` - ---- - -## ๐Ÿ”ง Configuration - -### In Fine-Tuning Config - -```python -config = FinetuningConfig( - # W&B settings - use_wandb=True, # Enable/disable - wandb_project="stt-finetuning", # Project name - wandb_entity="my-team", # Optional: team/user - - # Other settings... - min_error_cases=100 -) -``` - -### Standalone Tracker - -```python -tracker = WandbTracker( - project_name="my-project", - entity="my-team", - enabled=True, - config={'model': 'whisper-base'} -) -``` - ---- - -## ๐Ÿงช Testing - -### Run Demo - -```bash -# Install W&B and login -pip install wandb -wandb login - -# Run comprehensive demo -python experiments/demo_wandb_tracking.py -``` - -### Demo Includes - -1. โœ… Basic training metrics -2. โœ… Validation results -3. โœ… Regression tests -4. โœ… Dataset information -5. โœ… System metrics -6. โœ… Full orchestrator integration - ---- - -## ๐Ÿ“š Documentation - -**Complete Guide:** `docs/WANDB_INTEGRATION.md` - -Topics covered: -- Setup and installation -- Configuration options -- All tracked metrics -- Visualization examples -- Best practices -- Troubleshooting -- Advanced features -- CI/CD integration - ---- - -## ๐ŸŽ“ Benefits - -### For Experimentation - -- **Track Everything** - Never lose experiment results -- **Compare Runs** - Easily compare different configurations -- **Visualize Trends** - See performance over time -- **Share Results** - Collaborate with team - -### For Production - -- **Monitor Performance** - Track live metrics -- **Model Registry** - Manage production models -- **Alerts** - Get notified of issues -- **Audit Trail** - Complete history - -### For Teams - -- **Collaboration** - Share insights easily -- **Reports** - Create custom reports -- **Documentation** - Experiments self-document -- **Knowledge Sharing** - Learn from each other - ---- - -## ๐ŸŽฏ Next Steps - -### 1. Setup - -```bash -pip install wandb -wandb login -``` - -### 2. Try Demo - -```bash -python experiments/demo_wandb_tracking.py -``` - -### 3. Enable in Your Code - -```python -config = FinetuningConfig(use_wandb=True) -``` - -### 4. Run Fine-Tuning - -```bash -python experiments/demo_finetuning_orchestration.py -``` - -### 5. Check Dashboard - -Visit https://wandb.ai/ to see your results! - ---- - -## ๐Ÿ“Š Dashboard Preview - -Your W&B dashboard will show: - -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Fine-Tuning Run - experiment_42 โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ Status: โœ… Completed โ”‚ -โ”‚ Duration: 2h 15m โ”‚ -โ”‚ Final WER: 0.12 (โ†“ 40% vs baseline) โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ ๐Ÿ“Š Charts โ”‚ -โ”‚ โ€ข Training Loss Curve โ”‚ -โ”‚ โ€ข WER Comparison (Model vs Baseline) โ”‚ -โ”‚ โ€ข Regression Test Results โ”‚ -โ”‚ โ€ข Error Type Distribution โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ ๐Ÿ“ฆ Artifacts โ”‚ -โ”‚ โ€ข Model: finetuned_v1 (150MB) โ”‚ -โ”‚ โ€ข Dataset: dataset_001 โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - ---- - -## ๐Ÿค Support - -- **W&B Docs:** https://docs.wandb.ai/ -- **Integration Guide:** `docs/WANDB_INTEGRATION.md` -- **Demo:** `experiments/demo_wandb_tracking.py` - ---- - -## โœ… Summary - -**Added:** -- โœ… Complete W&B integration (650+ lines) -- โœ… Automatic metric tracking -- โœ… Beautiful visualizations -- โœ… Comprehensive demo (450+ lines) -- โœ… Full documentation (500+ lines) -- โœ… Easy configuration - -**No Changes Required:** -- Existing code works as-is -- W&B is optional (can be disabled) -- No breaking changes - -**Get Started:** -```bash -pip install wandb && wandb login -python experiments/demo_wandb_tracking.py -``` - -**View Results:** -https://wandb.ai/ - ---- - -๐ŸŽ‰ **Track your experiments with professional-grade tools!** - diff --git a/docs/WANDB_SWEEPS_BENEFITS.md b/docs/WANDB_SWEEPS_BENEFITS.md deleted file mode 100644 index a7fde3f..0000000 --- a/docs/WANDB_SWEEPS_BENEFITS.md +++ /dev/null @@ -1,337 +0,0 @@ -# Will This Project Benefit from W&B Random Search? - -## TL;DR: YES! ๐ŸŽฏ - -**Expected Benefits:** -- ๐ŸŽฏ **20-40% better WER/CER** through optimal hyperparameters -- โšก **Saves 2-4 weeks** of manual experimentation -- ๐Ÿ’ฐ **Better ROI** on GPU spending -- ๐Ÿ”„ **Automated** - Run once, benefit forever - ---- - -## Why This Project Specifically Benefits - -### 1. **Complex Hyperparameter Space** - -Your STT fine-tuning has many parameters affecting performance: - -``` -Learning Rate โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -Batch Size โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -Epochs โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -Warmup Steps โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€> Final WER/CER Performance -Weight Decay โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -Dropout โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -Gradient Accumulationโ”˜ -``` - -**Manual Testing:** -- Try 3 learning rates ร— 3 batch sizes ร— 3 epochs = 27 combinations -- Takes 2-4 weeks -- Still might miss optimal combination - -**W&B Random Search:** -- Tests 20-50 combinations automatically -- Takes 1-2 days -- Smarter sampling of parameter space -- Visualizes relationships - ---- - -### 2. **Different Error Types Need Different Tuning** - -Your system handles multiple error types: -- Word substitutions -- Missing words -- Extra words -- Pronunciation errors - -**Each error type may benefit from different hyperparameters!** - -W&B Sweeps can find: -- Best general-purpose config -- Or specialized configs per error type - ---- - -### 3. **Data Evolves Over Time** - -As your system learns: -- New error patterns emerge -- Data distribution changes -- Optimal hyperparameters may shift - -**Solution:** Periodic mini-sweeps (5 trials) to adapt - ---- - -## ๐Ÿ“Š Concrete Example for Your Project - -### Scenario: You have 1000 error cases for fine-tuning - -### Without Optimization - -```python -# Use default hyperparameters -training_params = { - 'learning_rate': 1e-5, # Guess - 'batch_size': 16, # Guess - 'epochs': 5 # Guess -} - -# Result after fine-tuning -WER: 0.18 (okay but not optimal) -Training time: 2 hours -``` - -### With Random Search (20 trials) - -```python -# Let W&B find optimal hyperparameters -sweep_config = SweepConfig.create_finetuning_sweep( - method='random', - num_trials=20 -) - -sweep_id = sweep_orch.create_sweep(sweep_config) -sweep_orch.run_sweep_agent(train_fn, sweep_id, count=20) - -# W&B finds optimal configuration -best = sweep_orch.get_best_run(sweep_id) -# Best hyperparameters: -# { -# 'learning_rate': 3.2e-5, # Found by sweep -# 'batch_size': 24, # Found by sweep -# 'epochs': 12 # Found by sweep -# } - -# Result with optimal hyperparameters -WER: 0.12 (33% better!) -Training time: 1.5 hours (faster convergence) -``` - -**Cost:** 20 trials ร— 2 hours = 40 GPU hours -**Benefit:** 33% better WER for EVERY future fine-tuning -**ROI:** Pays for itself after 2-3 production fine-tuning runs - ---- - -## ๐ŸŽ“ Which Strategy to Use? - -### For This Project: **Random Search First** - -**Reasons:** -1. **Efficient** - Good results with 20-30 trials -2. **Parallel** - Can run multiple trials simultaneously -3. **Robust** - Works well for all types of problems -4. **Fast** - 1-2 days vs weeks of manual tuning - -**Later (Optional): Bayesian Optimization** -- After random search, if you want to squeeze out last 5-10% -- Use narrower search space around best values from random - -**Skip: Grid Search** -- Too expensive for 7+ parameters -- Better to use random or Bayesian - ---- - -## ๐Ÿ’ก Practical Recommendations - -### Phase 1: Initial Optimization (Do This!) - -**Goal:** Find good hyperparameters quickly - -```python -# Run once when setting up your system -sweep_config = SweepConfig.create_finetuning_sweep( - method='random', - num_trials=20 # Good balance -) - -sweep_id = sweep_orch.create_sweep(sweep_config) -sweep_orch.run_sweep_agent(train_fn, sweep_id, count=20) - -# Get and save best config -best = sweep_orch.get_best_run(sweep_id) -save_config('optimal_config.json', best['hyperparameters']) -``` - -**Investment:** 1-2 days, 40-60 GPU hours -**Return:** Use optimal config forever - ---- - -### Phase 2: Production Use (Ongoing) - -**Goal:** Use optimized hyperparameters for all fine-tuning - -```python -# Load optimal hyperparameters -optimal_params = load_config('optimal_config.json') - -# Use in all automated fine-tuning -def custom_training(job, params): - # Merge with optimal hyperparameters - final_params = {**optimal_params, **params} - return train_model(job, final_params) - -coordinator.set_training_callback(custom_training) - -# All future fine-tuning uses optimal config! -``` - -**No Additional Cost:** Just using what you learned -**Benefit:** 20-40% better performance on every run - ---- - -### Phase 3: Periodic Re-tune (Every 6 months) - -**Goal:** Adapt to evolving data - -```python -# Quick check if optimal config still works -mini_sweep = SweepConfig.create_minimal_sweep(num_trials=5) -sweep_id = sweep_orch.create_sweep(mini_sweep) -sweep_orch.run_sweep_agent(train_fn, sweep_id, count=5) - -# Update config if significant improvement found -new_best = sweep_orch.get_best_run(sweep_id) -if new_best['metric_value'] < current_best * 0.95: # 5% improvement - update_optimal_config(new_best['hyperparameters']) -``` - -**Cost:** ~10 GPU hours every 6 months -**Benefit:** Stay optimal as data evolves - ---- - -## ๐Ÿ“ˆ Expected Results - -### Realistic Improvements for STT Fine-Tuning - -Based on published research and our experience: - -| Metric | Default | After Random Search | Improvement | -|--------|---------|---------------------|-------------| -| **WER** | 0.20 | 0.12-0.14 | **30-40%** | -| **CER** | 0.10 | 0.06-0.07 | **30-40%** | -| **Training Time** | 2.0h | 1.2-1.5h | **25-40% faster** | -| **Convergence** | Unstable | Stable | **More reliable** | - -### What Affects Your Results - -**Better Results If:** -- โœ… Large dataset (>500 samples) -- โœ… Diverse error types -- โœ… GPU available for parallel trials -- โœ… Can run 20+ trials - -**Good Results Even If:** -- โœ… Small dataset (100-500 samples) -- โœ… Limited GPU (sequential trials) -- โœ… Only 10 trials - ---- - -## ๐Ÿš€ Getting Started - -### Step 1: Run Demo (5 minutes) - -```bash -python experiments/demo_wandb_sweeps.py -``` - -See how sweeps work with mock training. - ---- - -### Step 2: Quick Test (2 hours) - -```bash -# Try with 3 trials on your actual data -python experiments/quick_sweep_test.py --trials 3 -``` - -Verify it works with your setup. - ---- - -### Step 3: Full Optimization (1-2 days) - -```python -from src.data.wandb_sweeps import WandbSweepOrchestrator, SweepConfig - -sweep_orch = WandbSweepOrchestrator(project_name="stt-optimization") -config = SweepConfig.create_finetuning_sweep(method='random', num_trials=20) -sweep_id = sweep_orch.create_sweep(config) - -# Run sweep (can parallelize on multiple GPUs) -sweep_orch.run_sweep_agent(your_train_fn, sweep_id, count=20) - -# Get best hyperparameters -best = sweep_orch.get_best_run(sweep_id) -print(f"Optimal config: {best['hyperparameters']}") -``` - ---- - -## ๐ŸŽฏ Final Recommendation - -### For This Project: - -**โœ… YES - Use W&B Random Search with 20 trials** - -**Timing:** -- Do it ONCE at the beginning -- Takes 1-2 days -- Use results forever - -**ROI:** -- Investment: 40-60 GPU hours -- Return: 30% better WER on every fine-tuning -- Lifetime value: Huge (if you fine-tune 10+ times) - -**Strategy:** -1. Random Search (20 trials) - Find good config -2. Use optimal config in production - Benefit forever -3. Re-optimize every 6 months - Stay current - ---- - -## ๐Ÿ“ฆ What's Already Implemented - -You already have: -- โœ… `src/data/wandb_sweeps.py` - Full sweep integration -- โœ… `SweepConfig.create_finetuning_sweep()` - Pre-configured for STT -- โœ… `WandbSweepOrchestrator` - Easy-to-use API -- โœ… `experiments/demo_wandb_sweeps.py` - Working demo -- โœ… `docs/WANDB_SWEEPS_GUIDE.md` - Complete guide - -**Just run it!** - -```bash -python experiments/demo_wandb_sweeps.py -``` - ---- - -## ๐ŸŽ‰ Bottom Line - -**Question:** Will this project benefit from W&B Random Search? - -**Answer:** **ABSOLUTELY YES!** - -- โœ… 30-40% better WER/CER -- โœ… Automated (save weeks of work) -- โœ… Excellent ROI -- โœ… Already implemented and ready to use -- โœ… Only costs 1-2 days of GPU time -- โœ… Benefits last forever - -**Recommendation: Run 20-trial random search sweep as soon as possible!** - -๐Ÿš€ **Start here:** `python experiments/demo_wandb_sweeps.py` - diff --git a/docs/WANDB_SWEEPS_GUIDE.md b/docs/WANDB_SWEEPS_GUIDE.md deleted file mode 100644 index 288bfe7..0000000 --- a/docs/WANDB_SWEEPS_GUIDE.md +++ /dev/null @@ -1,716 +0,0 @@ -# W&B Sweeps for Hyperparameter Optimization - -## Should You Use W&B Random Search for This Project? - -**YES! This project will significantly benefit from W&B Sweeps. Here's why:** - ---- - -## ๐ŸŽฏ Benefits for Your STT Fine-Tuning Project - -### 1. **Optimize Multiple Hyperparameters Simultaneously** - -When fine-tuning speech-to-text models, performance depends on: - -- **Learning Rate** - Most critical parameter -- **Batch Size** - Affects convergence and memory -- **Training Epochs** - Balance between underfitting and overfitting -- **Warmup Steps** - Critical for transformer models -- **Weight Decay** - Regularization strength -- **Dropout** - Prevent overfitting -- **Gradient Accumulation** - Effective batch size - -**Manual tuning = weeks of experimentation** -**W&B Sweeps = automated optimization in hours** - -### 2. **Automatic Discovery of Best Configuration** - -Different datasets and error types may need different hyperparameters: - -- **Word Substitution Errors** โ†’ May need lower learning rate -- **Missing Word Errors** โ†’ May benefit from more epochs -- **Pronunciation Errors** โ†’ May need different dropout - -W&B Sweeps finds the optimal configuration for YOUR specific data. - -### 3. **Cost Efficiency** - -- Avoid wasting GPU time on suboptimal configurations -- Early termination of poor-performing trials -- Parallel execution on multiple GPUs -- Smart search strategies reduce total trials needed - -### 4. **Reproducibility** - -- All hyperparameters logged automatically -- Easy to reproduce best results -- Track what worked and what didn't -- Share configurations with team - ---- - -## ๐Ÿ“Š Search Strategies Comparison - -### Random Search (RECOMMENDED START) - -**Best For:** Initial exploration, limited compute budget - -**Pros:** -- โœ… Fast and efficient -- โœ… Good coverage of search space -- โœ… No assumptions about parameter space -- โœ… Parallelizes perfectly -- โœ… Often finds good solutions quickly - -**Cons:** -- โŒ May miss optimal configuration -- โŒ Doesn't learn from previous trials - -**Recommended Trials:** 20-50 - -**When to Use:** -- First time optimizing -- Limited GPU hours -- Need quick results -- Exploring new dataset - -```python -sweep_config = SweepConfig.create_finetuning_sweep( - method='random', - num_trials=20 -) -``` - ---- - -### Bayesian Optimization (RECOMMENDED FOR REFINEMENT) - -**Best For:** Expensive training, refined optimization - -**Pros:** -- โœ… Learns from previous trials -- โœ… More efficient than random -- โœ… Focuses on promising regions -- โœ… Better for expensive computations -- โœ… Can find better solutions with fewer trials - -**Cons:** -- โŒ Sequential (harder to parallelize) -- โŒ Needs more trials to warm up -- โŒ Can get stuck in local optima - -**Recommended Trials:** 30-100 - -**When to Use:** -- Refining after random search -- Long training times -- Want best possible results -- Have computational budget - -```python -sweep_config = SweepConfig.create_finetuning_sweep( - method='bayes', - num_trials=50 -) -``` - ---- - -### Grid Search - -**Best For:** Final tuning, specific parameter ranges - -**Pros:** -- โœ… Exhaustive search -- โœ… No configurations missed -- โœ… Guaranteed to find best in search space -- โœ… Good for final validation - -**Cons:** -- โŒ Exponentially grows with parameters -- โŒ Very expensive -- โŒ Overkill for most cases - -**Recommended Trials:** Limited parameter space only - -**When to Use:** -- Final tuning with narrow ranges -- Few parameters to tune -- Need absolute certainty -- Publication/production validation - -```python -sweep_config = SweepConfig.create_custom_sweep( - parameters={ - 'learning_rate': {'values': [1e-5, 5e-5, 1e-4]}, - 'batch_size': {'values': [16, 32]} - }, - method='grid' -) -# This creates 3 ร— 2 = 6 trials -``` - ---- - -## ๐ŸŽ“ Recommended Strategy for This Project - -### Phase 1: Random Search (Quick Exploration) - -```python -# Start with random search - 20 trials -sweep_config = SweepConfig.create_finetuning_sweep( - method='random', - num_trials=20 -) - -# Focus on critical parameters -sweep_config['parameters'] = { - 'learning_rate': { - 'distribution': 'log_uniform_values', - 'min': 1e-6, - 'max': 1e-4 - }, - 'batch_size': {'values': [8, 16, 32]}, - 'epochs': {'values': [5, 10, 15]} -} -``` - -**Expected Time:** 4-8 hours (depending on GPU) -**Expected Benefit:** Find configurations within 80-90% of optimal - ---- - -### Phase 2: Bayesian Optimization (Refinement) - -```python -# Use best parameters from Phase 1 as starting point -# Narrow down the search space - -sweep_config = SweepConfig.create_finetuning_sweep( - method='bayes', - num_trials=30 -) - -# Refine learning rate around best value from Phase 1 -best_lr = 3e-5 # From Phase 1 -sweep_config['parameters']['learning_rate'] = { - 'distribution': 'log_uniform_values', - 'min': best_lr * 0.5, - 'max': best_lr * 2.0 -} -``` - -**Expected Time:** 6-12 hours -**Expected Benefit:** Achieve 95-99% of optimal performance - ---- - -### Phase 3: Production (Use Best Config) - -```python -# Use best hyperparameters for all future fine-tuning -best_config = sweep_orch.get_best_run(sweep_id) - -production_params = { - 'learning_rate': best_config['hyperparameters']['learning_rate'], - 'batch_size': best_config['hyperparameters']['batch_size'], - 'epochs': best_config['hyperparameters']['epochs'] -} - -# Use in orchestrator -orchestrator.start_training(job_id, training_params=production_params) -``` - ---- - -## ๐Ÿ’ก Practical Implementation - -### Option 1: One-Time Optimization (Recommended) - -```python -from src.data.wandb_sweeps import WandbSweepOrchestrator, SweepConfig - -# Create sweep -sweep_orch = WandbSweepOrchestrator(project_name="stt-finetuning") -sweep_config = SweepConfig.create_finetuning_sweep( - method='random', - num_trials=20 -) - -sweep_id = sweep_orch.create_sweep(sweep_config) - -# Run sweep -sweep_orch.run_sweep_agent(your_training_function, sweep_id, count=20) - -# Get best hyperparameters -best = sweep_orch.get_best_run(sweep_id) - -# Save for future use -sweep_orch.save_best_config('optimal_hyperparameters.json', sweep_id) - -# Use best config for all future fine-tuning -with open('optimal_hyperparameters.json') as f: - optimal_params = json.load(f)['hyperparameters'] -``` - -**Cost:** 20 training runs -**Time:** 1-2 days -**Benefit:** Optimal hyperparameters for lifetime of project - ---- - -### Option 2: Continuous Optimization (Advanced) - -```python -# Run mini-sweeps periodically as data evolves -# Every 1000 new error cases, run a small sweep - -if new_error_cases > 1000: - # Quick 5-trial sweep - sweep_config = SweepConfig.create_minimal_sweep(num_trials=5) - sweep_id = sweep_orch.create_sweep(sweep_config) - sweep_orch.run_sweep_agent(train, sweep_id, count=5) - - # Update production config if significantly better - new_best = sweep_orch.get_best_run(sweep_id) - if new_best['metric_value'] < current_best_wer * 0.95: - update_production_config(new_best) -``` - -**Cost:** 5 runs per 1000 error cases -**Benefit:** Adapt to evolving data distribution - ---- - -## ๐Ÿ“ˆ Expected Improvements - -Based on typical hyperparameter optimization results: - -### Baseline (No Optimization) -- Default hyperparameters -- WER: 0.20 -- Training time: 2 hours - -### After Random Search (20 trials) -- Optimized hyperparameters -- **WER: 0.14** (30% improvement) -- Training time: 1.5 hours (faster convergence) - -### After Bayesian Refinement (30 more trials) -- Fine-tuned hyperparameters -- **WER: 0.12** (40% improvement) -- Training time: 1.2 hours - -**ROI:** 50 trials ร— 2 hours = 100 GPU hours -**Result:** Permanent 40% performance boost for all future fine-tuning - ---- - -## ๐Ÿ”ง Implementation Guide - -### Step 1: Create Sweep Configuration - -```python -from src.data.wandb_sweeps import SweepConfig - -# For this STT project, start with: -sweep_config = SweepConfig.create_finetuning_sweep( - metric_name="validation/model_wer", # Optimize WER - goal="minimize", # Lower is better - method="random", # Start with random - num_trials=20 # 20 trials -) - -# Customize for your needs -sweep_config['parameters']['learning_rate'] = { - 'distribution': 'log_uniform_values', - 'min': 5e-6, # Lower bound based on your model - 'max': 5e-5 # Upper bound -} -``` - -### Step 2: Integrate with Training - -```python -from src.data.finetuning_orchestrator import FinetuningOrchestrator -from src.data.data_manager import DataManager -import wandb - -def train_with_sweep(): - """Training function compatible with W&B sweep.""" - - # Get hyperparameters from sweep - config = wandb.config - - # Initialize orchestrator - data_manager = DataManager(use_gcs=True) - orchestrator = FinetuningOrchestrator(data_manager) - - # Trigger fine-tuning - job = orchestrator.trigger_finetuning(force=True) - - # Train with sweep hyperparameters - training_params = { - 'learning_rate': config.learning_rate, - 'batch_size': config.batch_size, - 'epochs': config.epochs, - 'warmup_steps': config.warmup_steps, - 'weight_decay': config.weight_decay - } - - # Your actual training code here - result = run_actual_training(job, training_params) - - # Log final metrics (W&B uses these to find best run) - wandb.log({ - 'validation/model_wer': result['wer'], - 'validation/model_cer': result['cer'], - 'validation/wer_improvement': result['wer_improvement'] - }) -``` - -### Step 3: Launch Sweep - -```python -from src.data.wandb_sweeps import WandbSweepOrchestrator - -sweep_orch = WandbSweepOrchestrator(project_name="stt-optimization") - -# Create sweep -sweep_id = sweep_orch.create_sweep( - sweep_config, - sweep_name="initial_hyperparameter_optimization" -) - -# Run sweep (can run on multiple GPUs in parallel) -sweep_orch.run_sweep_agent(train_with_sweep, sweep_id, count=20) -``` - -### Step 4: Get and Use Best Config - -```python -# Get best hyperparameters -best = sweep_orch.get_best_run(sweep_id) - -print(f"Best WER: {best['metric_value']:.4f}") -print("Optimal hyperparameters:") -for param, value in best['hyperparameters'].items(): - print(f" {param}: {value}") - -# Save for production -sweep_orch.save_best_config('config/optimal_hyperparameters.json', sweep_id) - -# Use in all future fine-tuning -optimal_params = best['hyperparameters'] -orchestrator.start_training(job_id, training_params=optimal_params) -``` - ---- - -## ๐Ÿš€ Quick Start - -### Minimal Example (5 trials, ~2 hours) - -```python -from src.data.wandb_sweeps import WandbSweepOrchestrator, SweepConfig - -# 1. Create minimal sweep -sweep_orch = WandbSweepOrchestrator(project_name="quick-test") -config = SweepConfig.create_minimal_sweep(num_trials=5) -sweep_id = sweep_orch.create_sweep(config) - -# 2. Run sweep -sweep_orch.run_sweep_agent(your_train_function, sweep_id, count=5) - -# 3. Get best -best = sweep_orch.get_best_run(sweep_id) -print(f"Best config: {best['hyperparameters']}") -``` - ---- - -## ๐Ÿ“Š What You'll See in W&B Dashboard - -### Sweeps Overview Page - -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Sweep: initial_hyperparameter_optimization โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ Status: โœ… Completed (20/20 runs) โ”‚ -โ”‚ Best WER: 0.12 (Run: helpful-surf-42) โ”‚ -โ”‚ Improvement: 40% vs worst run โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ ๐Ÿ“Š Visualizations โ”‚ -โ”‚ โ€ข Parallel Coordinates Plot โ”‚ -โ”‚ โ””โ”€ Shows relationship between hyperparameters โ”‚ -โ”‚ and performance โ”‚ -โ”‚ โ€ข Parameter Importance โ”‚ -โ”‚ โ””โ”€ Which parameters matter most โ”‚ -โ”‚ โ€ข Optimization History โ”‚ -โ”‚ โ””โ”€ Performance improving over trials โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ ๐Ÿ† Best Configuration โ”‚ -โ”‚ โ€ข learning_rate: 3.2e-5 โ”‚ -โ”‚ โ€ข batch_size: 16 โ”‚ -โ”‚ โ€ข epochs: 10 โ”‚ -โ”‚ โ€ข warmup_steps: 500 โ”‚ -โ”‚ โ€ข weight_decay: 0.01 โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -### Parallel Coordinates Plot - -Shows how each hyperparameter affects WER: -- Lines colored by performance -- Best runs highlighted -- Easy to see patterns -- Interactive exploration - -### Parameter Importance - -Bar chart showing: -- Which parameters affect performance most -- Focus future optimization on these -- Ignore parameters with low impact - ---- - -## ๐ŸŽฏ Recommended Approach - -### For This STT Project: - -**1. Initial Optimization (One-Time, ~1-2 days)** - -```python -# Random search with 20-30 trials -sweep_config = SweepConfig.create_finetuning_sweep( - method='random', - num_trials=25 -) - -sweep_id = sweep_orch.create_sweep(sweep_config, "initial_optimization") -sweep_orch.run_sweep_agent(train_fn, sweep_id, count=25) - -# Find best hyperparameters -best = sweep_orch.get_best_run(sweep_id) -``` - -**Result:** Optimal hyperparameters for your specific error cases and data - ---- - -**2. Use Best Config in Production** - -```python -# Load optimized hyperparameters -with open('optimal_hyperparameters.json') as f: - optimal = json.load(f)['hyperparameters'] - -# Use for all automated fine-tuning -config = FinetuningConfig( - use_wandb=True, - # Use optimal hyperparameters -) - -# All future fine-tuning uses optimal config -coordinator = FinetuningCoordinator( - data_manager=data_manager, - finetuning_config=config -) -``` - ---- - -**3. Periodic Re-Optimization (Optional)** - -```python -# Every 6 months or after major data changes -# Run a small sweep to check if optimal config still works - -mini_sweep = SweepConfig.create_minimal_sweep(num_trials=5) -# Re-validate optimal hyperparameters -``` - ---- - -## ๐Ÿ’ฐ Cost-Benefit Analysis - -### Without Sweeps (Manual Tuning) - -- **Time:** 2-4 weeks of experimentation -- **GPU Hours:** 100-200 hours (trial and error) -- **Result:** Suboptimal configuration -- **Confidence:** Low (only tested a few combinations) - -### With Random Search (20 trials) - -- **Time:** 1-2 days automated -- **GPU Hours:** 40-60 hours (systematic) -- **Result:** Near-optimal configuration -- **Confidence:** High (tested 20 combinations) - -### ROI - -**Investment:** 40-60 GPU hours -**Return:** 20-40% better WER for EVERY future fine-tuning -**Lifetime Benefit:** If you fine-tune 10 times โ†’ 10x better performance - -**Verdict: Excellent ROI! ๐ŸŽฏ** - ---- - -## ๐Ÿ”ฅ Quick Implementation - -### Add to Your Workflow - -```python -# In your fine-tuning coordinator - -from src.data.wandb_sweeps import WandbSweepOrchestrator, SweepConfig - -class FinetuningCoordinator: - - def optimize_hyperparameters( - self, - num_trials: int = 20, - method: str = 'random' - ) -> Dict: - """ - Run hyperparameter optimization sweep. - - Returns best configuration found. - """ - # Create sweep orchestrator - sweep_orch = WandbSweepOrchestrator( - project_name=f"{self.wandb_project}_sweeps" - ) - - # Create sweep config - sweep_config = SweepConfig.create_finetuning_sweep( - method=method, - num_trials=num_trials - ) - - # Create sweep - sweep_id = sweep_orch.create_sweep(sweep_config) - - # Define training wrapper - def train(): - config = wandb.config - job = self.orchestrator.trigger_finetuning(force=True) - result = self.train_with_params(job, dict(config)) - wandb.log({'validation/model_wer': result['wer']}) - - # Run sweep - sweep_orch.run_sweep_agent(train, sweep_id, count=num_trials) - - # Get best - best = sweep_orch.get_best_run(sweep_id) - - # Save best config - self.optimal_hyperparameters = best['hyperparameters'] - - return best -``` - ---- - -## ๐ŸŽฏ Answer to Your Question - -### **Should you use W&B Random Search?** - -**YES! Here's the recommendation:** - -1. **Use Random Search (20 trials) for initial optimization** - - Quick and efficient - - Good results - - Low cost - -2. **Then use best hyperparameters for all automated fine-tuning** - - 20-40% better performance - - Faster convergence - - More stable training - -3. **Optionally refine with Bayesian (30 trials) if:** - - You have the GPU budget - - You want absolute best performance - - You're preparing for production deployment - -### Cost vs Benefit - -| Strategy | Trials | Time | GPU Cost | WER Improvement | Recommended | -|----------|--------|------|----------|-----------------|-------------| -| Manual | ~10 | 2 weeks | Low | Unknown | โŒ | -| Random Search | 20 | 1-2 days | Medium | 20-30% | โœ… YES | -| Bayesian | 50 | 3-4 days | High | 30-40% | โœ… If budget allows | -| Grid Search | 100+ | 1 week+ | Very High | 40%+ | โŒ Overkill | - -### For Your Project - -**Start with:** -- โœ… Random Search with 20 trials -- โœ… Optimize for `validation/model_wer` -- โœ… Focus on learning_rate, batch_size, epochs - -**Then:** -- โœ… Use best config for all future fine-tuning -- โœ… Re-optimize every 6 months or when data changes significantly - ---- - -## ๐Ÿš€ Get Started Now - -```bash -# 1. Run the demo -python experiments/demo_wandb_sweeps.py - -# 2. Try a quick 5-trial sweep -python -c " -from src.data.wandb_sweeps import WandbSweepOrchestrator, SweepConfig -sweep_orch = WandbSweepOrchestrator(project_name='quick-test') -config = SweepConfig.create_minimal_sweep(num_trials=5) -sweep_id = sweep_orch.create_sweep(config) -print(f'Sweep created: {sweep_id}') -print('Run: wandb agent {sweep_id}') -" - -# 3. View results -# Go to https://wandb.ai/ โ†’ Your Project โ†’ Sweeps -``` - ---- - -## ๐Ÿ“š Additional Resources - -- **W&B Sweeps Docs:** https://docs.wandb.ai/guides/sweeps -- **Best Practices:** https://wandb.ai/site/articles/bayesian-hyperparameter-optimization -- **Examples:** https://github.com/wandb/examples/tree/master/examples/keras/keras-cnn-fashion - ---- - -## โœ… Summary - -**Will this project benefit from W&B Random Search?** - -**ABSOLUTELY YES! ๐ŸŽฏ** - -**Benefits:** -- โœ… 20-40% better WER/CER -- โœ… Automated optimization -- โœ… Save weeks of manual tuning -- โœ… Reproducible results -- โœ… Confidence in hyperparameters - -**Recommendation:** -1. Run ONE random search sweep (20 trials) -2. Use best config for ALL future fine-tuning -3. ROI is excellent - pays for itself quickly - -**Get started:** -```bash -python experiments/demo_wandb_sweeps.py -``` - -**The system is already integrated and ready to use!** ๐Ÿš€ - diff --git a/experiments/comprehensive_test_suite.py b/experiments/comprehensive_test_suite.py deleted file mode 100644 index be360bc..0000000 --- a/experiments/comprehensive_test_suite.py +++ /dev/null @@ -1,365 +0,0 @@ -""" -Comprehensive Test Suite - Week 4 -Complete testing framework integrating all Week 4 components. -""" - -import sys -from pathlib import Path -import logging -import json -from datetime import datetime -from typing import List, Dict, Optional - -# Add src to path -sys.path.insert(0, str(Path(__file__).parent.parent)) - -from src.integration import UnifiedSTTSystem -from src.integration.end_to_end_testing import EndToEndTester -from src.integration.statistical_analysis import StatisticalAnalyzer -from src.integration.ablation_studies import AblationStudy -from src.baseline_model import BaselineSTTModel - -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') -logger = logging.getLogger(__name__) - - -class ComprehensiveTestSuite: - """ - Comprehensive test suite that integrates all testing components. - """ - - def __init__(self, output_dir: str = "experiments/test_outputs"): - """ - Initialize comprehensive test suite. - - Args: - output_dir: Directory to save test outputs - """ - self.output_dir = Path(output_dir) - self.output_dir.mkdir(parents=True, exist_ok=True) - self.test_results = {} - - def run_all_tests( - self, - audio_files: List[str], - reference_transcripts: List[str], - model_name: str = "whisper" - ) -> Dict: - """ - Run complete test suite. - - Args: - audio_files: List of audio file paths - reference_transcripts: List of reference transcripts - model_name: Model name to use - - Returns: - Dictionary with all test results - """ - logger.info("="*70) - logger.info("COMPREHENSIVE TEST SUITE - WEEK 4") - logger.info("="*70) - logger.info(f"Testing {len(audio_files)} audio files") - logger.info(f"Output directory: {self.output_dir}") - logger.info("") - - all_results = { - 'timestamp': datetime.now().isoformat(), - 'num_files': len(audio_files), - 'model_name': model_name - } - - # Test 1: System Integration - logger.info("\n" + "="*70) - logger.info("TEST 1: System Integration") - logger.info("="*70) - integration_results = self._test_system_integration(audio_files, reference_transcripts, model_name) - all_results['integration'] = integration_results - - # Test 2: End-to-End Testing - logger.info("\n" + "="*70) - logger.info("TEST 2: End-to-End Testing") - logger.info("="*70) - e2e_results = self._test_end_to_end(audio_files, reference_transcripts, model_name) - all_results['end_to_end'] = e2e_results - - # Test 3: Statistical Analysis - logger.info("\n" + "="*70) - logger.info("TEST 3: Statistical Analysis") - logger.info("="*70) - statistical_results = self._test_statistical_analysis(audio_files, reference_transcripts, model_name) - all_results['statistical_analysis'] = statistical_results - - # Test 4: Ablation Studies - logger.info("\n" + "="*70) - logger.info("TEST 4: Ablation Studies") - logger.info("="*70) - ablation_results = self._test_ablation_studies(audio_files, reference_transcripts, model_name) - all_results['ablation_studies'] = ablation_results - - # Generate comprehensive report - logger.info("\n" + "="*70) - logger.info("GENERATING COMPREHENSIVE REPORT") - logger.info("="*70) - report = self._generate_comprehensive_report(all_results) - - # Save results - results_file = self.output_dir / f"test_results_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" - with open(results_file, 'w') as f: - json.dump(all_results, f, indent=2, default=str) - logger.info(f"Results saved to: {results_file}") - - report_file = self.output_dir / f"test_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.txt" - with open(report_file, 'w') as f: - f.write(report) - logger.info(f"Report saved to: {report_file}") - - return all_results - - def _test_system_integration( - self, - audio_files: List[str], - reference_transcripts: List[str], - model_name: str - ) -> Dict: - """Test system integration.""" - logger.info("Initializing unified system...") - system = UnifiedSTTSystem( - model_name=model_name, - enable_error_detection=True, - enable_llm_correction=True, - enable_adaptive_fine_tuning=True - ) - - # Test system status - status = system.get_system_status() - - # Test batch evaluation - logger.info("Running batch evaluation...") - batch_results = system.evaluate_batch( - audio_files[:min(5, len(audio_files))], # Test on subset - reference_transcripts[:min(5, len(reference_transcripts))] - ) - - return { - 'system_status': status, - 'batch_evaluation': batch_results, - 'components_enabled': status['component_status'] - } - - def _test_end_to_end( - self, - audio_files: List[str], - reference_transcripts: List[str], - model_name: str - ) -> Dict: - """Test end-to-end functionality.""" - system = UnifiedSTTSystem(model_name=model_name) - tester = EndToEndTester(system) - - # Run full test suite - test_files = audio_files[:min(10, len(audio_files))] - test_refs = reference_transcripts[:min(10, len(reference_transcripts))] - - results = tester.run_full_test_suite(test_files, test_refs) - - return results - - def _test_statistical_analysis( - self, - audio_files: List[str], - reference_transcripts: List[str], - model_name: str - ) -> Dict: - """Test statistical analysis.""" - analyzer = StatisticalAnalyzer() - - # Create baseline and full system - baseline_system = UnifiedSTTSystem( - model_name=model_name, - enable_error_detection=False, - enable_llm_correction=False, - enable_adaptive_fine_tuning=False - ) - - full_system = UnifiedSTTSystem( - model_name=model_name, - enable_error_detection=True, - enable_llm_correction=True, - enable_adaptive_fine_tuning=True - ) - - # Evaluate both systems - test_files = audio_files[:min(20, len(audio_files))] - test_refs = reference_transcripts[:min(20, len(reference_transcripts))] - - baseline_scores = [] - full_system_scores = [] - - for audio_path, reference in zip(test_files, test_refs): - baseline_result = baseline_system.transcribe(audio_path, reference) - full_result = full_system.transcribe(audio_path, reference) - - if 'evaluation' in baseline_result: - baseline_scores.append(baseline_result['evaluation']['wer']) - if 'evaluation' in full_result: - full_system_scores.append(full_result['evaluation']['wer']) - - # Perform paired t-test - if len(baseline_scores) == len(full_system_scores) and len(baseline_scores) > 0: - comparison = analyzer.compare_systems( - baseline_scores, - full_system_scores, - "Baseline System", - "Full System" - ) - - return { - 'baseline_scores': baseline_scores, - 'full_system_scores': full_system_scores, - 'statistical_comparison': comparison - } - else: - return {'status': 'insufficient_data'} - - def _test_ablation_studies( - self, - audio_files: List[str], - reference_transcripts: List[str], - model_name: str - ) -> Dict: - """Test ablation studies.""" - study = AblationStudy() - - # Run ablation study on subset - test_files = audio_files[:min(15, len(audio_files))] - test_refs = reference_transcripts[:min(15, len(reference_transcripts))] - - results = study.run_ablation_study(test_files, test_refs, model_name) - - # Generate report - report = study.generate_ablation_report(results) - - return { - 'ablation_results': results, - 'report': report - } - - def _generate_comprehensive_report(self, all_results: Dict) -> str: - """Generate comprehensive test report.""" - report_lines = [] - report_lines.append("="*70) - report_lines.append("COMPREHENSIVE TEST SUITE REPORT - WEEK 4") - report_lines.append("="*70) - report_lines.append(f"Generated: {all_results.get('timestamp', 'Unknown')}") - report_lines.append(f"Files Tested: {all_results.get('num_files', 0)}") - report_lines.append("") - - # Integration Results - if 'integration' in all_results: - report_lines.append("1. SYSTEM INTEGRATION") - report_lines.append("-"*70) - integration = all_results['integration'] - components = integration.get('components_enabled', {}) - for component, enabled in components.items(): - status = "โœ…" if enabled else "โŒ" - report_lines.append(f" {status} {component}: {enabled}") - report_lines.append("") - - # End-to-End Results - if 'end_to_end' in all_results: - report_lines.append("2. END-TO-END TESTING") - report_lines.append("-"*70) - e2e = all_results['end_to_end'] - results = e2e.get('results', {}) - if 'feedback_loop' in results: - fb = results['feedback_loop'] - summary = fb.get('summary', {}) - report_lines.append(f" Feedback Loop Iterations: {summary.get('num_iterations', 0)}") - report_lines.append(f" Total Errors Detected: {summary.get('total_errors_detected', 0)}") - report_lines.append(f" Total Corrections Applied: {summary.get('total_corrections_applied', 0)}") - report_lines.append("") - - # Statistical Analysis - if 'statistical_analysis' in all_results: - report_lines.append("3. STATISTICAL ANALYSIS") - report_lines.append("-"*70) - stats = all_results['statistical_analysis'] - if 'statistical_comparison' in stats: - comp = stats['statistical_comparison'] - report_lines.append(f" Baseline Mean WER: {comp.get('mean_baseline', 0):.4f}") - report_lines.append(f" Full System Mean WER: {comp.get('mean_treatment', 0):.4f}") - report_lines.append(f" Improvement: {comp.get('mean_difference', 0):.4f}") - report_lines.append(f" p-value: {comp.get('p_value', 0):.4f}") - report_lines.append(f" Significant: {comp.get('is_significant', False)}") - report_lines.append("") - - # Ablation Studies - if 'ablation_studies' in all_results: - report_lines.append("4. ABLATION STUDIES") - report_lines.append("-"*70) - ablation = all_results['ablation_studies'] - if 'ablation_results' in ablation: - results = ablation['ablation_results'] - summary = results.get('summary', {}) - report_lines.append(f" Baseline WER: {summary.get('baseline_performance', 0):.4f}") - report_lines.append(f" Full System WER: {summary.get('full_system_performance', 0):.4f}") - report_lines.append(f" Overall Improvement: {summary.get('overall_improvement', 0):.4f}") - - contributions = summary.get('component_contributions', {}) - if contributions: - report_lines.append(" Component Contributions:") - for component, contrib in contributions.items(): - sig = "โœ…" if contrib.get('is_significant', False) else "โŒ" - report_lines.append(f" {sig} {component}: {contrib.get('improvement', 0):.4f}") - report_lines.append("") - - report_lines.append("="*70) - report_lines.append("END OF REPORT") - report_lines.append("="*70) - - return "\n".join(report_lines) - - -def main(): - """Main function to run comprehensive test suite.""" - import argparse - - parser = argparse.ArgumentParser(description='Run comprehensive test suite') - parser.add_argument('--audio-dir', type=str, help='Directory containing audio files') - parser.add_argument('--references', type=str, help='JSON file with reference transcripts') - parser.add_argument('--output-dir', type=str, default='experiments/test_outputs', - help='Output directory for test results') - parser.add_argument('--model', type=str, default='whisper', help='Model name') - - args = parser.parse_args() - - # For demonstration, use test audio if available - test_audio_dir = Path("src/test_audio") - audio_files = [] - reference_transcripts = [] - - if test_audio_dir.exists(): - audio_files = list(test_audio_dir.glob("*.wav"))[:10] # Limit for testing - # Create dummy references for testing - reference_transcripts = ["Test transcript"] * len(audio_files) - - if not audio_files: - logger.warning("No audio files found. Please provide audio files for testing.") - logger.info("Usage: python comprehensive_test_suite.py --audio-dir --references ") - return - - # Run test suite - suite = ComprehensiveTestSuite(output_dir=args.output_dir) - results = suite.run_all_tests( - audio_files=[str(f) for f in audio_files], - reference_transcripts=reference_transcripts, - model_name=args.model - ) - - logger.info("\nโœ… Comprehensive test suite completed!") - logger.info(f"Results saved to: {args.output_dir}") - - -if __name__ == "__main__": - main() diff --git a/experiments/create_test_evaluation.py b/experiments/create_test_evaluation.py deleted file mode 100644 index dce393d..0000000 --- a/experiments/create_test_evaluation.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -Create a simple test evaluation dataset with ground truth for demonstration. -""" - -import sys -from pathlib import Path -from datasets import Dataset, DatasetDict -import soundfile as sf -import librosa - -sys.path.append(str(Path(__file__).parent.parent)) - -def create_test_dataset(): - """Create a simple test dataset with audio files and ground truth.""" - - # Ground truth transcripts (from known test files) - test_data = [ - { - "audio_path": "data/test_audio/addf8-Alaw-GW.wav", - "reference": "add the sum to the product of these three", - "id": "test_001" - }, - { - "audio_path": "data/test_audio/test_1.wav", - "reference": "you", # Simple test audio - "id": "test_002" - } - ] - - # Load audio files and create dataset - audio_data = [] - references = [] - ids = [] - - for item in test_data: - audio_path = Path(item["audio_path"]) - if audio_path.exists(): - # Load audio - audio, sr = librosa.load(str(audio_path), sr=16000) - audio_data.append({"array": audio, "sampling_rate": sr}) - references.append(item["reference"]) - ids.append(item["id"]) - - # Create dataset - dataset = Dataset.from_dict({ - "audio": audio_data, - "text": references, - "id": ids - }) - - # Create splits (all test for now) - dataset_dict = DatasetDict({ - "test": dataset - }) - - # Save dataset - output_path = Path("data/evaluation/test_dataset") - output_path.mkdir(parents=True, exist_ok=True) - dataset_dict.save_to_disk(str(output_path)) - - print(f"โœ… Created test dataset with {len(dataset)} samples at {output_path}") - return str(output_path) - -if __name__ == "__main__": - create_test_dataset() - diff --git a/experiments/demo_finetuning_orchestration.py b/experiments/demo_finetuning_orchestration.py deleted file mode 100644 index 8ab9ba1..0000000 --- a/experiments/demo_finetuning_orchestration.py +++ /dev/null @@ -1,427 +0,0 @@ -#!/usr/bin/env python3 -""" -Comprehensive Demo: Fine-Tuning Orchestration System - -Demonstrates the complete fine-tuning lifecycle: -1. Automated fine-tuning trigger based on error accumulation -2. Model validation against baseline -3. Model versioning and deployment -4. Regression testing to prevent degradation - -Usage: - python experiments/demo_finetuning_orchestration.py -""" - -import sys -from pathlib import Path - -# Add src to path -sys.path.insert(0, str(Path(__file__).parent.parent)) - -import json -import logging -from datetime import datetime - -from src.data.data_manager import DataManager -from src.data.finetuning_coordinator import FinetuningCoordinator -from src.data.finetuning_orchestrator import FinetuningConfig -from src.data.model_validator import ValidationConfig -from src.data.model_deployer import DeploymentConfig -from src.data.regression_tester import RegressionConfig -from src.baseline_model import BaselineSTTModel - -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' -) -logger = logging.getLogger(__name__) - - -def print_section(title): - """Print section header.""" - print("\n" + "="*80) - print(f" {title}") - print("="*80 + "\n") - - -def demo_data_manager(): - """Demo 1: Data Manager - Track error cases.""" - print_section("DEMO 1: Data Manager - Tracking Error Cases") - - # Initialize data manager - data_manager = DataManager( - local_storage_dir="data/test_demo", - use_gcs=False # Set to True for GCS integration - ) - - print("๐Ÿ“Š Simulating error case accumulation...") - - # Simulate storing failed cases - for i in range(5): - case_id = data_manager.store_failed_case( - audio_path=f"data/test_audio/test_{i}.wav", - original_transcript=f"This is test transcription number {i}", - corrected_transcript=f"This is test transcription number {i} (corrected)", - error_types=["word_substitution", "missing_word"], - error_score=0.75, - metadata={"model": "baseline", "timestamp": datetime.now().isoformat()} - ) - print(f" Stored case {i+1}: {case_id}") - - # Get statistics - stats = data_manager.get_statistics() - print(f"\n๐Ÿ“ˆ Statistics:") - print(f" Total error cases: {stats['total_failed_cases']}") - print(f" Corrected cases: {stats['corrected_cases']}") - print(f" Correction rate: {stats['correction_rate']:.1%}") - - return data_manager - - -def demo_orchestrator(data_manager): - """Demo 2: Fine-Tuning Orchestrator - Automated triggering.""" - print_section("DEMO 2: Fine-Tuning Orchestrator - Automated Triggering") - - # Configure orchestrator - config = FinetuningConfig( - min_error_cases=3, # Low threshold for demo - min_corrected_cases=2, - auto_approve_finetuning=True - ) - - from src.data.finetuning_orchestrator import FinetuningOrchestrator - - orchestrator = FinetuningOrchestrator( - data_manager=data_manager, - config=config, - use_gcs=False - ) - - print("๐Ÿ” Checking trigger conditions...") - trigger_result = orchestrator.check_trigger_conditions() - - if trigger_result['should_trigger']: - print("โœ… Trigger conditions met!") - print(f" Reasons: {', '.join(trigger_result['reasons'])}") - - # Trigger fine-tuning - print("\n๐Ÿš€ Triggering fine-tuning job...") - job = orchestrator.trigger_finetuning(force=True) - - if job: - print(f"โœ… Job created: {job.job_id}") - print(f" Status: {job.status}") - print(f" Dataset: {job.dataset_id}") - - # Get job info - job_info = orchestrator.get_job_info(job.job_id) - print(f"\n๐Ÿ“‹ Job Details:") - print(f" Training samples: {job_info.get('dataset_info', {}).get('split_sizes', {}).get('train', 'N/A')}") - print(f" Validation samples: {job_info.get('dataset_info', {}).get('split_sizes', {}).get('val', 'N/A')}") - - return job - else: - print("โธ๏ธ Trigger conditions not met") - print(f" Need {config.min_error_cases - trigger_result['metrics']['total_error_cases']} more cases") - - return None - - -def demo_validator(): - """Demo 3: Model Validator - Validate against baseline.""" - print_section("DEMO 3: Model Validator - Validation Against Baseline") - - config = ValidationConfig( - min_wer_improvement=0.0, # Allow any improvement - require_significance=False # Skip significance test for demo - ) - - from src.data.model_validator import ModelValidator - - validator = ModelValidator( - config=config, - use_gcs=False - ) - - print("๐Ÿ“ Creating mock evaluation data...") - - # Create simple mock data - eval_data = [ - { - 'audio_path': f'data/test_audio/test_{i}.wav', - 'reference': f'This is test audio number {i}' - } - for i in range(3) - ] - - print(f" Evaluation samples: {len(eval_data)}") - - # Mock transcription functions - def baseline_transcribe(audio_path): - """Mock baseline transcription.""" - return "This is test audio number 0" # Simplified - - def model_transcribe(audio_path): - """Mock fine-tuned model transcription.""" - return "This is test audio number 0" # Same for demo - - print("\nโš–๏ธ Running validation...") - print(" (Using mock transcription functions for demo)") - - try: - result = validator.validate_model( - model_id="demo_finetuned_v1", - model_transcribe_fn=model_transcribe, - baseline_id="baseline_v1", - baseline_transcribe_fn=baseline_transcribe, - evaluation_set=eval_data - ) - - print(f"\n๐Ÿ“Š Validation Result:") - print(f" Passed: {'Yes โœ…' if result.passed else 'No โŒ'}") - print(f" Model WER: {result.model_wer:.4f}") - print(f" Baseline WER: {result.baseline_wer:.4f}") - print(f" Improvement: {result.wer_improvement:+.4f}") - - return result - except Exception as e: - print(f"โš ๏ธ Validation demo skipped: {e}") - return None - - -def demo_deployer(): - """Demo 4: Model Deployer - Version management and deployment.""" - print_section("DEMO 4: Model Deployer - Version Management") - - config = DeploymentConfig( - keep_previous_versions=3, - auto_backup_before_deploy=True - ) - - from src.data.model_deployer import ModelDeployer - - deployer = ModelDeployer( - config=config, - storage_dir="data/test_deployed_models", - use_gcs=False - ) - - print("๐Ÿ“ฆ Registering mock model versions...") - - # Register some mock versions - for i in range(3): - version_id = deployer.register_model( - model_name=f"test-model-v{i+1}", - model_path=f"/tmp/mock_model_v{i+1}", - validation_result={ - 'passed': True, - 'model_wer': 0.15 - (i * 0.02), # Improving - 'model_cer': 0.08 - (i * 0.01) - } - ) - print(f" โœ“ Registered: {version_id}") - - # List versions - print("\n๐Ÿ“‹ Registered Versions:") - for version in deployer.list_versions(limit=5): - print(f" - {version.version_id}: WER={version.wer:.4f}" if version.wer else f" - {version.version_id}") - - # Show deployment status - deployer.print_status() - - return deployer - - -def demo_regression_tester(): - """Demo 5: Regression Tester - Prevent degradation.""" - print_section("DEMO 5: Regression Tester - Preventing Degradation") - - config = RegressionConfig( - fail_on_any_degradation=False, - max_failed_samples_rate=0.1 - ) - - from src.data.regression_tester import RegressionTester - - tester = RegressionTester( - config=config, - storage_dir="data/test_regression", - use_gcs=False - ) - - print("๐Ÿ“ Registering regression tests...") - - # Register a test - test_id = tester.register_test( - test_name="Core Benchmark Test", - test_type="benchmark", - test_data_path="data/test_regression_samples.jsonl", - baseline_wer=0.15, - baseline_cer=0.08, - baseline_version="baseline_v1", - max_wer_degradation=0.05, - description="Critical benchmark that should not degrade" - ) - - print(f" โœ“ Registered test: {test_id}") - - print(f"\n๐Ÿ“Š Registered Tests: {len(tester.tests)}") - for tid, test in tester.tests.items(): - print(f" - {test.test_name} ({test.test_type})") - print(f" Baseline WER: {test.baseline_wer:.4f}") - print(f" Max degradation: {test.max_wer_degradation:.4f}") - - return tester - - -def demo_coordinator(data_manager): - """Demo 6: Full Coordinator - Complete workflow.""" - print_section("DEMO 6: Fine-Tuning Coordinator - Complete Workflow") - - # Initialize coordinator with all components - coordinator = FinetuningCoordinator( - data_manager=data_manager, - finetuning_config=FinetuningConfig( - min_error_cases=3, - auto_approve_finetuning=True - ), - validation_config=ValidationConfig( - min_wer_improvement=0.0, - require_significance=False - ), - deployment_config=DeploymentConfig( - keep_previous_versions=3 - ), - regression_config=RegressionConfig( - fail_on_any_degradation=False - ), - use_gcs=False, - storage_dir="data/test_orchestration" - ) - - # Show system status - coordinator.print_status() - - print("\n๐Ÿ“‹ System Configuration:") - print(f" Auto-trigger: {coordinator.orchestrator.config.auto_approve_finetuning}") - print(f" Min error cases: {coordinator.orchestrator.config.min_error_cases}") - print(f" Validation required: Yes") - print(f" Regression testing: Yes") - - # Check if ready to trigger - trigger_result = coordinator.orchestrator.check_trigger_conditions() - - if trigger_result['should_trigger']: - print("\nโœ… System is ready to trigger fine-tuning!") - print("\n๐Ÿ’ก To run complete workflow:") - print(" coordinator.run_complete_workflow(force_trigger=True, auto_deploy=True)") - else: - print("\nโธ๏ธ Waiting for more error cases...") - - return coordinator - - -def demo_complete_workflow(): - """Demo 7: Complete End-to-End Workflow.""" - print_section("DEMO 7: Complete End-to-End Workflow (Simulated)") - - print("This would demonstrate the complete workflow:") - print("\n1๏ธโƒฃ Monitor error cases") - print(" โ””โ”€ Accumulate failed transcriptions") - print(" โ””โ”€ Track corrections") - - print("\n2๏ธโƒฃ Trigger fine-tuning (when threshold met)") - print(" โ””โ”€ Prepare training dataset") - print(" โ””โ”€ Create data version") - print(" โ””โ”€ Launch training job") - - print("\n3๏ธโƒฃ Validate trained model") - print(" โ””โ”€ Run on standardized evaluation set") - print(" โ””โ”€ Compare against baseline") - print(" โ””โ”€ Check statistical significance") - - print("\n4๏ธโƒฃ Run regression tests") - print(" โ””โ”€ Test on critical samples") - print(" โ””โ”€ Check for degradation") - print(" โ””โ”€ Verify edge cases") - - print("\n5๏ธโƒฃ Deploy model (if validation passes)") - print(" โ””โ”€ Register model version") - print(" โ””โ”€ Backup current model") - print(" โ””โ”€ Deploy new version") - print(" โ””โ”€ Update active model pointer") - - print("\n6๏ธโƒฃ Continuous monitoring") - print(" โ””โ”€ Track performance metrics") - print(" โ””โ”€ Alert on degradation") - print(" โ””โ”€ Enable rollback if needed") - - print("\n๐Ÿ’ก For production use:") - print(" - Set use_gcs=True for cloud storage") - print(" - Configure actual training callbacks") - print(" - Set up monitoring and alerting") - print(" - Use real evaluation datasets") - - -def main(): - """Run all demos.""" - print("\n" + "="*80) - print(" FINE-TUNING ORCHESTRATION SYSTEM - COMPREHENSIVE DEMO") - print("="*80) - print("\nThis demo showcases the complete fine-tuning orchestration system:") - print(" 1. Data Management") - print(" 2. Automated Triggering") - print(" 3. Model Validation") - print(" 4. Model Deployment") - print(" 5. Regression Testing") - print(" 6. Complete Workflow Coordination") - - try: - # Run demos - data_manager = demo_data_manager() - job = demo_orchestrator(data_manager) - validation_result = demo_validator() - deployer = demo_deployer() - tester = demo_regression_tester() - coordinator = demo_coordinator(data_manager) - demo_complete_workflow() - - # Final summary - print_section("DEMO COMPLETE - Summary") - - print("โœ… Successfully demonstrated:") - print(" โœ“ Data Manager: Error case tracking") - print(" โœ“ Orchestrator: Automated fine-tuning triggers") - print(" โœ“ Validator: Model validation against baseline") - print(" โœ“ Deployer: Version management and deployment") - print(" โœ“ Regression Tester: Degradation prevention") - print(" โœ“ Coordinator: Complete workflow orchestration") - - print("\n๐Ÿ“š Next Steps:") - print(" 1. Review the generated data in data/test_* directories") - print(" 2. Check src/data/ for implementation details") - print(" 3. See docs/ for comprehensive documentation") - print(" 4. Configure for production with GCS integration") - print(" 5. Set up training callbacks for actual model training") - - print("\n๐Ÿš€ Ready for Production:") - print(" - Enable GCS: use_gcs=True") - print(" - Configure training: set_training_callback()") - print(" - Set up validation: set_baseline_transcribe_function()") - print(" - Deploy to GCP: python scripts/deploy_finetuning_to_gcp.py") - - print("\n" + "="*80) - print(" Demo completed successfully!") - print("="*80 + "\n") - - except Exception as e: - logger.error(f"Demo failed: {e}", exc_info=True) - print(f"\nโŒ Demo failed with error: {e}") - print(" Check logs for details") - sys.exit(1) - - -if __name__ == "__main__": - main() - - diff --git a/experiments/demo_wandb_sweeps.py b/experiments/demo_wandb_sweeps.py deleted file mode 100644 index a5edb90..0000000 --- a/experiments/demo_wandb_sweeps.py +++ /dev/null @@ -1,415 +0,0 @@ -#!/usr/bin/env python3 -""" -Demo: W&B Sweeps for Hyperparameter Optimization - -Shows how to use W&B Sweeps to automatically find the best hyperparameters -for fine-tuning your STT model. - -Usage: - pip install wandb - wandb login - python experiments/demo_wandb_sweeps.py -""" - -import sys -from pathlib import Path -sys.path.insert(0, str(Path(__file__).parent.parent)) - -import logging -import time -import random - -from src.data.wandb_sweeps import ( - WandbSweepOrchestrator, - SweepConfig, - create_sweep_training_wrapper -) - -try: - import wandb - WANDB_AVAILABLE = True -except ImportError: - WANDB_AVAILABLE = False - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -def print_section(title): - """Print section header.""" - print("\n" + "="*80) - print(f" {title}") - print("="*80 + "\n") - - -def demo_mock_training(): - """Mock training function for demonstration.""" - # Get hyperparameters from W&B sweep - config = wandb.config if WANDB_AVAILABLE else {} - - learning_rate = config.get('learning_rate', 1e-5) - batch_size = config.get('batch_size', 16) - epochs = config.get('epochs', 5) - - logger.info(f"Training with lr={learning_rate}, batch_size={batch_size}, epochs={epochs}") - - # Simulate training - for epoch in range(1, min(epochs, 3) + 1): # Quick demo - # Simulate metrics (better with lower learning rate in this mock) - train_loss = 0.5 / epoch + (learning_rate * 1000) - val_loss = train_loss * 1.2 - - # Mock WER (better with optimal hyperparameters) - wer = 0.20 - (0.02 * epoch) + abs(learning_rate - 3e-5) * 1000 - cer = wer * 0.6 - - if WANDB_AVAILABLE: - wandb.log({ - 'epoch': epoch, - 'train/loss': train_loss, - 'val/loss': val_loss, - 'validation/model_wer': wer, - 'validation/model_cer': cer - }) - - time.sleep(0.5) # Simulate training time - - # Final metrics - final_wer = wer - final_cer = cer - - if WANDB_AVAILABLE: - wandb.log({ - 'validation/model_wer': final_wer, - 'validation/model_cer': final_cer, - 'validation/wer_improvement': 0.20 - final_wer - }) - - logger.info(f"Final WER: {final_wer:.4f}, CER: {final_cer:.4f}") - - return {'wer': final_wer, 'cer': final_cer} - - -def demo_1_create_sweep_config(): - """Demo 1: Creating different sweep configurations.""" - print_section("DEMO 1: Creating Sweep Configurations") - - print("๐Ÿ“ Creating default fine-tuning sweep config...") - config1 = SweepConfig.create_finetuning_sweep( - metric_name="validation/model_wer", - goal="minimize", - method="random", - num_trials=10 - ) - - print(f" Method: {config1['method']}") - print(f" Metric: {config1['metric']['name']} (goal: {config1['metric']['goal']})") - print(f" Parameters optimized: {len(config1['parameters'])}") - print(f" - Learning rate: log-uniform") - print(f" - Batch size: {config1['parameters']['batch_size']['values']}") - print(f" - Epochs: {config1['parameters']['epochs']['values']}") - - print("\n๐Ÿ“ Creating minimal sweep config (for quick testing)...") - config2 = SweepConfig.create_minimal_sweep(num_trials=5) - - print(f" Parameters optimized: {len(config2['parameters'])}") - print(f" - Learning rate: {config2['parameters']['learning_rate']['values']}") - print(f" - Batch size: {config2['parameters']['batch_size']['values']}") - print(f" - Epochs: {config2['parameters']['epochs']['values']}") - - print("\n๐Ÿ“ Creating custom sweep config...") - custom_params = { - 'learning_rate': {'values': [1e-5, 5e-5]}, - 'batch_size': {'values': [16, 32]}, - 'model_type': {'values': ['whisper-tiny', 'whisper-base']} - } - config3 = SweepConfig.create_custom_sweep( - parameters=custom_params, - metric_name="validation/model_wer", - goal="minimize", - method="grid" # Grid search for exhaustive testing - ) - - print(f" Method: {config3['method']} (exhaustive search)") - print(f" Custom parameters: {list(config3['parameters'].keys())}") - - print("\nโœ… Sweep configurations demo completed") - return config2 # Return minimal config for next demo - - -def demo_2_create_sweep(sweep_config): - """Demo 2: Creating a sweep on W&B.""" - print_section("DEMO 2: Creating a W&B Sweep") - - if not WANDB_AVAILABLE: - print("โš ๏ธ W&B not available. Skipping this demo.") - print(" Install with: pip install wandb && wandb login") - return None - - print("๐Ÿš€ Initializing sweep orchestrator...") - orchestrator = WandbSweepOrchestrator( - project_name="stt-sweeps-demo", - enabled=True - ) - - print("\n๐Ÿ“Š Creating sweep on W&B...") - sweep_id = orchestrator.create_sweep( - sweep_config=sweep_config, - sweep_name="demo_hyperparameter_optimization" - ) - - if sweep_id: - print(f" โœ… Sweep created: {sweep_id}") - print(f" View at: https://wandb.ai/") - return sweep_id - else: - print(" โŒ Failed to create sweep") - return None - - -def demo_3_run_sweep(sweep_id): - """Demo 3: Running sweep trials.""" - print_section("DEMO 3: Running Sweep Trials") - - if not WANDB_AVAILABLE or not sweep_id: - print("โš ๏ธ Skipping - W&B not available or no sweep ID") - return - - print("๐Ÿ”„ Running sweep agent (3 trials for demo)...") - print(" Each trial tests different hyperparameter combinations") - print(" W&B automatically tracks all metrics and compares runs") - - orchestrator = WandbSweepOrchestrator( - project_name="stt-sweeps-demo", - enabled=True - ) - - # Run limited trials for demo - orchestrator.run_sweep_agent( - train_function=demo_mock_training, - sweep_id=sweep_id, - count=3 # Just 3 trials for demo - ) - - print("\nโœ… Sweep trials completed") - print(" Check W&B dashboard to see:") - print(" - Parallel coordinate plot of hyperparameters") - print(" - Metric comparison across runs") - print(" - Best performing configuration") - - -def demo_4_get_best_config(sweep_id): - """Demo 4: Getting best hyperparameters.""" - print_section("DEMO 4: Getting Best Hyperparameters") - - if not WANDB_AVAILABLE or not sweep_id: - print("โš ๏ธ Skipping - W&B not available or no sweep ID") - return None - - print("๐Ÿ” Analyzing sweep results...") - - orchestrator = WandbSweepOrchestrator( - project_name="stt-sweeps-demo", - enabled=True - ) - - # Give W&B a moment to process results - time.sleep(2) - - best_config = orchestrator.get_best_run( - sweep_id=sweep_id, - metric_name="validation/model_wer", - minimize=True - ) - - if best_config: - print(f"\nโœ… Best configuration found:") - print(f" Run: {best_config['run_name']}") - print(f" Best WER: {best_config['metric_value']:.4f}") - print(f"\n Optimal Hyperparameters:") - for param, value in best_config['hyperparameters'].items(): - print(f" - {param}: {value}") - - # Save to file - output_path = "experiments/best_hyperparameters.json" - orchestrator.save_best_config(output_path, sweep_id) - print(f"\n ๐Ÿ’พ Saved to: {output_path}") - - return best_config - else: - print(" โš ๏ธ Could not retrieve best configuration") - return None - - -def demo_5_sweep_strategies(): - """Demo 5: Different sweep strategies.""" - print_section("DEMO 5: Sweep Strategy Comparison") - - print("๐Ÿ“Š Available sweep strategies:\n") - - print("1๏ธโƒฃ RANDOM SEARCH") - print(" - Fast and efficient") - print(" - Good for initial exploration") - print(" - Recommended starting point") - print(" - Works well with 20-50 trials") - - print("\n2๏ธโƒฃ BAYESIAN OPTIMIZATION") - print(" - Smart, adaptive search") - print(" - Learns from previous trials") - print(" - More efficient than random") - print(" - Best for expensive training") - print(" - Works well with 30-100 trials") - - print("\n3๏ธโƒฃ GRID SEARCH") - print(" - Exhaustive search") - print(" - Tests all combinations") - print(" - Most thorough but slowest") - print(" - Good for final tuning") - print(" - Use when you have specific values to test") - - print("\n๐Ÿ’ก Recommendations for this project:") - print(" - Start with RANDOM (10-20 trials)") - print(" - Refine with BAYESIAN (20-30 trials)") - print(" - Final tune with GRID (if needed)") - - # Show config examples - print("\n๐Ÿ“ Example configurations:") - - print("\n Random Search (recommended):") - print(" ```python") - print(" config = SweepConfig.create_finetuning_sweep(") - print(" method='random',") - print(" num_trials=20") - print(" )") - print(" ```") - - print("\n Bayesian Optimization:") - print(" ```python") - print(" config = SweepConfig.create_finetuning_sweep(") - print(" method='bayes',") - print(" num_trials=30") - print(" )") - print(" ```") - - -def demo_6_integration_example(): - """Demo 6: Integration with fine-tuning pipeline.""" - print_section("DEMO 6: Integration with Fine-Tuning Pipeline") - - print("๐Ÿ”— How to integrate sweeps with your fine-tuning:") - - print("\n1๏ธโƒฃ Setup (one-time):") - print("```python") - print("from src.data.wandb_sweeps import WandbSweepOrchestrator, SweepConfig") - print("") - print("# Create sweep") - print("sweep_orch = WandbSweepOrchestrator(project_name='my-project')") - print("sweep_config = SweepConfig.create_finetuning_sweep(method='random')") - print("sweep_id = sweep_orch.create_sweep(sweep_config)") - print("```") - - print("\n2๏ธโƒฃ Define training function:") - print("```python") - print("def train():") - print(" # Get hyperparameters from sweep") - print(" config = wandb.config") - print(" ") - print(" # Use them in training") - print(" job = orchestrator.trigger_finetuning()") - print(" result = train_model(") - print(" learning_rate=config.learning_rate,") - print(" batch_size=config.batch_size") - print(" )") - print(" ") - print(" # Log metrics") - print(" wandb.log({'validation/model_wer': result['wer']})") - print("```") - - print("\n3๏ธโƒฃ Run sweep:") - print("```python") - print("# Run 20 trials") - print("sweep_orch.run_sweep_agent(train, sweep_id, count=20)") - print("") - print("# Get best hyperparameters") - print("best = sweep_orch.get_best_run(sweep_id)") - print("") - print("# Use best config for production") - print("production_config = best['hyperparameters']") - print("```") - - print("\n๐Ÿ’ก Benefits:") - print(" โœ… Automatically finds optimal hyperparameters") - print(" โœ… Saves hours of manual tuning") - print(" โœ… Improves model performance") - print(" โœ… Tracks all experiments") - print(" โœ… Reproducible results") - - -def main(): - """Run all demos.""" - print("\n" + "="*80) - print(" W&B SWEEPS - HYPERPARAMETER OPTIMIZATION DEMO") - print("="*80) - print("\nAutomatic hyperparameter tuning for fine-tuning!") - print("\nTopics covered:") - print(" 1. Creating sweep configurations") - print(" 2. Launching sweeps on W&B") - print(" 3. Running sweep trials") - print(" 4. Getting best hyperparameters") - print(" 5. Sweep strategy comparison") - print(" 6. Integration with fine-tuning") - - try: - # Demo 1: Create configs - sweep_config = demo_1_create_sweep_config() - - # Demo 2-4: Run actual sweep (if W&B available) - sweep_id = demo_2_create_sweep(sweep_config) - if sweep_id: - demo_3_run_sweep(sweep_id) - best_config = demo_4_get_best_config(sweep_id) - - # Demo 5-6: Educational content - demo_5_sweep_strategies() - demo_6_integration_example() - - # Summary - print_section("DEMO COMPLETE - Summary") - - print("โœ… Successfully demonstrated:") - print(" โœ“ Sweep configuration creation") - print(" โœ“ Different optimization strategies") - print(" โœ“ Best hyperparameter extraction") - print(" โœ“ Integration patterns") - - print("\n๐ŸŽฏ Next Steps:") - print(" 1. Create your sweep config") - print(" 2. Integrate with your training function") - print(" 3. Run sweep with 10-20 trials") - print(" 4. Use best hyperparameters in production") - - print("\n๐Ÿ“Š Expected Improvements:") - print(" โ€ข 10-30% better WER/CER") - print(" โ€ข Faster convergence") - print(" โ€ข More stable training") - print(" โ€ข Optimal resource usage") - - print("\n๐Ÿ’ก View Results:") - print(" https://wandb.ai/ โ†’ Your Project โ†’ Sweeps Tab") - - print("\n" + "="*80) - print(" Demo completed successfully!") - print("="*80 + "\n") - - except KeyboardInterrupt: - print("\n\nโš ๏ธ Demo interrupted by user") - except Exception as e: - logger.error(f"Demo failed: {e}", exc_info=True) - print(f"\nโŒ Demo failed: {e}") - print(" Make sure wandb is installed: pip install wandb") - print(" And you're logged in: wandb login") - - -if __name__ == "__main__": - main() - diff --git a/experiments/demo_wandb_tracking.py b/experiments/demo_wandb_tracking.py deleted file mode 100644 index ef966c3..0000000 --- a/experiments/demo_wandb_tracking.py +++ /dev/null @@ -1,370 +0,0 @@ -#!/usr/bin/env python3 -""" -Demo: Weights & Biases Integration for Fine-Tuning - -Demonstrates how W&B tracks metrics, creates visualizations, and logs experiments. - -Usage: - # Make sure wandb is installed and logged in - pip install wandb - wandb login - - # Run demo - python experiments/demo_wandb_tracking.py -""" - -import sys -from pathlib import Path - -# Add src to path -sys.path.insert(0, str(Path(__file__).parent.parent)) - -import logging -from datetime import datetime - -from src.data.data_manager import DataManager -from src.data.finetuning_orchestrator import FinetuningOrchestrator, FinetuningConfig -from src.data.wandb_tracker import WandbTracker - -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' -) -logger = logging.getLogger(__name__) - - -def print_section(title): - """Print section header.""" - print("\n" + "="*80) - print(f" {title}") - print("="*80 + "\n") - - -def demo_basic_tracking(): - """Demo 1: Basic W&B tracking.""" - print_section("DEMO 1: Basic W&B Tracking") - - # Initialize tracker - tracker = WandbTracker( - project_name="stt-finetuning-demo", - enabled=True - ) - - # Start a run - tracker.start_run( - run_name="demo_basic_tracking", - tags=["demo", "basic"] - ) - - print("โœ… W&B run started") - print(f" View at: {tracker.run.url if tracker.run else 'N/A'}") - - # Log some training metrics - print("\n๐Ÿ“Š Logging training metrics...") - for epoch in range(1, 6): - tracker.log_training_metrics( - epoch=epoch, - train_loss=0.5 / epoch, # Decreasing loss - val_loss=0.6 / epoch, - learning_rate=0.001 * (0.9 ** epoch) - ) - print(f" Epoch {epoch}: train_loss={0.5/epoch:.4f}, val_loss={0.6/epoch:.4f}") - - # Finish run - tracker.finish_run() - print("\nโœ… Basic tracking demo completed") - - -def demo_validation_tracking(): - """Demo 2: Validation results tracking.""" - print_section("DEMO 2: Validation Results Tracking") - - tracker = WandbTracker( - project_name="stt-finetuning-demo", - enabled=True - ) - - tracker.start_run( - run_name="demo_validation", - tags=["demo", "validation"] - ) - - print("๐Ÿ“Š Logging validation results...") - - # Mock validation result - validation_result = { - 'model_wer': 0.12, - 'model_cer': 0.06, - 'baseline_wer': 0.20, - 'baseline_cer': 0.10, - 'wer_improvement': 0.08, - 'cer_improvement': 0.04, - 'passed': True, - 'num_samples': 100 - } - - tracker.log_validation_results( - validation_result=validation_result, - model_id="demo_model_v1" - ) - - print(f" Model WER: {validation_result['model_wer']:.4f}") - print(f" Baseline WER: {validation_result['baseline_wer']:.4f}") - print(f" Improvement: {validation_result['wer_improvement']:.4f}") - print(" โœ… W&B will generate comparison charts automatically!") - - tracker.finish_run() - print("\nโœ… Validation tracking demo completed") - - -def demo_regression_tracking(): - """Demo 3: Regression test tracking.""" - print_section("DEMO 3: Regression Test Tracking") - - tracker = WandbTracker( - project_name="stt-finetuning-demo", - enabled=True - ) - - tracker.start_run( - run_name="demo_regression", - tags=["demo", "regression"] - ) - - print("๐Ÿ“Š Logging regression test results...") - - # Mock regression results - regression_results = { - 'total_tests': 5, - 'passed': 4, - 'failed': 1, - 'pass_rate': 0.8, - 'avg_wer_degradation': -0.02, # Negative means improvement - 'avg_cer_degradation': -0.01, - 'results': [ - {'test_id': 'test_001', 'wer_degradation': -0.05, 'passed': True}, - {'test_id': 'test_002', 'wer_degradation': -0.03, 'passed': True}, - {'test_id': 'test_003', 'wer_degradation': 0.02, 'passed': False}, - {'test_id': 'test_004', 'wer_degradation': -0.01, 'passed': True}, - {'test_id': 'test_005', 'wer_degradation': -0.04, 'passed': True} - ] - } - - tracker.log_regression_results( - test_results=regression_results, - model_version="demo_model_v1" - ) - - print(f" Total tests: {regression_results['total_tests']}") - print(f" Pass rate: {regression_results['pass_rate']:.1%}") - print(f" Average WER change: {regression_results['avg_wer_degradation']:+.4f}") - print(" โœ… W&B will generate test result charts!") - - tracker.finish_run() - print("\nโœ… Regression tracking demo completed") - - -def demo_dataset_tracking(): - """Demo 4: Dataset information tracking.""" - print_section("DEMO 4: Dataset Information Tracking") - - tracker = WandbTracker( - project_name="stt-finetuning-demo", - enabled=True - ) - - tracker.start_run( - run_name="demo_dataset", - tags=["demo", "dataset"] - ) - - print("๐Ÿ“Š Logging dataset information...") - - split_sizes = { - 'train': 800, - 'val': 100, - 'test': 100 - } - - error_distribution = { - 'word_substitution': 450, - 'missing_word': 300, - 'extra_word': 150, - 'pronunciation_error': 100 - } - - tracker.log_dataset_info( - dataset_id="demo_dataset_001", - split_sizes=split_sizes, - error_type_distribution=error_distribution - ) - - print(f" Total samples: {sum(split_sizes.values())}") - print(f" Train: {split_sizes['train']}, Val: {split_sizes['val']}, Test: {split_sizes['test']}") - print(f" Error types: {len(error_distribution)}") - print(" โœ… W&B will visualize error distribution!") - - tracker.finish_run() - print("\nโœ… Dataset tracking demo completed") - - -def demo_system_metrics(): - """Demo 5: System-level metrics tracking.""" - print_section("DEMO 5: System Metrics Tracking") - - tracker = WandbTracker( - project_name="stt-finetuning-demo", - enabled=True - ) - - tracker.start_run( - run_name="demo_system_metrics", - tags=["demo", "system"] - ) - - print("๐Ÿ“Š Logging system metrics over time...") - - # Simulate system evolution - for iteration in range(1, 6): - error_cases = 50 * iteration - corrected_cases = int(error_cases * 0.6) - correction_rate = corrected_cases / error_cases - - tracker.log_system_metrics( - error_cases=error_cases, - corrected_cases=corrected_cases, - correction_rate=correction_rate, - models_deployed=iteration - ) - - print(f" Iteration {iteration}: {error_cases} errors, {corrected_cases} corrected") - - print(" โœ… W&B tracks system growth over time!") - - tracker.finish_run() - print("\nโœ… System metrics demo completed") - - -def demo_orchestrator_integration(): - """Demo 6: Full orchestrator integration with W&B.""" - print_section("DEMO 6: Orchestrator Integration") - - print("๐Ÿ”ง Initializing fine-tuning orchestrator with W&B...") - - # Create data manager - data_manager = DataManager( - local_storage_dir="data/wandb_demo", - use_gcs=False - ) - - # Add some test error cases - for i in range(15): - data_manager.store_failed_case( - audio_path=f"test_{i}.wav", - original_transcript=f"original transcript {i}", - corrected_transcript=f"corrected transcript {i}", - error_types=["word_substitution"], - error_score=0.8 - ) - - # Create orchestrator with W&B enabled - config = FinetuningConfig( - min_error_cases=10, - auto_approve_finetuning=True, - use_wandb=True, # Enable W&B - wandb_project="stt-finetuning-demo" - ) - - orchestrator = FinetuningOrchestrator( - data_manager=data_manager, - config=config, - storage_dir="data/wandb_orchestration", - use_gcs=False - ) - - print(f" W&B tracking enabled: {orchestrator.wandb_tracker is not None}") - - # Check trigger conditions - trigger_result = orchestrator.check_trigger_conditions() - print(f" Should trigger: {trigger_result['should_trigger']}") - print(f" Error cases: {trigger_result['metrics']['total_error_cases']}") - - # Trigger fine-tuning (this will start a W&B run automatically) - print("\n๐Ÿš€ Triggering fine-tuning...") - job = orchestrator.trigger_finetuning(force=True) - - if job and orchestrator.wandb_tracker and orchestrator.wandb_tracker.run: - print(f" โœ… Job created: {job.job_id}") - print(f" ๐Ÿ“Š W&B run: {orchestrator.wandb_tracker.run.url}") - print(" All metrics will be automatically logged to W&B!") - - # Finish W&B run - orchestrator.wandb_tracker.finish_run() - - print("\nโœ… Orchestrator integration demo completed") - - -def main(): - """Run all demos.""" - print("\n" + "="*80) - print(" WEIGHTS & BIASES INTEGRATION - COMPREHENSIVE DEMO") - print("="*80) - print("\nThis demo shows how W&B tracks fine-tuning experiments:") - print(" 1. Basic Training Metrics") - print(" 2. Validation Results") - print(" 3. Regression Tests") - print(" 4. Dataset Information") - print(" 5. System Metrics") - print(" 6. Full Orchestrator Integration") - - try: - # Run demos - demo_basic_tracking() - demo_validation_tracking() - demo_regression_tracking() - demo_dataset_tracking() - demo_system_metrics() - demo_orchestrator_integration() - - # Final summary - print_section("DEMO COMPLETE - Summary") - - print("โœ… Successfully demonstrated:") - print(" โœ“ Training metrics tracking") - print(" โœ“ Validation results visualization") - print(" โœ“ Regression test tracking") - print(" โœ“ Dataset information logging") - print(" โœ“ System metrics over time") - print(" โœ“ Full orchestrator integration") - - print("\n๐Ÿ“Š W&B Features Used:") - print(" โ€ข Automatic chart generation") - print(" โ€ข Metric comparison across runs") - print(" โ€ข Dataset visualization") - print(" โ€ข Model artifact logging") - print(" โ€ข Custom plots and tables") - - print("\n๐Ÿš€ Next Steps:") - print(" 1. Check W&B dashboard for visualizations") - print(" 2. Compare multiple training runs") - print(" 3. Analyze performance trends") - print(" 4. Share results with team") - - print("\n๐Ÿ’ก View your W&B dashboard at:") - print(" https://wandb.ai/") - - print("\n" + "="*80) - print(" Demo completed successfully!") - print("="*80 + "\n") - - except Exception as e: - logger.error(f"Demo failed: {e}", exc_info=True) - print(f"\nโŒ Demo failed with error: {e}") - print(" Make sure wandb is installed: pip install wandb") - print(" And you're logged in: wandb login") - sys.exit(1) - - -if __name__ == "__main__": - main() - diff --git a/experiments/evaluate_models.py b/experiments/evaluate_models.py deleted file mode 100644 index 7e8ff15..0000000 --- a/experiments/evaluate_models.py +++ /dev/null @@ -1,93 +0,0 @@ -""" -Task 1: Compare Whisper vs Wav2Vec2 -Run this script to decide which model to deploy -""" - -import sys -from pathlib import Path - -# Add project root to Python path -sys.path.insert(0, str(Path(__file__).parent.parent)) - -from src.model_selector import STTModelEvaluator -import os - -# Download a small test audio file or use your own -# For testing, we'll create a dummy audio or point to existing one -TEST_AUDIO_FILES = [ - "data/test_audio/addf8-Alaw-GW.wav", - # "test_audio_2.wav" -] - -# If you don't have test files, create a synthetic one: -def create_test_audio(): - """Generate a simple test audio file""" - import numpy as np - from scipy.io import wavfile - - sr = 16000 - duration = 3 # 3 seconds - t = np.linspace(0, duration, int(sr * duration)) - - # Simple sine wave for testing - frequency = 440 # A4 note - audio = np.sin(2 * np.pi * frequency * t) * 0.3 - - os.makedirs("test_audio", exist_ok=True) - wavfile.write("data/test_audio/test_1.wav", sr, (audio * 32767).astype(np.int16)) - print("โœ… Created data/test_audio/test_1.wav") - return ["data/test_audio/test_1.wav"] - -if __name__ == "__main__": - print("=" * 50) - print("TASK 1: Model Evaluation") - print("=" * 50) - - # Create test audio if needed - # test_files = create_test_audio() - test_files = TEST_AUDIO_FILES - - evaluator = STTModelEvaluator() - - print("\n๐Ÿ” Comparing Whisper vs Wav2Vec2...\n") - results = evaluator.compare_models(test_files) - - print("\n๐Ÿ”Ž SAMPLE TRANSCRIPTS:") - print("-" * 50) - - test_file = test_files[0] - print(f"\nAudio file: {test_file}") - - # Run both models on that audio, print their transcripts - whisper_proc, whisper_model = evaluator.load_whisper_base() - wav2vec_proc, wav2vec_model = evaluator.load_wav2vec2_base() - - try: - whisper_out = evaluator.benchmark_inference(test_file, whisper_proc, whisper_model, "whisper") - print("\nWHISPER TRANSCRIPT:") - print(whisper_out['transcript']) - except Exception as e: - print(f"Whisper failed: {e}") - - try: - wav2vec_out = evaluator.benchmark_inference(test_file, wav2vec_proc, wav2vec_model, "wav2vec2") - print("\nWAV2VEC2 TRANSCRIPT:") - print(wav2vec_out['transcript']) - except Exception as e: - print(f"Wav2Vec2 failed: {e}") - - print("-" * 50) - - print("\n๐Ÿ“Š COMPARISON RESULTS:") - print("-" * 50) - for model_name, metrics in results.items(): - print(f"\n{model_name.upper()}:") - for key, value in metrics.items(): - if isinstance(value, float): - print(f" {key}: {value:.4f}") - else: - print(f" {key}: {value}") - - print("\n" + "=" * 50) - print("โœ… Evaluation complete. Choose model for Task 2.") - print("=" * 50) diff --git a/experiments/example_usage.py b/experiments/example_usage.py deleted file mode 100644 index 03a3cda..0000000 --- a/experiments/example_usage.py +++ /dev/null @@ -1,346 +0,0 @@ -""" -Example Usage of Data Management System -Demonstrates practical usage scenarios. -""" - -import sys -from pathlib import Path - -# Add src to path -sys.path.insert(0, str(Path(__file__).parent.parent)) - -from src.data.integration import IntegratedDataManagementSystem -from src.agent.agent import STTAgent -from src.baseline_model import BaselineSTTModel -import logging - -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger(__name__) - - -def example_1_record_failures(): - """Example 1: Record failed transcriptions during agent operation.""" - logger.info("=" * 80) - logger.info("Example 1: Recording Failed Transcriptions") - logger.info("=" * 80) - - # Initialize data management system - data_system = IntegratedDataManagementSystem( - base_dir="data/production", - use_gcs=True # Enable GCS for production - ) - - # Initialize STT agent (using mock for example) - # In production, you would use actual model - # baseline_model = BaselineSTTModel() - # agent = STTAgent(baseline_model) - - # Simulate processing audio files - audio_files = [ - "audio/user_recording_1.wav", - "audio/user_recording_2.wav", - "audio/user_recording_3.wav" - ] - - for audio_path in audio_files: - # In production, you would call: result = agent.transcribe_with_agent(audio_path) - # For this example, we'll simulate the result - - result = { - 'transcript': 'THIS IS ALL CAPS', - 'original_transcript': 'THIS IS ALL CAPS', - 'inference_time_seconds': 0.5, - 'error_detection': { - 'has_errors': True, - 'error_score': 0.7, - 'error_types': {'all_caps': 1}, - 'errors': [{'type': 'all_caps', 'confidence': 0.7}] - }, - 'confidence': 0.85 - } - - # If errors detected, record the case - if result['error_detection']['has_errors']: - case_id = data_system.record_failed_transcription( - audio_path=audio_path, - original_transcript=result['original_transcript'], - corrected_transcript=None, # Will be added later when user provides feedback - error_types=list(result['error_detection']['error_types'].keys()), - error_score=result['error_detection']['error_score'], - inference_time=result['inference_time_seconds'], - model_confidence=result.get('confidence'), - additional_metadata={ - 'error_details': result['error_detection']['errors'] - } - ) - logger.info(f"Recorded failed case {case_id} for {audio_path}") - - # Get statistics - stats = data_system.data_manager.get_statistics() - logger.info(f"\nCurrent statistics:") - logger.info(f" Total failed cases: {stats['total_failed_cases']}") - logger.info(f" Uncorrected cases: {stats['uncorrected_cases']}") - - -def example_2_add_corrections(): - """Example 2: Add user corrections to failed cases.""" - logger.info("\n" + "=" * 80) - logger.info("Example 2: Adding User Corrections") - logger.info("=" * 80) - - # Initialize system - data_system = IntegratedDataManagementSystem( - base_dir="data/production", - use_gcs=True - ) - - # Get uncorrected cases - uncorrected = data_system.data_manager.get_uncorrected_cases() - logger.info(f"Found {len(uncorrected)} uncorrected cases") - - # Simulate user providing corrections - for case in uncorrected[:3]: # Correct first 3 cases - # In production, this would come from user feedback - corrected_text = case.original_transcript.capitalize() - - success = data_system.add_correction( - case_id=case.case_id, - corrected_transcript=corrected_text, - correction_method='user_feedback' - ) - - if success: - logger.info(f"Added correction for case {case.case_id}") - - # Updated statistics - stats = data_system.data_manager.get_statistics() - logger.info(f"\nUpdated statistics:") - logger.info(f" Corrected cases: {stats['corrected_cases']}") - logger.info(f" Correction rate: {stats['correction_rate']:.2%}") - - -def example_3_prepare_finetuning(): - """Example 3: Prepare fine-tuning dataset.""" - logger.info("\n" + "=" * 80) - logger.info("Example 3: Preparing Fine-tuning Dataset") - logger.info("=" * 80) - - # Initialize system - data_system = IntegratedDataManagementSystem( - base_dir="data/production", - use_gcs=True - ) - - # Prepare dataset with versioning - dataset_info = data_system.prepare_finetuning_dataset( - min_error_score=0.5, - train_ratio=0.8, - val_ratio=0.1, - test_ratio=0.1, - max_samples=1000, - balance_error_types=True, - create_version=True - ) - - if 'error' not in dataset_info: - logger.info(f"Dataset prepared successfully!") - logger.info(f" Dataset ID: {dataset_info['dataset_id']}") - logger.info(f" Version ID: {dataset_info.get('version_id', 'N/A')}") - logger.info(f" Total samples: {dataset_info['total_samples']}") - logger.info(f" Train: {dataset_info['split_sizes']['train']}") - logger.info(f" Val: {dataset_info['split_sizes']['val']}") - logger.info(f" Test: {dataset_info['split_sizes']['test']}") - - # Prepare in HuggingFace format for training - hf_path = data_system.finetuning_pipeline.prepare_huggingface_dataset( - dataset_id=dataset_info['dataset_id'], - output_format='json' - ) - logger.info(f"\nHuggingFace format dataset: {hf_path}") - - return dataset_info['dataset_id'] - else: - logger.error(f"Dataset preparation failed: {dataset_info['error']}") - return None - - -def example_4_track_training(): - """Example 4: Track model training performance.""" - logger.info("\n" + "=" * 80) - logger.info("Example 4: Tracking Training Performance") - logger.info("=" * 80) - - # Initialize system - data_system = IntegratedDataManagementSystem( - base_dir="data/production", - use_gcs=True - ) - - # Simulate training iterations - model_versions = [ - ("whisper_base_v1", 0.15, 0.08), - ("whisper_base_v2", 0.13, 0.07), - ("whisper_base_v3", 0.11, 0.06) - ] - - for version, wer, cer in model_versions: - data_system.record_training_performance( - model_version=version, - wer=wer, - cer=cer, - training_metadata={ - 'model_name': 'whisper-base', - 'training_data_size': 1000, - 'epochs': 10, - 'batch_size': 16, - 'learning_rate': 1e-5 - } - ) - logger.info(f"Recorded performance for {version}: WER={wer:.4f}, CER={cer:.4f}") - - # Get performance trends - wer_trend = data_system.metadata_tracker.get_performance_trend('wer') - logger.info(f"\nWER Improvement:") - logger.info(f" Initial: {wer_trend['values'][0]:.4f}") - logger.info(f" Latest: {wer_trend['latest']:.4f}") - logger.info(f" Improvement: {wer_trend['improvement']:.4f} ({wer_trend['improvement_percent']:.1f}%)") - - -def example_5_quality_control(): - """Example 5: Quality control and validation.""" - logger.info("\n" + "=" * 80) - logger.info("Example 5: Quality Control") - logger.info("=" * 80) - - # Initialize system - data_system = IntegratedDataManagementSystem( - base_dir="data/production", - use_gcs=True - ) - - # List all datasets - datasets = data_system.finetuning_pipeline.list_datasets() - logger.info(f"Found {len(datasets)} datasets") - - # Validate each dataset - for dataset in datasets: - dataset_id = dataset['dataset_id'] - validation = data_system.finetuning_pipeline.validate_dataset(dataset_id) - - logger.info(f"\nDataset: {dataset_id}") - logger.info(f" Valid: {validation['is_valid']}") - logger.info(f" Issues: {len(validation['issues'])}") - logger.info(f" Warnings: {len(validation['warnings'])}") - - if validation['statistics']: - logger.info(f" Train samples: {validation['statistics'].get('train', {}).get('num_samples', 0)}") - - -def example_6_comprehensive_report(): - """Example 6: Generate comprehensive system report.""" - logger.info("\n" + "=" * 80) - logger.info("Example 6: Comprehensive System Report") - logger.info("=" * 80) - - # Initialize system - data_system = IntegratedDataManagementSystem( - base_dir="data/production", - use_gcs=True - ) - - # Generate report - report = data_system.generate_comprehensive_report( - output_path="data/production/reports/system_report.json" - ) - - logger.info("System Report Generated:") - logger.info(f" Data Quality: {report['data_quality']['quality_status']}") - logger.info(f" Total Failed Cases: {report['data_quality']['total_failed_cases']}") - logger.info(f" Correction Rate: {report['data_quality']['correction_rate']:.2%}") - - logger.info("\nRecommendations:") - for i, rec in enumerate(report['recommendations'], 1): - logger.info(f" {i}. {rec}") - - -def example_7_version_management(): - """Example 7: Dataset version management.""" - logger.info("\n" + "=" * 80) - logger.info("Example 7: Version Management") - logger.info("=" * 80) - - # Initialize system - data_system = IntegratedDataManagementSystem( - base_dir="data/production", - use_gcs=True - ) - - # List all versions - versions = data_system.version_control.list_versions() - logger.info(f"Found {len(versions)} dataset versions") - - # Show version details - for version in versions[:3]: # Show first 3 - logger.info(f"\nVersion: {version.version_id}") - logger.info(f" Created: {version.created_at}") - logger.info(f" Checksum: {version.checksum}") - - quality_report = version.metadata.get('quality_report', {}) - if quality_report: - logger.info(f" Quality Score: {quality_report.get('quality_metrics', {}).get('overall_score', 0):.2f}") - - # Compare versions if we have at least 2 - if len(versions) >= 2: - comparison = data_system.version_control.compare_versions( - versions[0].version_id, - versions[1].version_id - ) - logger.info(f"\nVersion Comparison:") - logger.info(f" Checksum match: {comparison['checksum_match']}") - logger.info(f" Quality improvement: {comparison['quality_comparison']['improvement']:.2f}") - - -def main(): - """Run all examples.""" - logger.info("Data Management System - Example Usage") - logger.info("=" * 80) - - try: - # Note: These examples assume you have some data already - # For a fresh start, run test_data_management.py first - - # Example 1: Record failures during operation - example_1_record_failures() - - # Example 2: Add user corrections - example_2_add_corrections() - - # Example 3: Prepare fine-tuning dataset - example_3_prepare_finetuning() - - # Example 4: Track training performance - example_4_track_training() - - # Example 5: Quality control - example_5_quality_control() - - # Example 6: Generate comprehensive report - example_6_comprehensive_report() - - # Example 7: Version management - example_7_version_management() - - logger.info("\n" + "=" * 80) - logger.info("All examples completed!") - logger.info("=" * 80) - - except Exception as e: - logger.error(f"Example failed: {e}", exc_info=True) - return 1 - - return 0 - - -if __name__ == "__main__": - exit(main()) - diff --git a/experiments/run_agent_evaluation.py b/experiments/run_agent_evaluation.py deleted file mode 100644 index 9b1d7b1..0000000 --- a/experiments/run_agent_evaluation.py +++ /dev/null @@ -1,268 +0,0 @@ -""" -Main Agent Evaluation Script - Week 2 - -Comprehensive evaluation of agent correction accuracy, false positives, -ablation testing, and latency benchmarking. - -This script: -1. Creates a test dataset with ground truth transcripts -2. Runs agent evaluation on the dataset -3. Generates comprehensive reports - -Usage: - python -m experiments.run_agent_evaluation -""" - -import sys -from pathlib import Path - -# Add src to path -sys.path.insert(0, str(Path(__file__).parent.parent)) - -import logging -from typing import Optional, List, Dict -import librosa - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -def create_test_evaluation_dataset() -> Dict: - """ - Create a test evaluation dataset with ground truth transcripts. - - This is based on the create_test_evaluation.py pattern but specifically - designed for agent evaluation with multiple reference samples. - - Returns: - Dictionary with 'audio_paths' and 'reference_transcripts' - """ - - logger.info("Creating test evaluation dataset...") - - # Define test samples with ground truth transcripts - # UPDATE THESE with your actual ground truth transcriptions! - test_samples = [ - { - "audio_path": "data/test_audio/addf8-Alaw-GW.wav", - "reference": "add the sum to the product of these three", - "id": "test_001" - }, - # Add more samples as needed: - # { - # "audio_path": "data/test_audio/another_file.wav", - # "reference": "your ground truth transcription", - # "id": "test_002" - # } - ] - - audio_paths = [] - reference_transcripts = [] - - for sample in test_samples: - audio_path = Path(sample["audio_path"]) - - # Check if file exists - if not audio_path.exists(): - logger.warning(f"โš ๏ธ Audio file not found: {audio_path}") - continue - - try: - # Verify audio can be loaded - audio, sr = librosa.load(str(audio_path), sr=16000) - logger.info(f"โœ… Loaded {audio_path} ({len(audio)/sr:.2f}s audio)") - - audio_paths.append(str(audio_path)) - reference_transcripts.append(sample["reference"]) - - except Exception as e: - logger.error(f"โŒ Error loading {audio_path}: {e}") - continue - - if not audio_paths: - logger.error("โŒ No valid audio files found!") - return None - - logger.info(f"โœ… Created test dataset with {len(audio_paths)} samples") - - return { - "audio_paths": audio_paths, - "reference_transcripts": reference_transcripts - } - - -def run_evaluation(audio_paths: List[str], reference_transcripts: List[str]): - """Run complete agent evaluation""" - - print("\n" + "="*70) - print("AGENT EVALUATION FRAMEWORK - WEEK 2") - print("="*70 + "\n") - - # Import after path setup - from src.baseline_model import BaselineSTTModel - from src.agent import STTAgent - from src.agent_evaluation import ( - AgentEvaluator, - AblationTester, - AgentBenchmark, - FalsePositiveDetector - ) - - # Initialize models - logger.info("Initializing models...") - baseline_model = BaselineSTTModel(model_name="whisper") - agent = STTAgent(baseline_model=baseline_model) - - output_dir = Path("experiments/evaluation_outputs") - output_dir.mkdir(parents=True, exist_ok=True) - - # ============================================================ - # 1. CORRECTION ACCURACY EVALUATION - # ============================================================ - - logger.info("\n" + "="*70) - logger.info("1. CORRECTION ACCURACY EVALUATION") - logger.info("="*70) - logger.info(f"โœ… Running WITH {len(reference_transcripts)} reference transcripts") - - evaluator = AgentEvaluator( - agent=agent, - baseline_model=baseline_model, - output_dir=str(output_dir) - ) - - batch_results = evaluator.evaluate_batch( - audio_paths=audio_paths, - reference_transcripts=reference_transcripts, - enable_correction=True - ) - - evaluator.print_summary() - evaluator.save_results(filename="agent_evaluator_results.json") - - # ============================================================ - # 2. ABLATION TESTING - # ============================================================ - - logger.info("\n" + "="*70) - logger.info("2. ABLATION TESTING") - logger.info("="*70) - - ablation_tester = AblationTester( - agent=agent, - baseline_model=baseline_model, - output_dir=str(output_dir) - ) - - ablation_tester.run_full_ablation(audio_paths=audio_paths) - ablation_tester.print_summary() - ablation_tester.save_ablation_report(filename="ablation_study_results.json") - - # ============================================================ - # 3. LATENCY BENCHMARKING - # ============================================================ - - logger.info("\n" + "="*70) - logger.info("3. LATENCY BENCHMARKING") - logger.info("="*70) - - benchmarker = AgentBenchmark( - agent=agent, - baseline_model=baseline_model, - output_dir=str(output_dir) - ) - - benchmarker.benchmark_batch(audio_paths=audio_paths, verbose=True) - benchmarker.print_summary() - benchmarker.save_benchmark_report(filename="agent_benchmark_results.json") - - # ============================================================ - # 4. FALSE POSITIVE DETECTION - # ============================================================ - - logger.info("\n" + "="*70) - logger.info("4. FALSE POSITIVE DETECTION") - logger.info("="*70) - - fp_detector = FalsePositiveDetector(output_dir=str(output_dir)) - fp_count = 0 - - for idx, (audio_path, ref) in enumerate(zip(audio_paths, reference_transcripts)): - try: - agent_result = agent.transcribe_with_agent(audio_path) - - # Detect false positives for each error detected - for error in agent_result['error_detection'].get('errors', []): - fp = fp_detector.detect_false_positive( - original_transcript=agent_result['original_transcript'], - corrected_transcript=agent_result['transcript'], - reference_transcript=ref, - error_type=error.get('type', 'unknown'), - error_confidence=error.get('confidence', 0.0), - audio_path=str(audio_path) - ) - if fp: - fp_count += 1 - - logger.info(f" Processed {idx+1}/{len(audio_paths)} samples for false positive detection") - except Exception as e: - logger.error(f"Error processing {audio_path}: {e}") - continue - - fp_detector.print_summary() - fp_detector.save_analysis(filename="false_positives_analysis.json") - - # ============================================================ - # FINAL SUMMARY - # ============================================================ - - print("\n" + "="*70) - print("โœ… AGENT EVALUATION COMPLETE") - print("="*70) - print(f"\n๐Ÿ“ All results saved to: {output_dir}") - print("\nGenerated files:") - print(" 1. agent_evaluator_results.json - Correction accuracy metrics") - print(" 2. ablation_study_results.json - Ablation test breakdown") - print(" 3. agent_benchmark_results.json - Latency performance") - print(" 4. false_positives_analysis.json - False positive detection") - print("\nNext steps:") - print(" - Review the JSON files in experiments/evaluation_outputs/") - print(" - Add more test samples to expand evaluation") - print(" - Compare metrics with baseline model") - print("\n" + "="*70 + "\n") - - -def main(): - """Main execution""" - - logger.info("\n" + "="*70) - logger.info("AGENT EVALUATION PIPELINE") - logger.info("="*70) - - # Step 1: Create test evaluation dataset - logger.info("\n๐Ÿ“Š Step 1: Creating test evaluation dataset...") - dataset = create_test_evaluation_dataset() - - if dataset is None or not dataset.get("audio_paths"): - logger.error("โŒ Failed to create test dataset") - return - - audio_paths = dataset["audio_paths"] - reference_transcripts = dataset["reference_transcripts"] - - logger.info(f"\nโœ… Ready to evaluate:") - logger.info(f" - {len(audio_paths)} audio file(s)") - logger.info(f" - {len(reference_transcripts)} reference transcript(s)") - - # Step 2: Run evaluation - logger.info("\n๐Ÿ“ˆ Step 2: Running agent evaluation...") - run_evaluation( - audio_paths=audio_paths, - reference_transcripts=reference_transcripts - ) - - logger.info("\n๐ŸŽ‰ Evaluation pipeline complete!") - - -if __name__ == "__main__": - main() diff --git a/experiments/run_hyperparameter_sweep.py b/experiments/run_hyperparameter_sweep.py deleted file mode 100644 index 1598f9b..0000000 --- a/experiments/run_hyperparameter_sweep.py +++ /dev/null @@ -1,305 +0,0 @@ -#!/usr/bin/env python3 -""" -Run Hyperparameter Optimization Sweep - -Production-ready script to optimize hyperparameters for STT fine-tuning. - -Usage: - # Quick test (5 trials) - python experiments/run_hyperparameter_sweep.py --trials 5 --method random - - # Full optimization (20 trials) - python experiments/run_hyperparameter_sweep.py --trials 20 --method random - - # Refined search (30 trials) - python experiments/run_hyperparameter_sweep.py --trials 30 --method bayes -""" - -import sys -from pathlib import Path -sys.path.insert(0, str(Path(__file__).parent.parent)) - -import argparse -import logging -import json - -from src.data.wandb_sweeps import WandbSweepOrchestrator, SweepConfig -from src.data.data_manager import DataManager -from src.data.finetuning_orchestrator import FinetuningOrchestrator, FinetuningConfig - -try: - import wandb - WANDB_AVAILABLE = True -except ImportError: - WANDB_AVAILABLE = False - print("โŒ wandb not installed. Install with: pip install wandb") - sys.exit(1) - -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s' -) -logger = logging.getLogger(__name__) - - -def create_training_function(data_manager, dataset_id=None): - """ - Create training function for W&B sweep. - - Args: - data_manager: DataManager instance - dataset_id: Optional dataset ID to use - - Returns: - Training function compatible with W&B sweep - """ - def train(): - """Training function that uses wandb.config for hyperparameters.""" - - # Get hyperparameters from sweep - config = wandb.config - - logger.info("="*60) - logger.info(f"Running trial with hyperparameters:") - logger.info(f" Learning rate: {config.learning_rate}") - logger.info(f" Batch size: {config.batch_size}") - logger.info(f" Epochs: {config.epochs}") - logger.info(f" Warmup steps: {config.get('warmup_steps', 500)}") - logger.info("="*60) - - try: - # Initialize orchestrator - orchestrator = FinetuningOrchestrator( - data_manager=data_manager, - config=FinetuningConfig( - min_error_cases=10, - auto_approve_finetuning=True, - use_wandb=False # Don't double-track - ), - use_gcs=False # Local for sweep - ) - - # Trigger fine-tuning - job = orchestrator.trigger_finetuning(force=True) - - if not job: - logger.error("Failed to trigger fine-tuning") - wandb.log({'validation/model_wer': 1.0}) # Worst possible - return - - # Training parameters from sweep - training_params = { - 'learning_rate': config.learning_rate, - 'batch_size': config.batch_size, - 'epochs': config.epochs, - 'warmup_steps': config.get('warmup_steps', 500), - 'weight_decay': config.get('weight_decay', 0.01), - 'gradient_accumulation_steps': config.get('gradient_accumulation_steps', 1), - 'dropout': config.get('dropout', 0.1) - } - - # TODO: Replace with actual training - # For demo, simulate training results - logger.info("โš ๏ธ Using mock training for demo") - logger.info(" Replace with actual training code in production") - - # Mock result (better with certain hyperparameters) - mock_wer = 0.20 - (0.03 if config.learning_rate < 5e-5 else 0) - mock_wer -= (0.02 if config.batch_size == 16 else 0) - mock_wer -= (0.01 * min(config.epochs, 10) / 10) - mock_wer = max(0.05, mock_wer) # Floor at 0.05 - - mock_cer = mock_wer * 0.6 - - # Log final metrics - wandb.log({ - 'validation/model_wer': mock_wer, - 'validation/model_cer': mock_cer, - 'validation/wer_improvement': 0.20 - mock_wer, - 'training/final_loss': 0.15, - 'training/epochs_completed': config.epochs - }) - - logger.info(f"โœ… Trial completed - WER: {mock_wer:.4f}") - - except Exception as e: - logger.error(f"Trial failed: {e}") - wandb.log({'validation/model_wer': 1.0}) # Mark as failed - - return train - - -def main(): - """Main entry point.""" - parser = argparse.ArgumentParser( - description="Run hyperparameter optimization sweep for STT fine-tuning" - ) - parser.add_argument( - '--trials', - type=int, - default=10, - help='Number of trials to run (default: 10)' - ) - parser.add_argument( - '--method', - choices=['random', 'bayes', 'grid'], - default='random', - help='Sweep method (default: random)' - ) - parser.add_argument( - '--project', - default='stt-hyperparameter-optimization', - help='W&B project name' - ) - parser.add_argument( - '--metric', - default='validation/model_wer', - help='Metric to optimize (default: validation/model_wer)' - ) - parser.add_argument( - '--sweep-name', - help='Name for the sweep' - ) - parser.add_argument( - '--parallel', - type=int, - default=1, - help='Number of parallel agents (requires multiple GPUs)' - ) - - args = parser.parse_args() - - print("="*80) - print(" HYPERPARAMETER OPTIMIZATION SWEEP") - print("="*80) - print(f"\nConfiguration:") - print(f" Method: {args.method}") - print(f" Trials: {args.trials}") - print(f" Project: {args.project}") - print(f" Metric: {args.metric}") - print(f" Parallel agents: {args.parallel}") - - # Check W&B login - try: - wandb.login() - except: - print("\nโŒ Not logged into W&B") - print(" Run: wandb login") - sys.exit(1) - - # Initialize data manager - print("\n๐Ÿ“Š Initializing data manager...") - data_manager = DataManager( - local_storage_dir="data/sweep_optimization", - use_gcs=False - ) - - # Add some test cases if needed - stats = data_manager.get_statistics() - if stats['total_failed_cases'] < 10: - print(" Adding test error cases...") - for i in range(15): - data_manager.store_failed_case( - audio_path=f"test_{i}.wav", - original_transcript=f"original {i}", - corrected_transcript=f"corrected {i}", - error_types=["test_error"], - error_score=0.8 - ) - - # Create sweep orchestrator - print(f"\n๐Ÿš€ Creating sweep orchestrator...") - sweep_orch = WandbSweepOrchestrator( - project_name=args.project, - enabled=True - ) - - # Create sweep configuration - print(f"\n๐Ÿ“ Creating {args.method} sweep configuration...") - - if args.method == 'random' and args.trials <= 10: - sweep_config = SweepConfig.create_minimal_sweep(num_trials=args.trials) - else: - sweep_config = SweepConfig.create_finetuning_sweep( - metric_name=args.metric, - goal='minimize', - method=args.method, - num_trials=args.trials - ) - - # Create sweep on W&B - print(f"\n๐Ÿ“Š Creating sweep on W&B...") - sweep_id = sweep_orch.create_sweep( - sweep_config, - sweep_name=args.sweep_name or f"{args.method}_optimization_{args.trials}trials" - ) - - if not sweep_id: - print("โŒ Failed to create sweep") - sys.exit(1) - - print(f"\nโœ… Sweep created: {sweep_id}") - print(f" View at: https://wandb.ai/") - - # Create training function - print(f"\n๐Ÿ‹๏ธ Preparing training function...") - train_fn = create_training_function(data_manager) - - # Run sweep - print(f"\n๐Ÿ”„ Starting sweep with {args.trials} trials...") - print(f" This will take approximately {args.trials * 0.5} hours with mock training") - print(f" With real training: ~{args.trials * 2} hours") - print(f"\n Progress will be visible at: https://wandb.ai/") - - if args.parallel > 1: - print(f"\n๐Ÿ’ก To run {args.parallel} parallel agents:") - print(f" In {args.parallel} separate terminals, run:") - print(f" wandb agent {sweep_id}") - else: - # Run single agent - sweep_orch.run_sweep_agent(train_fn, sweep_id, count=args.trials) - - # Get best configuration - print(f"\n๐Ÿ” Analyzing results...") - best = sweep_orch.get_best_run(sweep_id, metric_name=args.metric) - - if best: - print(f"\n๐Ÿ† BEST CONFIGURATION FOUND:") - print(f" Run: {best['run_name']}") - print(f" Best {args.metric}: {best['metric_value']:.4f}") - print(f"\n Optimal Hyperparameters:") - for param, value in best['hyperparameters'].items(): - print(f" โ€ข {param}: {value}") - - # Save to file - output_file = f"experiments/optimal_hyperparameters_{args.method}.json" - sweep_orch.save_best_config(output_file, sweep_id) - print(f"\n๐Ÿ’พ Saved to: {output_file}") - - print(f"\n๐Ÿ“Š Summary:") - summary = best['summary'] - if 'validation/model_wer' in summary: - print(f" WER: {summary['validation/model_wer']:.4f}") - if 'validation/model_cer' in summary: - print(f" CER: {summary['validation/model_cer']:.4f}") - if 'validation/wer_improvement' in summary: - print(f" Improvement: {summary['validation/wer_improvement']:.4f}") - else: - print("\nโš ๏ธ Could not retrieve best configuration") - - print("\n" + "="*80) - print(" SWEEP COMPLETED!") - print("="*80) - print(f"\n๐Ÿ“Š View detailed results at: https://wandb.ai/") - print(f"\n๐Ÿ’ก Next steps:") - print(f" 1. Review sweep results in W&B dashboard") - print(f" 2. Analyze parameter importance plots") - print(f" 3. Use best config in production:") - print(f" config = load_json('{output_file}')") - print(f" orchestrator.start_training(job_id, config['hyperparameters'])") - print() - - -if __name__ == "__main__": - main() - diff --git a/experiments/sample_outputs/api_response.txt b/experiments/sample_outputs/api_response.txt deleted file mode 100644 index 2706b2e..0000000 --- a/experiments/sample_outputs/api_response.txt +++ /dev/null @@ -1 +0,0 @@ -{"transcript":" add the sum to the product of these three.","model":"whisper","version":"baseline-v1","inference_time_seconds":2.5228610038757324}% diff --git a/experiments/sample_outputs/evaluate_models_1.txt b/experiments/sample_outputs/evaluate_models_1.txt deleted file mode 100644 index 75cce24..0000000 --- a/experiments/sample_outputs/evaluate_models_1.txt +++ /dev/null @@ -1,47 +0,0 @@ -================================================== -TASK 1: Model Evaluation -================================================== - -๐Ÿ” Comparing Whisper vs Wav2Vec2... - -๐Ÿ” Loading models... -Some weights of Wav2Vec2ForCTC were not initialized from the model checkpoint at facebook/wav2vec2-base-960h and are newly initialized: ['wav2vec2.masked_spec_embed'] -You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. - -๐Ÿ“Š Benchmarking on 1 samples... -Using custom `forced_decoder_ids` from the (generation) config. This is deprecated in favor of the `task` and `language` flags/config options. -Transcription using a multilingual Whisper will default to language detection followed by transcription instead of translation to English. This might be a breaking change for your use case. If you want to instead always translate your audio to English, make sure to pass `language='en'`. See https://github.com/huggingface/transformers/pull/28687 for more details. -The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results. - -๐Ÿ”Ž SAMPLE TRANSCRIPTS: --------------------------------------------------- - -Audio file: data/test_audio/addf8-Alaw-GW.wav -Some weights of Wav2Vec2ForCTC were not initialized from the model checkpoint at facebook/wav2vec2-base-960h and are newly initialized: ['wav2vec2.masked_spec_embed'] -You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. - -WHISPER TRANSCRIPT: - add the sum to the product of these three. - -WAV2VEC2 TRANSCRIPT: -ADD THE SUM TO THE PRODUCT OF THESE THREE --------------------------------------------------- - -๐Ÿ“Š COMPARISON RESULTS: --------------------------------------------------- - -WHISPER: - avg_latency: 0.2781 - model_size_mb: 276.9238 - parameters: 72593920 - samples_processed: 1 - -WAV2VEC2: - avg_latency: 0.0609 - model_size_mb: 360.0934 - parameters: 94396320 - samples_processed: 1 - -================================================== -โœ… Evaluation complete. Choose model for Task 2. -================================================== \ No newline at end of file diff --git a/experiments/sample_outputs/test_baseline_1.txt b/experiments/sample_outputs/test_baseline_1.txt deleted file mode 100644 index a6515fd..0000000 --- a/experiments/sample_outputs/test_baseline_1.txt +++ /dev/null @@ -1,25 +0,0 @@ -================================================== -TASK 2: Baseline Model Loading -================================================== - -๐Ÿ“ฆ Loading Whisper baseline model... - -๐Ÿ“‹ Model Info: - name: whisper - parameters: 72,593,920 - device: cpu - trainable_params: 71,825,920 - -๐ŸŽค Testing inference on data/test_audio/addf8-Alaw-GW.wav... -Using custom `forced_decoder_ids` from the (generation) config. This is deprecated in favor of the `task` and `language` flags/config options. -Transcription using a multilingual Whisper will default to language detection followed by transcription instead of translation to English. This might be a breaking change for your use case. If you want to instead always translate your audio to English, make sure to pass `language='en'`. See https://github.com/huggingface/transformers/pull/28687 for more details. -The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results. - -โœ… Inference Result: - Transcript: add the sum to the product of these three. - Model: whisper - Version: baseline-v1 - -================================================== -โœ… Baseline model ready for deployment! -================================================== \ No newline at end of file diff --git a/experiments/test_adaptive_scheduler.py b/experiments/test_adaptive_scheduler.py deleted file mode 100644 index 86ae085..0000000 --- a/experiments/test_adaptive_scheduler.py +++ /dev/null @@ -1,234 +0,0 @@ -""" -Test script for Adaptive Scheduling Algorithm - Week 3 -Tests the adaptive scheduling mechanism, fine-tuning, and closed-loop system. -""" - -import sys -from pathlib import Path - -# Add src to path -sys.path.insert(0, str(Path(__file__).parent.parent)) - -from src.baseline_model import BaselineSTTModel -from src.agent import STTAgent -from src.agent.adaptive_scheduler import AdaptiveScheduler -import logging - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -def test_adaptive_scheduler(): - """Test adaptive scheduler functionality""" - print("\n" + "="*60) - print("Testing Adaptive Scheduler") - print("="*60) - - scheduler = AdaptiveScheduler( - initial_threshold_n=50, - min_threshold_n=20, - max_threshold_n=200 - ) - - # Test 1: Initial state - print("\n1. Testing initial state...") - stats = scheduler.get_scheduler_stats() - print(f" Initial threshold_n: {stats['current_threshold_n']}") - print(f" Error samples collected: {stats['error_samples_collected']}") - assert stats['current_threshold_n'] == 50, "Initial threshold should be 50" - - # Test 2: Recording error samples - print("\n2. Recording error samples...") - scheduler.record_error_sample(count=30) - should_trigger, info = scheduler.should_trigger_fine_tuning() - print(f" Samples collected: {scheduler.error_samples_collected}") - print(f" Should trigger: {should_trigger}") - assert scheduler.error_samples_collected == 30, "Should have 30 samples" - assert not should_trigger, "Should not trigger yet" - - # Test 3: Recording performance metrics - print("\n3. Recording performance metrics...") - for i in range(10): - accuracy = 0.85 + (i * 0.01) # Gradually improving - scheduler.record_performance( - error_count=5, - accuracy=accuracy, - inference_time=0.5 - ) - stats = scheduler.get_scheduler_stats() - print(f" Recent accuracy: {stats['recent_accuracy']:.4f}") - print(f" Accuracy trend: {stats['accuracy_trend']:.4f}") - assert stats['recent_accuracy'] is not None, "Should have recent accuracy" - - # Test 4: Trigger fine-tuning - print("\n4. Triggering fine-tuning threshold...") - scheduler.record_error_sample(count=25) # Now at 55, above threshold of 50 - should_trigger, info = scheduler.should_trigger_fine_tuning() - print(f" Should trigger: {should_trigger}") - assert should_trigger, "Should trigger fine-tuning" - - # Test 5: Record fine-tuning event - print("\n5. Recording fine-tuning event...") - scheduler.record_fine_tuning_event( - samples_used=55, - validation_accuracy_before=0.85, - validation_accuracy_after=0.87, - training_cost=10.0 - ) - stats = scheduler.get_scheduler_stats() - print(f" New threshold_n: {stats['current_threshold_n']}") - print(f" Samples reset: {scheduler.error_samples_collected}") - print(f" Total fine-tuning events: {stats['total_fine_tuning_events']}") - assert scheduler.error_samples_collected == 0, "Samples should be reset" - assert stats['total_fine_tuning_events'] == 1, "Should have 1 fine-tuning event" - - # Test 6: Diminishing gains detection - print("\n6. Testing diminishing gains detection...") - scheduler.record_fine_tuning_event( - samples_used=60, - validation_accuracy_before=0.87, - validation_accuracy_after=0.872, # Small gain - training_cost=10.0 - ) - scheduler.record_fine_tuning_event( - samples_used=65, - validation_accuracy_before=0.872, - validation_accuracy_after=0.873, # Very small gain - training_cost=10.0 - ) - stats = scheduler.get_scheduler_stats() - print(f" Diminishing gains detected: {stats['diminishing_gains_detected']}") - print(f" Threshold after diminishing gains: {stats['current_threshold_n']}") - assert stats['diminishing_gains_detected'], "Should detect diminishing gains" - - print("\nโœ… All adaptive scheduler tests passed!") - - -def test_overfitting_detection(): - """Test overfitting detection""" - print("\n" + "="*60) - print("Testing Overfitting Detection") - print("="*60) - - scheduler = AdaptiveScheduler(overfitting_threshold=0.1) - - # Test 1: No overfitting - print("\n1. Testing normal case (no overfitting)...") - is_overfitting, info = scheduler.check_overfitting( - train_accuracy=0.90, - validation_accuracy=0.88 - ) - print(f" Train accuracy: {info['train_accuracy']:.4f}") - print(f" Validation accuracy: {info['validation_accuracy']:.4f}") - print(f" Accuracy gap: {info['accuracy_gap']:.4f}") - print(f" Is overfitting: {is_overfitting}") - assert not is_overfitting, "Should not detect overfitting" - - # Test 2: Overfitting detected - print("\n2. Testing overfitting case...") - is_overfitting, info = scheduler.check_overfitting( - train_accuracy=0.95, - validation_accuracy=0.82 # Large gap - ) - print(f" Train accuracy: {info['train_accuracy']:.4f}") - print(f" Validation accuracy: {info['validation_accuracy']:.4f}") - print(f" Accuracy gap: {info['accuracy_gap']:.4f}") - print(f" Is overfitting: {is_overfitting}") - assert is_overfitting, "Should detect overfitting" - - print("\nโœ… Overfitting detection tests passed!") - - -def test_cost_efficiency(): - """Test cost efficiency tracking""" - print("\n" + "="*60) - print("Testing Cost Efficiency Tracking") - print("="*60) - - scheduler = AdaptiveScheduler() - - # Record some fine-tuning events with varying costs - print("\n1. Recording fine-tuning events with costs...") - scheduler.record_fine_tuning_event( - samples_used=100, - validation_accuracy_before=0.80, - validation_accuracy_after=0.85, # Good gain - training_cost=5.0 # Low cost - ) - - scheduler.record_fine_tuning_event( - samples_used=100, - validation_accuracy_before=0.85, - validation_accuracy_after=0.86, # Small gain - training_cost=20.0 # High cost - ) - - stats = scheduler.get_scheduler_stats() - print(f" Total training cost: {stats['total_training_cost']:.2f}") - print(f" Cost efficiency: {stats['cost_efficiency']:.4f}") - print(f" Average accuracy gain: {stats['average_accuracy_gain']:.4f}") - - assert stats['total_training_cost'] > 0, "Should track training cost" - assert 0 <= stats['cost_efficiency'] <= 1, "Cost efficiency should be 0-1" - - print("\nโœ… Cost efficiency tracking tests passed!") - - -def test_integrated_agent(): - """Test agent with adaptive scheduling integrated""" - print("\n" + "="*60) - print("Testing Integrated Agent with Adaptive Scheduling") - print("="*60) - - # Initialize baseline model and agent - print("\n1. Initializing agent with adaptive fine-tuning...") - baseline_model = BaselineSTTModel(model_name="whisper") - agent = STTAgent( - baseline_model=baseline_model, - enable_adaptive_fine_tuning=True, - scheduler_history_path="data/processed/test_scheduler_history.json" - ) - - # Check if adaptive scheduler is initialized - scheduler_stats = agent.get_adaptive_scheduler_stats() - if scheduler_stats: - print(f" โœ… Adaptive scheduler initialized") - print(f" Initial threshold_n: {scheduler_stats['current_threshold_n']}") - else: - print(" โš ๏ธ Adaptive scheduler not available (may need model/processor access)") - return - - # Test getting stats - print("\n2. Getting agent stats...") - agent_stats = agent.get_agent_stats() - print(f" Agent stats available: {len(agent_stats)} keys") - - print("\nโœ… Integrated agent tests passed!") - - -if __name__ == "__main__": - print("\n" + "="*60) - print("Adaptive Scheduling Algorithm - Week 3 Tests") - print("="*60) - - try: - # Test individual components - test_adaptive_scheduler() - test_overfitting_detection() - test_cost_efficiency() - - # Test integrated system (may skip if model loading fails) - try: - test_integrated_agent() - except Exception as e: - print(f"\nโš ๏ธ Integrated agent test skipped: {e}") - - print("\n" + "="*60) - print("โœ… All tests completed successfully!") - print("="*60) - - except Exception as e: - print(f"\nโŒ Test failed: {e}") - import traceback - traceback.print_exc() - sys.exit(1) diff --git a/experiments/test_api.py b/experiments/test_api.py deleted file mode 100644 index af4cc60..0000000 --- a/experiments/test_api.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -Task 3: Test the live API -Run this while the API server is running in another terminal -""" - -import requests -import json - -API_URL = "http://localhost:8000" - -if __name__ == "__main__": - print("=" * 50) - print("TASK 3: API Testing") - print("=" * 50) - - # Test 1: Health check - print("\n๐Ÿฅ Testing health endpoint...") - response = requests.get(f"{API_URL}/health") - print(f"Status: {response.json()}") - - # Test 2: Model info - print("\n๐Ÿ“‹ Getting model info...") - response = requests.get(f"{API_URL}/model-info") - model_info = response.json() - for key, value in model_info.items(): - print(f" {key}: {value}") - - # Test 3: Transcription - print("\n๐ŸŽค Testing transcription endpoint...") - test_audio_path = "data/test_audio/test_1.wav" - - with open(test_audio_path, "rb") as f: - files = {"file": f} - response = requests.post(f"{API_URL}/transcribe", files=files) - - result = response.json() - print(f"\nโœ… Transcription Result:") - for key, value in result.items(): - print(f" {key}: {value}") - - print("\n" + "=" * 50) - print("โœ… API is working!") - print("=" * 50) diff --git a/experiments/visualize_evaluation_results.py b/experiments/visualize_evaluation_results.py deleted file mode 100644 index 5c89069..0000000 --- a/experiments/visualize_evaluation_results.py +++ /dev/null @@ -1,197 +0,0 @@ -""" -Visualization script for evaluation framework results. -Generates charts and plots for evaluation metrics. -""" - -import json -import sys -from pathlib import Path -import matplotlib.pyplot as plt -import seaborn as sns -import pandas as pd - -# Add src to path -sys.path.append(str(Path(__file__).parent.parent)) - -# Set style -sns.set_style("whitegrid") -plt.rcParams['figure.figsize'] = (12, 6) - - -def load_evaluation_results(results_path: str = "experiments/evaluation_outputs/evaluation_summary.json"): - """Load evaluation results from JSON file.""" - with open(results_path, 'r') as f: - return json.load(f) - - -def plot_wer_cer_comparison(summary: dict, output_path: str): - """Plot WER and CER comparison across datasets.""" - per_dataset = summary.get('per_dataset_metrics', {}) - - if not per_dataset: - print("No per-dataset metrics found") - return - - datasets = list(per_dataset.keys()) - wers = [per_dataset[d]['wer'] for d in datasets] - cers = [per_dataset[d]['cer'] for d in datasets] - - x = range(len(datasets)) - width = 0.35 - - fig, ax = plt.subplots() - ax.bar([i - width/2 for i in x], wers, width, label='WER', alpha=0.8) - ax.bar([i + width/2 for i in x], cers, width, label='CER', alpha=0.8) - - ax.set_xlabel('Dataset') - ax.set_ylabel('Error Rate') - ax.set_title('WER and CER Comparison Across Datasets') - ax.set_xticks(x) - ax.set_xticklabels(datasets, rotation=45, ha='right') - ax.legend() - ax.grid(True, alpha=0.3) - - plt.tight_layout() - plt.savefig(output_path, dpi=300, bbox_inches='tight') - print(f"โœ… Saved plot to {output_path}") - plt.close() - - -def plot_error_distribution(detailed_results_path: str, output_path: str): - """Plot distribution of WER across samples.""" - with open(detailed_results_path, 'r') as f: - results = json.load(f) - - all_wers = [] - for dataset_name, dataset_results in results.get('detailed_results', {}).items(): - if 'error_analysis' in dataset_results: - worst_errors = dataset_results['error_analysis'].get('worst_errors', []) - wers = [e['wer'] for e in worst_errors] - all_wers.extend(wers) - - if not all_wers: - print("No error data found for plotting") - return - - plt.figure() - plt.hist(all_wers, bins=30, edgecolor='black', alpha=0.7) - plt.xlabel('Word Error Rate (WER)') - plt.ylabel('Frequency') - plt.title('Distribution of WER Across Evaluation Samples') - plt.grid(True, alpha=0.3) - - plt.tight_layout() - plt.savefig(output_path, dpi=300, bbox_inches='tight') - print(f"โœ… Saved plot to {output_path}") - plt.close() - - -def create_evaluation_dashboard(summary_path: str, output_path: str): - """Create comprehensive evaluation dashboard.""" - with open(summary_path, 'r') as f: - summary = json.load(f) - - fig, axes = plt.subplots(2, 2, figsize=(15, 12)) - - # 1. WER/CER comparison - per_dataset = summary.get('per_dataset_metrics', {}) - if per_dataset: - datasets = list(per_dataset.keys()) - wers = [per_dataset[d]['wer'] for d in datasets] - cers = [per_dataset[d]['cer'] for d in datasets] - - axes[0, 0].bar(datasets, wers, alpha=0.7, label='WER') - axes[0, 0].set_title('Word Error Rate by Dataset') - axes[0, 0].set_ylabel('WER') - axes[0, 0].tick_params(axis='x', rotation=45) - axes[0, 0].grid(True, alpha=0.3) - - axes[0, 1].bar(datasets, cers, alpha=0.7, color='orange', label='CER') - axes[0, 1].set_title('Character Error Rate by Dataset') - axes[0, 1].set_ylabel('CER') - axes[0, 1].tick_params(axis='x', rotation=45) - axes[0, 1].grid(True, alpha=0.3) - - # 2. Overall metrics summary - overall = summary.get('overall_metrics', {}) - if overall.get('mean_wer') is not None: - metrics_text = f""" - Model: {summary.get('model', 'N/A')} - Total Samples: {summary.get('total_samples_evaluated', 0)} - - Mean WER: {overall['mean_wer']:.4f} ยฑ {overall.get('std_wer', 0):.4f} - Mean CER: {overall['mean_cer']:.4f} ยฑ {overall.get('std_cer', 0):.4f} - Best WER: {overall.get('best_wer', 0):.4f} - Worst WER: {overall.get('worst_wer', 0):.4f} - """ - axes[1, 0].text(0.1, 0.5, metrics_text, fontsize=12, - verticalalignment='center', family='monospace') - axes[1, 0].set_title('Overall Metrics Summary') - axes[1, 0].axis('off') - - # 3. Sample count by dataset - if per_dataset: - datasets = list(per_dataset.keys()) - sample_counts = [per_dataset[d]['num_samples'] for d in datasets] - - axes[1, 1].bar(datasets, sample_counts, alpha=0.7, color='green') - axes[1, 1].set_title('Samples Evaluated per Dataset') - axes[1, 1].set_ylabel('Number of Samples') - axes[1, 1].tick_params(axis='x', rotation=45) - axes[1, 1].grid(True, alpha=0.3) - - plt.suptitle('STT Model Evaluation Dashboard', fontsize=16, y=0.995) - plt.tight_layout() - plt.savefig(output_path, dpi=300, bbox_inches='tight') - print(f"โœ… Saved dashboard to {output_path}") - plt.close() - - -def main(): - """Main visualization execution.""" - print("="*70) - print("EVALUATION RESULTS VISUALIZATION") - print("="*70) - - results_dir = Path("experiments/evaluation_outputs") - summary_path = results_dir / "evaluation_summary.json" - detailed_path = results_dir / "evaluation_report.json" - - if not summary_path.exists(): - print(f"โŒ Evaluation results not found at {summary_path}") - print(" Run kavya_evaluation_framework.py first") - return - - # Load results - summary = load_evaluation_results(str(summary_path)) - - # Create visualizations - output_dir = results_dir / "visualizations" - output_dir.mkdir(exist_ok=True) - - print("\n๐Ÿ“Š Generating visualizations...") - - # WER/CER comparison - plot_wer_cer_comparison( - summary, - str(output_dir / "wer_cer_comparison.png") - ) - - # Error distribution - if detailed_path.exists(): - plot_error_distribution( - str(detailed_path), - str(output_dir / "error_distribution.png") - ) - - # Dashboard - create_evaluation_dashboard( - str(summary_path), - str(output_dir / "evaluation_dashboard.png") - ) - - print(f"\nโœ… All visualizations saved to {output_dir}") - - -if __name__ == "__main__": - main() diff --git a/scripts/INSTALL_GCLOUD.md b/scripts/INSTALL_GCLOUD.md deleted file mode 100644 index 3cecb83..0000000 --- a/scripts/INSTALL_GCLOUD.md +++ /dev/null @@ -1,92 +0,0 @@ -# Installing Google Cloud SDK (gcloud CLI) - -## For macOS (Your System) - -### Option 1: Using Homebrew (Recommended - Easiest) - -```bash -# Install Homebrew if you don't have it -/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - -# Install gcloud CLI -brew install --cask google-cloud-sdk - -# Initialize gcloud -gcloud init -``` - -### Option 2: Direct Download - -```bash -# Download and install -curl https://sdk.cloud.google.com | bash - -# Restart your shell or run: -exec -l $SHELL - -# Initialize -gcloud init -``` - -### Option 3: Using the Installer Script - -```bash -# Download installer -curl https://sdk.cloud.google.com | bash - -# Follow the prompts, then: -gcloud init -``` - -## After Installation - -1. **Authenticate**: - ```bash - gcloud auth login - ``` - -2. **Set your project**: - ```bash - gcloud config set project stt-agentic-ai-2025 - ``` - (Or use your actual GCP project ID) - -3. **Enable required APIs**: - ```bash - gcloud services enable compute.googleapis.com - gcloud services enable storage-api.googleapis.com - ``` - -4. **Verify installation**: - ```bash - gcloud --version - gcloud compute zones list - ``` - -## Quick Test - -```bash -# Check if gcloud works -gcloud --version - -# List available zones -gcloud compute zones list | grep us-central -``` - -## Next Steps - -Once gcloud is installed: -1. Run: `bash scripts/setup_gcp_gpu.sh` -2. Or follow the manual setup in `docs/GCP_SETUP_GUIDE.md` - -## Troubleshooting - -- **Permission errors**: May need to add gcloud to PATH -- **Authentication issues**: Run `gcloud auth login` -- **Project not found**: Make sure you have access to the project - -## Official Documentation - -- [Install Guide](https://cloud.google.com/sdk/docs/install) -- [Quick Start](https://cloud.google.com/sdk/docs/quickstart) - diff --git a/scripts/deploy_complete_system.py b/scripts/deploy_complete_system.py deleted file mode 100755 index 66632fe..0000000 --- a/scripts/deploy_complete_system.py +++ /dev/null @@ -1,477 +0,0 @@ -#!/usr/bin/env python3 -""" -Complete GCP Deployment Script -Automates the entire deployment process for the STT system -""" - -import subprocess -import sys -import os -import json -import time -from pathlib import Path -from typing import Dict, List, Optional -import argparse - -class Colors: - """Terminal colors for output""" - HEADER = '\033[95m' - OKBLUE = '\033[94m' - OKCYAN = '\033[96m' - OKGREEN = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - ENDC = '\033[0m' - BOLD = '\033[1m' - -def print_header(text: str): - """Print formatted header""" - print(f"\n{Colors.HEADER}{Colors.BOLD}{'='*60}") - print(f"{text}") - print(f"{'='*60}{Colors.ENDC}\n") - -def print_step(step: int, text: str): - """Print formatted step""" - print(f"{Colors.OKBLUE}{Colors.BOLD}[STEP {step}]{Colors.ENDC} {text}") - -def print_success(text: str): - """Print success message""" - print(f"{Colors.OKGREEN}โœ… {text}{Colors.ENDC}") - -def print_warning(text: str): - """Print warning message""" - print(f"{Colors.WARNING}โš ๏ธ {text}{Colors.ENDC}") - -def print_error(text: str): - """Print error message""" - print(f"{Colors.FAIL}โŒ {text}{Colors.ENDC}") - -def run_command(cmd: List[str], check: bool = True, capture: bool = True) -> subprocess.CompletedProcess: - """Run a shell command""" - print(f" Running: {' '.join(cmd)}") - try: - result = subprocess.run( - cmd, - check=check, - capture_output=capture, - text=True - ) - return result - except subprocess.CalledProcessError as e: - if check: - print_error(f"Command failed: {e}") - if e.stderr: - print(e.stderr) - sys.exit(1) - return e - -def check_prerequisites(): - """Check if all prerequisites are installed""" - print_step(0, "Checking prerequisites...") - - required_tools = { - 'gcloud': ['gcloud', '--version'], - 'docker': ['docker', '--version'], - 'python': ['python3', '--version'], - 'git': ['git', '--version'] - } - - missing = [] - for tool, cmd in required_tools.items(): - try: - subprocess.run(cmd, capture_output=True, check=True) - print_success(f"{tool} installed") - except (subprocess.CalledProcessError, FileNotFoundError): - print_error(f"{tool} not found") - missing.append(tool) - - if missing: - print_error(f"Missing required tools: {', '.join(missing)}") - sys.exit(1) - - print_success("All prerequisites satisfied") - -def setup_gcp_project(project_id: str, region: str, zone: str): - """Setup GCP project configuration""" - print_step(1, "Setting up GCP project...") - - # Set project - run_command(['gcloud', 'config', 'set', 'project', project_id]) - - # Set region and zone - run_command(['gcloud', 'config', 'set', 'compute/region', region]) - run_command(['gcloud', 'config', 'set', 'compute/zone', zone]) - - print_success(f"Project configured: {project_id}") - -def enable_apis(project_id: str): - """Enable required GCP APIs""" - print_step(2, "Enabling GCP APIs...") - - apis = [ - 'compute.googleapis.com', - 'storage.googleapis.com', - 'run.googleapis.com', - 'cloudbuild.googleapis.com', - 'containerregistry.googleapis.com', - 'logging.googleapis.com', - 'monitoring.googleapis.com' - ] - - for api in apis: - print(f" Enabling {api}...") - run_command(['gcloud', 'services', 'enable', api], check=False) - - print_success("All APIs enabled") - -def create_storage_buckets(project_id: str, region: str): - """Create GCS buckets""" - print_step(3, "Creating GCS buckets...") - - buckets = { - 'datasets': f"{project_id}-stt-datasets", - 'models': f"{project_id}-stt-models", - 'logs': f"{project_id}-stt-logs" - } - - for name, bucket in buckets.items(): - print(f" Creating bucket: {bucket}") - result = run_command([ - 'gsutil', 'mb', - '-p', project_id, - '-c', 'STANDARD', - '-l', region, - f'gs://{bucket}' - ], check=False) - - if result.returncode == 0: - print_success(f"Created {name} bucket") - else: - print_warning(f"Bucket {bucket} may already exist") - - return buckets - -def create_service_account(project_id: str): - """Create service account with required permissions""" - print_step(4, "Creating service account...") - - sa_name = 'stt-service-account' - sa_email = f"{sa_name}@{project_id}.iam.gserviceaccount.com" - - # Create service account - result = run_command([ - 'gcloud', 'iam', 'service-accounts', 'create', sa_name, - '--display-name', 'STT System Service Account' - ], check=False) - - if result.returncode != 0: - print_warning("Service account may already exist") - - # Grant roles - roles = [ - 'roles/storage.objectAdmin', - 'roles/logging.logWriter', - 'roles/monitoring.metricWriter' - ] - - for role in roles: - print(f" Granting {role}...") - run_command([ - 'gcloud', 'projects', 'add-iam-policy-binding', project_id, - '--member', f'serviceAccount:{sa_email}', - '--role', role - ], check=False) - - print_success(f"Service account configured: {sa_email}") - return sa_email - -def build_and_push_image(project_id: str): - """Build and push Docker image""" - print_step(5, "Building and pushing Docker image...") - - image_url = f"gcr.io/{project_id}/stt-api:latest" - - print(" Building Docker image...") - run_command(['gcloud', 'builds', 'submit', '--tag', image_url], check=True) - - print_success(f"Image built and pushed: {image_url}") - return image_url - -def deploy_cloud_run( - project_id: str, - region: str, - image_url: str, - service_account: str, - buckets: Dict[str, str], - allow_unauthenticated: bool = True -): - """Deploy to Cloud Run""" - print_step(6, "Deploying to Cloud Run...") - - env_vars = ( - f"USE_GCS=true," - f"GCS_DATASETS_BUCKET={buckets['datasets']}," - f"GCS_MODELS_BUCKET={buckets['models']}," - f"GCS_LOGS_BUCKET={buckets['logs']}" - ) - - cmd = [ - 'gcloud', 'run', 'deploy', 'stt-api', - '--image', image_url, - '--platform', 'managed', - '--region', region, - '--memory', '4Gi', - '--cpu', '2', - '--timeout', '300', - '--max-instances', '10', - '--set-env-vars', env_vars, - '--service-account', service_account - ] - - if allow_unauthenticated: - cmd.append('--allow-unauthenticated') - - run_command(cmd) - - # Get service URL - result = run_command([ - 'gcloud', 'run', 'services', 'describe', 'stt-api', - '--region', region, - '--format', 'value(status.url)' - ]) - - service_url = result.stdout.strip() - print_success(f"Service deployed: {service_url}") - return service_url - -def test_deployment(service_url: str): - """Test the deployed service""" - print_step(7, "Testing deployment...") - - # Test health endpoint - print(" Testing /api/health...") - result = run_command(['curl', '-f', f'{service_url}/api/health'], check=False) - - if result.returncode == 0: - print_success("Health check passed") - try: - health_data = json.loads(result.stdout) - print(f" Status: {health_data.get('status')}") - except: - pass - else: - print_error("Health check failed") - return False - - # Test root endpoint - print(" Testing root endpoint...") - result = run_command(['curl', '-f', service_url], check=False) - - if result.returncode == 0: - print_success("Root endpoint accessible") - else: - print_warning("Root endpoint test failed") - - return True - -def setup_monitoring(project_id: str, service_url: str, notification_email: Optional[str]): - """Setup monitoring and alerts""" - print_step(8, "Setting up monitoring...") - - if notification_email: - print(f" Creating notification channel for {notification_email}...") - # Note: This requires alpha/beta gcloud features - print_warning("Manual setup required for email notifications") - print(f" Go to: https://console.cloud.google.com/monitoring/alerting") - - print_success("Monitoring setup guidance provided") - -def create_gpu_vm(project_id: str, zone: str, service_account: str, create_vm: bool = False): - """Create GPU VM for training""" - if not create_vm: - print_step(9, "Skipping GPU VM creation (use --create-gpu-vm to enable)") - return - - print_step(9, "Creating GPU VM...") - - vm_name = 'stt-training-vm' - - cmd = [ - 'gcloud', 'compute', 'instances', 'create', vm_name, - '--zone', zone, - '--machine-type', 'n1-standard-8', - '--accelerator', 'type=nvidia-tesla-t4,count=1', - '--image-family', 'pytorch-latest-gpu', - '--image-project', 'deeplearning-platform-release', - '--boot-disk-size', '200GB', - '--boot-disk-type', 'pd-ssd', - '--maintenance-policy', 'TERMINATE', - '--metadata', 'install-nvidia-driver=True', - '--scopes', 'cloud-platform', - '--service-account', service_account - ] - - result = run_command(cmd, check=False) - - if result.returncode == 0: - print_success(f"GPU VM created: {vm_name}") - print_warning("Remember to stop the VM when not in use to save costs!") - print(f" Stop: gcloud compute instances stop {vm_name} --zone={zone}") - else: - print_warning("GPU VM creation failed (may already exist or quota limit)") - -def save_deployment_info( - project_id: str, - region: str, - zone: str, - service_url: str, - buckets: Dict[str, str], - service_account: str -): - """Save deployment information to file""" - print_step(10, "Saving deployment information...") - - deployment_info = { - 'project_id': project_id, - 'region': region, - 'zone': zone, - 'service_url': service_url, - 'buckets': buckets, - 'service_account': service_account, - 'deployment_time': time.strftime('%Y-%m-%d %H:%M:%S'), - 'api_docs': f'{service_url}/docs', - 'frontend': f'{service_url}/app' - } - - output_file = Path('deployment-info.json') - with open(output_file, 'w') as f: - json.dump(deployment_info, f, indent=2) - - print_success(f"Deployment info saved to {output_file}") - - # Print summary - print_header("DEPLOYMENT SUMMARY") - print(f"{Colors.BOLD}Service URL:{Colors.ENDC} {service_url}") - print(f"{Colors.BOLD}API Docs:{Colors.ENDC} {service_url}/docs") - print(f"{Colors.BOLD}Frontend:{Colors.ENDC} {service_url}/app") - print(f"\n{Colors.BOLD}Buckets:{Colors.ENDC}") - for name, bucket in buckets.items(): - print(f" {name}: gs://{bucket}") - print(f"\n{Colors.BOLD}Service Account:{Colors.ENDC} {service_account}") - - # Print .env content - print(f"\n{Colors.BOLD}Add to your .env file:{Colors.ENDC}") - print(f"PROJECT_ID={project_id}") - print(f"REGION={region}") - print(f"ZONE={zone}") - print(f"API_URL={service_url}") - print(f"GCS_DATASETS_BUCKET={buckets['datasets']}") - print(f"GCS_MODELS_BUCKET={buckets['models']}") - print(f"GCS_LOGS_BUCKET={buckets['logs']}") - -def main(): - parser = argparse.ArgumentParser( - description='Deploy STT system to Google Cloud Platform', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -Examples: - # Basic deployment - python scripts/deploy_complete_system.py --project-id my-project - - # With GPU VM and custom region - python scripts/deploy_complete_system.py --project-id my-project --create-gpu-vm --region us-west1 - - # Skip Docker build (use existing image) - python scripts/deploy_complete_system.py --project-id my-project --skip-build - """ - ) - - parser.add_argument('--project-id', required=True, help='GCP Project ID') - parser.add_argument('--region', default='us-central1', help='GCP region (default: us-central1)') - parser.add_argument('--zone', default='us-central1-a', help='GCP zone (default: us-central1-a)') - parser.add_argument('--create-gpu-vm', action='store_true', help='Create GPU VM for training') - parser.add_argument('--skip-build', action='store_true', help='Skip Docker build (use existing image)') - parser.add_argument('--skip-prerequisites', action='store_true', help='Skip prerequisite checks') - parser.add_argument('--notification-email', help='Email for monitoring alerts') - parser.add_argument('--require-authentication', action='store_true', help='Require authentication for API access') - - args = parser.parse_args() - - print_header("GCP DEPLOYMENT - Adaptive Self-Learning Agentic AI System") - print(f"Project ID: {args.project_id}") - print(f"Region: {args.region}") - print(f"Zone: {args.zone}") - - try: - # Step 0: Prerequisites - if not args.skip_prerequisites: - check_prerequisites() - - # Step 1: Setup project - setup_gcp_project(args.project_id, args.region, args.zone) - - # Step 2: Enable APIs - enable_apis(args.project_id) - - # Step 3: Create buckets - buckets = create_storage_buckets(args.project_id, args.region) - - # Step 4: Create service account - service_account = create_service_account(args.project_id) - - # Step 5: Build and push image - if args.skip_build: - image_url = f"gcr.io/{args.project_id}/stt-api:latest" - print_warning("Skipping build, using existing image") - else: - image_url = build_and_push_image(args.project_id) - - # Step 6: Deploy to Cloud Run - service_url = deploy_cloud_run( - args.project_id, - args.region, - image_url, - service_account, - buckets, - allow_unauthenticated=not args.require_authentication - ) - - # Step 7: Test deployment - test_deployment(service_url) - - # Step 8: Setup monitoring - setup_monitoring(args.project_id, service_url, args.notification_email) - - # Step 9: Create GPU VM (optional) - create_gpu_vm(args.project_id, args.zone, service_account, args.create_gpu_vm) - - # Step 10: Save deployment info - save_deployment_info( - args.project_id, - args.region, - args.zone, - service_url, - buckets, - service_account - ) - - print_header("DEPLOYMENT COMPLETE!") - print_success("Your STT system is now deployed and ready to use!") - print(f"\n{Colors.BOLD}Next steps:{Colors.ENDC}") - print(" 1. Test the API:", f"curl {service_url}/api/health") - print(" 2. Open frontend:", f"open {service_url}/app") - print(" 3. View API docs:", f"open {service_url}/docs") - print(" 4. Monitor costs: python scripts/monitor_gcp_costs.py") - - except KeyboardInterrupt: - print_error("\nDeployment cancelled by user") - sys.exit(1) - except Exception as e: - print_error(f"Deployment failed: {e}") - import traceback - traceback.print_exc() - sys.exit(1) - -if __name__ == '__main__': - main() - diff --git a/scripts/deploy_finetuning_to_gcp.py b/scripts/deploy_finetuning_to_gcp.py deleted file mode 100644 index 61563be..0000000 --- a/scripts/deploy_finetuning_to_gcp.py +++ /dev/null @@ -1,421 +0,0 @@ -#!/usr/bin/env python3 -""" -Deploy and Run Fine-Tuning on Google Cloud Platform -Automates model fine-tuning on GCP GPU instances. -""" - -import sys -import subprocess -import os -import json -import argparse -from pathlib import Path -import time - -# Configuration -PROJECT_ID = "stt-agentic-ai-2025" -ZONE = "us-central1-a" -VM_NAME = "stt-finetuning-gpu-vm" -REMOTE_DIR = "~/stt-project" -BUCKET_NAME = "stt-project-models" - -def run_command(cmd, check=True, capture_output=True): - """Run a shell command.""" - print(f"Running: {cmd if isinstance(cmd, str) else ' '.join(cmd)}") - result = subprocess.run( - cmd if isinstance(cmd, list) else cmd.split(), - capture_output=capture_output, - text=True, - check=check - ) - return result - -def run_gcloud_ssh(vm_name, zone, command, check=True): - """Run command on GCP VM via SSH.""" - cmd = [ - "gcloud", "compute", "ssh", vm_name, - "--zone", zone, - "--command", command - ] - print(f"SSH: {command}") - result = subprocess.run(cmd, capture_output=True, text=True) - if check and result.returncode != 0: - print(f"โŒ Error: {result.stderr}") - sys.exit(1) - return result - -def check_vm_exists(vm_name, zone): - """Check if VM exists and is running.""" - result = run_command( - f"gcloud compute instances describe {vm_name} --zone {zone}", - check=False - ) - - if result.returncode != 0: - print(f"โŒ VM '{vm_name}' not found in zone {zone}") - return False - - if "RUNNING" not in result.stdout: - print(f"โš ๏ธ VM exists but is not running. Starting VM...") - run_command(f"gcloud compute instances start {vm_name} --zone {zone}") - print(" Waiting for VM to start...") - time.sleep(30) - - return True - -def create_finetuning_vm(vm_name, zone, machine_type="n1-standard-8"): - """Create a GPU-enabled VM for fine-tuning.""" - print(f"\n๐Ÿ”ง Creating fine-tuning VM: {vm_name}") - - cmd = [ - "gcloud", "compute", "instances", "create", vm_name, - "--project", PROJECT_ID, - "--zone", zone, - "--machine-type", machine_type, - "--accelerator", "type=nvidia-tesla-t4,count=1", - "--image-family", "pytorch-latest-gpu", - "--image-project", "deeplearning-platform-release", - "--boot-disk-size", "200GB", - "--boot-disk-type", "pd-ssd", - "--maintenance-policy", "TERMINATE", - "--metadata", "install-nvidia-driver=True", - "--scopes", "https://www.googleapis.com/auth/cloud-platform" - ] - - result = run_command(cmd, check=False) - - if result.returncode == 0: - print("โœ… VM created successfully") - print(" Waiting for startup (60 seconds)...") - time.sleep(60) - return True - else: - print(f"โŒ Failed to create VM: {result.stderr}") - return False - -def upload_code(vm_name, zone): - """Upload project code to VM.""" - print("\n๐Ÿ“ค Uploading code to VM...") - - # Create remote directory - run_gcloud_ssh(vm_name, zone, f"mkdir -p {REMOTE_DIR}") - - # Upload files - local_dir = Path(__file__).parent.parent - files_to_upload = [ - "src", - "experiments", - "scripts", - "requirements.txt", - "setup.py" - ] - - for item in files_to_upload: - local_path = local_dir / item - if local_path.exists(): - cmd = [ - "gcloud", "compute", "scp", - "--recurse" if local_path.is_dir() else "", - "--zone", zone, - str(local_path), - f"{vm_name}:{REMOTE_DIR}/" - ] - # Remove empty strings - cmd = [c for c in cmd if c] - run_command(cmd) - - print("โœ… Code uploaded") - -def install_dependencies(vm_name, zone): - """Install dependencies on VM.""" - print("\n๐Ÿ“ฆ Installing dependencies...") - - commands = [ - "sudo apt-get update", - f"cd {REMOTE_DIR} && pip install -U pip", - f"cd {REMOTE_DIR} && pip install -r requirements.txt", - "pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118", - "pip install transformers[torch] datasets accelerate" - ] - - for cmd in commands: - run_gcloud_ssh(vm_name, zone, cmd, check=False) - - print("โœ… Dependencies installed") - -def verify_gpu(vm_name, zone): - """Verify GPU is available.""" - print("\n๐Ÿ” Verifying GPU...") - - result = run_gcloud_ssh( - vm_name, zone, - "python3 -c 'import torch; print(f\"CUDA: {torch.cuda.is_available()}\"); " - "print(f\"GPU: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else \\\"None\\\"}\")'", - check=False - ) - - print(result.stdout) - - if "CUDA: True" in result.stdout: - print("โœ… GPU is available") - return True - else: - print("โš ๏ธ GPU not detected") - return False - -def prepare_finetuning_dataset(vm_name, zone, job_id=None): - """Prepare fine-tuning dataset on VM.""" - print("\n๐Ÿ“Š Preparing fine-tuning dataset...") - - script = f""" -import sys -sys.path.append('{REMOTE_DIR}') - -from src.data.data_manager import DataManager -from src.data.finetuning_pipeline import FinetuningDatasetPipeline - -# Initialize -data_manager = DataManager(use_gcs=True) -pipeline = FinetuningDatasetPipeline(data_manager, use_gcs=True) - -# Prepare dataset -dataset_info = pipeline.prepare_dataset( - min_error_score=0.5, - balance_error_types=True -) - -print(f"Dataset prepared: {{dataset_info['dataset_id']}}") -print(f"Training samples: {{dataset_info['split_sizes']['train']}}") -""" - - # Write script to file - script_path = "/tmp/prepare_dataset.py" - with open(script_path, 'w') as f: - f.write(script) - - # Upload and run script - run_command(f"gcloud compute scp --zone {zone} {script_path} {vm_name}:{REMOTE_DIR}/prepare_dataset.py") - result = run_gcloud_ssh(vm_name, zone, f"cd {REMOTE_DIR} && python3 prepare_dataset.py") - - print(result.stdout) - - return result.returncode == 0 - -def run_finetuning(vm_name, zone, dataset_id, model_name="openai/whisper-base", epochs=3): - """Run fine-tuning on VM.""" - print(f"\n๐Ÿš€ Starting fine-tuning...") - print(f" Dataset: {dataset_id}") - print(f" Base model: {model_name}") - print(f" Epochs: {epochs}") - - training_script = f""" -import sys -import torch -from pathlib import Path -sys.path.append('{REMOTE_DIR}') - -from transformers import ( - WhisperProcessor, - WhisperForConditionalGeneration, - Seq2SeqTrainingArguments, - Seq2SeqTrainer -) -from datasets import load_dataset - -print("Loading dataset...") -dataset_path = Path("{REMOTE_DIR}/data/finetuning/{dataset_id}") - -# Load dataset -dataset = load_dataset('json', data_files={{ - 'train': str(dataset_path / 'train.jsonl'), - 'validation': str(dataset_path / 'val.jsonl') -}}) - -print(f"Training samples: {{len(dataset['train'])}}") -print(f"Validation samples: {{len(dataset['validation'])}}") - -# Load model and processor -print("Loading model: {model_name}") -processor = WhisperProcessor.from_pretrained("{model_name}") -model = WhisperForConditionalGeneration.from_pretrained("{model_name}") - -# Move to GPU -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -model = model.to(device) -print(f"Using device: {{device}}") - -# Training arguments -output_dir = "{REMOTE_DIR}/models/finetuned_{dataset_id}" -training_args = Seq2SeqTrainingArguments( - output_dir=output_dir, - per_device_train_batch_size=4, - per_device_eval_batch_size=4, - gradient_accumulation_steps=2, - learning_rate=1e-5, - warmup_steps=500, - max_steps={epochs * 1000}, - eval_strategy="steps", - eval_steps=500, - save_steps=1000, - logging_steps=100, - fp16=True, - predict_with_generate=True, - generation_max_length=225, - save_total_limit=2, - load_best_model_at_end=True, -) - -print("Starting training...") -print(f"Output directory: {{output_dir}}") - -# Note: This is a simplified example -# For production, you'd need proper data collator and preprocessing -print("โœ… Training configuration prepared") -print("โš ๏ธ Full training implementation requires additional setup") -print(f"Model will be saved to: {{output_dir}}") -""" - - # Write and upload script - script_path = "/tmp/run_finetuning.py" - with open(script_path, 'w') as f: - f.write(training_script) - - run_command(f"gcloud compute scp --zone {zone} {script_path} {vm_name}:{REMOTE_DIR}/run_finetuning.py") - - # Run training (in background) - print("Launching training job...") - result = run_gcloud_ssh( - vm_name, zone, - f"cd {REMOTE_DIR} && nohup python3 run_finetuning.py > finetuning.log 2>&1 &", - check=False - ) - - print("โœ… Training job launched") - print(f" Monitor with: gcloud compute ssh {vm_name} --zone {zone} --command 'tail -f {REMOTE_DIR}/finetuning.log'") - - return True - -def download_model(vm_name, zone, model_path, local_dest): - """Download trained model from VM.""" - print(f"\n๐Ÿ“ฅ Downloading model from VM...") - - local_dest = Path(local_dest) - local_dest.mkdir(parents=True, exist_ok=True) - - cmd = [ - "gcloud", "compute", "scp", - "--recurse", - "--zone", zone, - f"{vm_name}:{model_path}", - str(local_dest) - ] - - result = run_command(cmd, check=False) - - if result.returncode == 0: - print(f"โœ… Model downloaded to {local_dest}") - return True - else: - print("โŒ Failed to download model") - return False - -def stop_vm(vm_name, zone): - """Stop VM to save costs.""" - print(f"\nโธ๏ธ Stopping VM: {vm_name}") - result = run_command(f"gcloud compute instances stop {vm_name} --zone {zone}", check=False) - - if result.returncode == 0: - print("โœ… VM stopped") - else: - print("โš ๏ธ Failed to stop VM") - -def delete_vm(vm_name, zone): - """Delete VM.""" - print(f"\n๐Ÿ—‘๏ธ Deleting VM: {vm_name}") - result = run_command(f"gcloud compute instances delete {vm_name} --zone {zone} --quiet", check=False) - - if result.returncode == 0: - print("โœ… VM deleted") - else: - print("โš ๏ธ Failed to delete VM") - -def main(): - parser = argparse.ArgumentParser(description="Deploy and run fine-tuning on GCP") - parser.add_argument("--create-vm", action="store_true", help="Create new VM") - parser.add_argument("--vm-name", default=VM_NAME, help="VM name") - parser.add_argument("--zone", default=ZONE, help="GCP zone") - parser.add_argument("--machine-type", default="n1-standard-8", help="Machine type") - parser.add_argument("--skip-upload", action="store_true", help="Skip code upload") - parser.add_argument("--skip-install", action="store_true", help="Skip dependency installation") - parser.add_argument("--prepare-dataset", action="store_true", help="Prepare dataset") - parser.add_argument("--run-training", action="store_true", help="Run training") - parser.add_argument("--dataset-id", help="Dataset ID for training") - parser.add_argument("--model-name", default="openai/whisper-base", help="Base model") - parser.add_argument("--epochs", type=int, default=3, help="Training epochs") - parser.add_argument("--download-model", help="Download model from path") - parser.add_argument("--local-dest", default="./models", help="Local destination for model") - parser.add_argument("--stop-vm", action="store_true", help="Stop VM after completion") - parser.add_argument("--delete-vm", action="store_true", help="Delete VM after completion") - - args = parser.parse_args() - - global VM_NAME, ZONE - VM_NAME = args.vm_name - ZONE = args.zone - - print("="*80) - print("GCP FINE-TUNING DEPLOYMENT") - print("="*80) - - # Create VM if requested - if args.create_vm: - if not create_finetuning_vm(VM_NAME, ZONE, args.machine_type): - sys.exit(1) - - # Check VM exists - if not check_vm_exists(VM_NAME, ZONE): - print("\n๐Ÿ’ก Create VM with --create-vm flag") - sys.exit(1) - - # Upload code - if not args.skip_upload: - upload_code(VM_NAME, ZONE) - - # Install dependencies - if not args.skip_install: - install_dependencies(VM_NAME, ZONE) - - # Verify GPU - verify_gpu(VM_NAME, ZONE) - - # Prepare dataset - if args.prepare_dataset: - prepare_finetuning_dataset(VM_NAME, ZONE) - - # Run training - if args.run_training: - if not args.dataset_id: - print("โŒ --dataset-id required for training") - sys.exit(1) - run_finetuning(VM_NAME, ZONE, args.dataset_id, args.model_name, args.epochs) - - # Download model - if args.download_model: - download_model(VM_NAME, ZONE, args.download_model, args.local_dest) - - # Stop VM - if args.stop_vm: - stop_vm(VM_NAME, ZONE) - - # Delete VM - if args.delete_vm: - delete_vm(VM_NAME, ZONE) - - print("\nโœ… Operations completed!") - print(f"\n๐Ÿ’ก SSH to VM: gcloud compute ssh {VM_NAME} --zone {ZONE}") - print(f"๐Ÿ’ก Monitor costs: gcloud billing accounts list") - -if __name__ == "__main__": - main() - - diff --git a/scripts/deploy_to_gcp.py b/scripts/deploy_to_gcp.py deleted file mode 100755 index 6319d13..0000000 --- a/scripts/deploy_to_gcp.py +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env python3 -""" -Deploy evaluation framework and code to GCP VM. -Uploads code, datasets, and runs evaluation on GPU-enabled VM. -""" - -import sys -import subprocess -import os -from pathlib import Path -import argparse - -# Configuration -PROJECT_ID = "stt-agentic-ai-2025" -ZONE = "us-central1-a" -VM_NAME = "stt-gpu-vm" -REMOTE_DIR = "~/stt-project" - -def run_gcloud_command(cmd, check=True): - """Run a gcloud command.""" - full_cmd = ["gcloud", "compute", "ssh", VM_NAME, "--zone", ZONE, "--command", cmd] - print(f"Running: {' '.join(full_cmd[:4])} ... {cmd}") - result = subprocess.run(full_cmd, capture_output=True, text=True) - if check and result.returncode != 0: - print(f"โŒ Error: {result.stderr}") - sys.exit(1) - return result - -def check_vm_exists(): - """Check if VM exists and is running.""" - result = subprocess.run( - ["gcloud", "compute", "instances", "describe", VM_NAME, "--zone", ZONE], - capture_output=True, - text=True - ) - if result.returncode != 0: - print(f"โŒ VM '{VM_NAME}' not found in zone {ZONE}") - print(" Run 'bash scripts/setup_gcp_gpu.sh' first to create the VM") - return False - - # Check if running - if "RUNNING" not in result.stdout: - print(f"โš ๏ธ VM exists but is not running. Starting VM...") - subprocess.run(["gcloud", "compute", "instances", "start", VM_NAME, "--zone", ZONE]) - print(" Waiting for VM to start (30 seconds)...") - import time - time.sleep(30) - - return True - -def upload_code(): - """Upload project code to VM.""" - print("\n๐Ÿ“ค Uploading code to VM...") - - # Create remote directory - run_gcloud_command(f"mkdir -p {REMOTE_DIR}") - - # Use gcloud compute scp to upload - local_dir = Path(__file__).parent.parent - subprocess.run([ - "gcloud", "compute", "scp", - "--recurse", - "--zone", ZONE, - str(local_dir / "src"), - str(local_dir / "experiments"), - str(local_dir / "scripts"), - str(local_dir / "requirements.txt"), - f"{VM_NAME}:{REMOTE_DIR}/" - ], check=True) - - print("โœ… Code uploaded successfully") - -def install_dependencies(): - """Install Python dependencies on VM.""" - print("\n๐Ÿ“ฆ Installing dependencies on VM...") - - run_gcloud_command( - f"cd {REMOTE_DIR} && pip install -q -r requirements.txt", - check=False # May have some warnings - ) - - print("โœ… Dependencies installed") - -def verify_gpu(): - """Verify GPU is available on VM.""" - print("\n๐Ÿ” Verifying GPU access...") - - result = run_gcloud_command( - "python3 -c 'import torch; print(f\"CUDA available: {torch.cuda.is_available()}\"); " - "print(f\"GPU count: {torch.cuda.device_count()}\" if torch.cuda.is_available() else \"\")'", - check=False - ) - - print(result.stdout) - if "CUDA available: True" in result.stdout: - print("โœ… GPU is available!") - else: - print("โš ๏ธ GPU not detected. Check NVIDIA drivers.") - -def run_evaluation(): - """Run evaluation framework on VM.""" - print("\n๐Ÿš€ Running evaluation framework on GPU...") - - result = run_gcloud_command( - f"cd {REMOTE_DIR} && python3 experiments/kavya_evaluation_framework.py", - check=False - ) - - print(result.stdout) - if result.stderr: - print("Errors:", result.stderr) - -def download_results(): - """Download evaluation results from VM.""" - print("\n๐Ÿ“ฅ Downloading results from VM...") - - local_output = Path(__file__).parent.parent / "experiments" / "evaluation_outputs" - local_output.mkdir(parents=True, exist_ok=True) - - subprocess.run([ - "gcloud", "compute", "scp", - "--recurse", - "--zone", ZONE, - f"{VM_NAME}:{REMOTE_DIR}/experiments/evaluation_outputs/*", - str(local_output) - ], check=False) # May not exist yet - - print("โœ… Results downloaded") - -def main(): - parser = argparse.ArgumentParser(description="Deploy and run STT evaluation on GCP GPU VM") - parser.add_argument("--skip-upload", action="store_true", help="Skip code upload") - parser.add_argument("--skip-install", action="store_true", help="Skip dependency installation") - parser.add_argument("--skip-eval", action="store_true", help="Skip running evaluation") - parser.add_argument("--vm-name", default=VM_NAME, help="VM name") - parser.add_argument("--zone", default=ZONE, help="GCP zone") - - args = parser.parse_args() - - global VM_NAME, ZONE - VM_NAME = args.vm_name - ZONE = args.zone - - print("="*60) - print("GCP Deployment Script") - print("="*60) - - if not check_vm_exists(): - sys.exit(1) - - if not args.skip_upload: - upload_code() - - if not args.skip_install: - install_dependencies() - - verify_gpu() - - if not args.skip_eval: - run_evaluation() - download_results() - - print("\nโœ… Deployment complete!") - print(f"\n๐Ÿ’ก To SSH into VM: gcloud compute ssh {VM_NAME} --zone {ZONE}") - -if __name__ == "__main__": - main() - diff --git a/scripts/monitor_gcp_costs.py b/scripts/monitor_gcp_costs.py deleted file mode 100755 index 76c226f..0000000 --- a/scripts/monitor_gcp_costs.py +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env python3 -""" -Monitor GCP costs and usage for the STT project. -Shows current VM usage, storage costs, and provides cost estimates. -""" - -import subprocess -import json -from datetime import datetime, timedelta -from typing import Dict, List - -PROJECT_ID = "stt-agentic-ai-2025" -VM_NAME = "stt-gpu-vm" -ZONE = "us-central1-a" - -def get_vm_status() -> Dict: - """Get VM status and uptime.""" - try: - result = subprocess.run( - ["gcloud", "compute", "instances", "describe", VM_NAME, - "--zone", ZONE, "--format", "json"], - capture_output=True, - text=True, - check=True - ) - vm_info = json.loads(result.stdout) - - status = vm_info.get("status", "UNKNOWN") - machine_type = vm_info.get("machineType", "").split("/")[-1] - - # Get creation timestamp - creation_time = vm_info.get("creationTimestamp", "") - - return { - "status": status, - "machine_type": machine_type, - "created": creation_time, - "exists": True - } - except subprocess.CalledProcessError: - return {"exists": False} - except FileNotFoundError: - print("โŒ gcloud CLI not found") - return {} - -def estimate_vm_cost(hours: float, machine_type: str = "n1-standard-4") -> float: - """Estimate VM cost based on machine type.""" - # Approximate hourly costs (as of 2024) - costs = { - "n1-standard-4": 0.19, # $/hour - "n1-standard-8": 0.38, - "n1-highmem-4": 0.24, - "n1-highmem-8": 0.48, - } - - base_cost = costs.get(machine_type, 0.19) - gpu_cost = 0.35 # T4 GPU cost per hour - - return (base_cost + gpu_cost) * hours - -def get_storage_usage() -> Dict: - """Get GCS storage usage.""" - buckets = [ - "stt-project-datasets", - "stt-project-models", - "stt-project-logs" - ] - - total_size_gb = 0 - bucket_sizes = {} - - for bucket in buckets: - try: - result = subprocess.run( - ["gsutil", "du", "-sh", f"gs://{bucket}"], - capture_output=True, - text=True, - check=True - ) - # Parse output (format: "SIZE\tgs://bucket") - size_str = result.stdout.split()[0] - if "G" in size_str: - size_gb = float(size_str.replace("G", "")) - elif "M" in size_str: - size_gb = float(size_str.replace("M", "")) / 1024 - else: - size_gb = 0 - - bucket_sizes[bucket] = size_gb - total_size_gb += size_gb - except (subprocess.CalledProcessError, FileNotFoundError): - bucket_sizes[bucket] = 0 - - # Storage cost: ~$0.02/GB/month - monthly_storage_cost = total_size_gb * 0.02 - - return { - "total_gb": total_size_gb, - "bucket_sizes": bucket_sizes, - "monthly_cost": monthly_storage_cost - } - -def get_billing_info() -> Dict: - """Get current billing information.""" - try: - result = subprocess.run( - ["gcloud", "billing", "accounts", "list", "--format", "json"], - capture_output=True, - text=True, - check=True - ) - accounts = json.loads(result.stdout) - return {"accounts": accounts} - except (subprocess.CalledProcessError, FileNotFoundError): - return {} - -def calculate_uptime_hours(creation_time: str, status: str) -> float: - """Calculate VM uptime in hours.""" - if status != "RUNNING": - return 0.0 - - try: - from dateutil import parser - created = parser.parse(creation_time) - now = datetime.now(created.tzinfo) - delta = now - created - return delta.total_seconds() / 3600 - except: - return 0.0 - -def main(): - print("="*60) - print("GCP Cost & Usage Monitor") - print("="*60) - print(f"Project: {PROJECT_ID}\n") - - # VM Status - print("๐Ÿ–ฅ๏ธ VM Status:") - print("-" * 60) - vm_status = get_vm_status() - - if not vm_status.get("exists"): - print(f" โŒ VM '{VM_NAME}' not found") - print(f" Run 'bash scripts/setup_gcp_gpu.sh' to create it") - else: - status = vm_status.get("status", "UNKNOWN") - machine_type = vm_status.get("machine_type", "unknown") - created = vm_status.get("created", "") - - print(f" Status: {status}") - print(f" Machine Type: {machine_type}") - print(f" Created: {created}") - - if status == "RUNNING": - uptime_hours = calculate_uptime_hours(created, status) - estimated_cost = estimate_vm_cost(uptime_hours, machine_type) - print(f" Uptime: {uptime_hours:.2f} hours") - print(f" Estimated Cost: ${estimated_cost:.2f}") - print(f" ๐Ÿ’ก Stop VM when not in use to save costs!") - - # Storage Usage - print("\n๐Ÿ’พ Storage Usage:") - print("-" * 60) - storage = get_storage_usage() - print(f" Total Storage: {storage['total_gb']:.2f} GB") - print(f" Monthly Cost: ${storage['monthly_cost']:.2f}") - print("\n Per Bucket:") - for bucket, size in storage['bucket_sizes'].items(): - print(f" {bucket}: {size:.2f} GB") - - # Cost Estimates - print("\n๐Ÿ’ฐ Cost Estimates:") - print("-" * 60) - print(" VM (T4 GPU + n1-standard-4):") - print(" - Per hour: ~$0.54") - print(" - Per day (24h): ~$12.96") - print(" - Per month (730h): ~$394.20") - print("\n Storage:") - print(f" - Current: ${storage['monthly_cost']:.2f}/month") - print("\n ๐Ÿ’ก Tips to Save:") - print(" - Use preemptible instances: 60-80% cheaper") - print(" - Stop VMs when not in use") - print(" - Use smaller GPUs (T4) for development") - print(" - Clean up old model checkpoints") - - # Billing - print("\n๐Ÿ’ณ Billing:") - print("-" * 60) - billing = get_billing_info() - if billing.get("accounts"): - print(f" Found {len(billing['accounts'])} billing account(s)") - print(" View detailed costs: https://console.cloud.google.com/billing") - else: - print(" โš ๏ธ Could not retrieve billing info") - - print("\n" + "="*60) - -if __name__ == "__main__": - main() - diff --git a/scripts/quick_setup.sh b/scripts/quick_setup.sh deleted file mode 100755 index 4fc8f0c..0000000 --- a/scripts/quick_setup.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash -# Quick setup script - Checks prerequisites and guides through GCP setup - -set -e - -echo "==========================================" -echo "GCP Setup Prerequisites Check" -echo "==========================================" -echo "" - -# Check if gcloud is installed -if command -v gcloud &> /dev/null; then - echo "โœ… gcloud CLI is installed" - gcloud --version | head -1 -else - echo "โŒ gcloud CLI not found" - echo "" - echo "Please install gcloud CLI first:" - echo "" - echo "Option 1 (macOS - Recommended):" - echo " brew install --cask google-cloud-sdk" - echo "" - echo "Option 2 (Direct):" - echo " curl https://sdk.cloud.google.com | bash" - echo "" - echo "See scripts/INSTALL_GCLOUD.md for detailed instructions" - echo "" - exit 1 -fi - -echo "" -echo "Checking authentication..." -if gcloud auth list --filter=status:ACTIVE --format="value(account)" &>/dev/null; then - ACCOUNT=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" | head -1) - echo "โœ… Authenticated as: $ACCOUNT" -else - echo "โš ๏ธ Not authenticated" - echo " Run: gcloud auth login" - exit 1 -fi - -echo "" -echo "Checking project..." -PROJECT=$(gcloud config get-value project 2>/dev/null) -if [ -n "$PROJECT" ]; then - echo "โœ… Project set to: $PROJECT" -else - echo "โš ๏ธ No project set" - echo " Run: gcloud config set project YOUR_PROJECT_ID" - echo " Or update PROJECT_ID in scripts/setup_gcp_gpu.sh" - exit 1 -fi - -echo "" -echo "Checking required APIs..." -APIS_ENABLED=$(gcloud services list --enabled --filter="name:compute.googleapis.com OR name:storage-api.googleapis.com" --format="value(name)" 2>/dev/null | wc -l) -if [ "$APIS_ENABLED" -ge 1 ]; then - echo "โœ… Required APIs appear to be enabled" -else - echo "โš ๏ธ Enabling required APIs..." - gcloud services enable compute.googleapis.com - gcloud services enable storage-api.googleapis.com - echo "โœ… APIs enabled" -fi - -echo "" -echo "==========================================" -echo "โœ… All prerequisites met!" -echo "==========================================" -echo "" -echo "You can now run:" -echo " bash scripts/setup_gcp_gpu.sh" -echo "" -echo "Or check GPU quota first:" -echo " gcloud compute project-info describe --project=$PROJECT | grep -i quota" -echo "" - diff --git a/scripts/setup_environment.py b/scripts/setup_environment.py deleted file mode 100644 index 0b31ff0..0000000 --- a/scripts/setup_environment.py +++ /dev/null @@ -1,239 +0,0 @@ -#!/usr/bin/env python3 -""" -Setup script for STT Agentic AI development environment. -Configures Google Cloud Storage access, verifies GPU, and installs dependencies. -""" - -import os -import sys -import subprocess -import json -from pathlib import Path - -# Google Cloud configuration -GCP_PROJECT_ID = "stt-agentic-ai-2025" -GCS_BUCKETS = { - "datasets": "gs://stt-project-datasets", - "models": "gs://stt-project-models", - "logs": "gs://stt-project-logs" -} - -def check_python_version(): - """Verify Python version is 3.8+""" - print("ํฐ Checking Python version...") - version = sys.version_info - if version.major < 3 or (version.major == 3 and version.minor < 8): - print("โŒ Python 3.8+ required. Current version:", sys.version) - return False - print(f"โœ“ Python {version.major}.{version.minor}.{version.micro}") - return True - -def install_dependencies(): - """Install required Python packages""" - print("\nํณฆ Installing dependencies...") - try: - subprocess.run([ - sys.executable, "-m", "pip", "install", "-r", "requirements.txt" - ], check=True) - print("โœ“ Dependencies installed successfully") - return True - except subprocess.CalledProcessError as e: - print(f"โŒ Failed to install dependencies: {e}") - return False - -def verify_gpu_access(): - """Check if GPU is available""" - print("\nํพฎ Checking GPU access...") - try: - import torch - if torch.cuda.is_available(): - gpu_name = torch.cuda.get_device_name(0) - gpu_count = torch.cuda.device_count() - print(f"โœ“ GPU detected: {gpu_name}") - print(f"โœ“ GPU count: {gpu_count}") - print(f"โœ“ CUDA version: {torch.version.cuda}") - return True - else: - print("โš ๏ธ No GPU detected. Running on CPU only.") - return False - except ImportError: - print("โš ๏ธ PyTorch not installed yet. Install dependencies first.") - return False - -def setup_gcp_credentials(): - """Configure Google Cloud credentials""" - print("\nโ˜๏ธ Setting up Google Cloud credentials...") - - # Check if gcloud is installed - try: - result = subprocess.run( - ["gcloud", "version"], - capture_output=True, - text=True, - check=True - ) - print("โœ“ gcloud CLI detected") - except (subprocess.CalledProcessError, FileNotFoundError): - print("โŒ gcloud CLI not found. Install from: https://cloud.google.com/sdk/install") - return False - - # Set project - try: - subprocess.run( - ["gcloud", "config", "set", "project", GCP_PROJECT_ID], - check=True - ) - print(f"โœ“ GCP project set to: {GCP_PROJECT_ID}") - except subprocess.CalledProcessError as e: - print(f"โŒ Failed to set GCP project: {e}") - return False - - # Authenticate application default credentials - try: - subprocess.run( - ["gcloud", "auth", "application-default", "login"], - check=True - ) - print("โœ“ Application default credentials configured") - except subprocess.CalledProcessError as e: - print(f"โš ๏ธ Authentication may be required: {e}") - - return True - -def verify_gcs_buckets(): - """Verify access to Google Cloud Storage buckets""" - print("\nํบฃ Verifying GCS bucket access...") - - try: - from google.cloud import storage - client = storage.Client(project=GCP_PROJECT_ID) - - for bucket_name, bucket_path in GCS_BUCKETS.items(): - bucket_id = bucket_path.replace("gs://", "") - try: - bucket = client.get_bucket(bucket_id) - print(f"โœ“ Access verified: {bucket_path}") - except Exception as e: - print(f"โŒ Cannot access {bucket_path}: {e}") - return False - - return True - except ImportError: - print("โš ๏ธ google-cloud-storage not installed. Install dependencies first.") - return False - except Exception as e: - print(f"โŒ GCS verification failed: {e}") - return False - -def create_local_directories(): - """Create necessary local directories""" - print("\nํณ Creating local directories...") - - dirs = [ - "data/raw", - "data/processed", - "data/augmented", - "data/evaluation", - "logs", - "models", - "checkpoints" - ] - - for dir_path in dirs: - Path(dir_path).mkdir(parents=True, exist_ok=True) - - print(f"โœ“ Created {len(dirs)} directories") - return True - -def create_config_file(): - """Create configuration file with GCS paths""" - print("\nโš™๏ธ Creating configuration file...") - - config = { - "gcp": { - "project_id": GCP_PROJECT_ID, - "buckets": GCS_BUCKETS - }, - "data": { - "raw_data_dir": "data/raw", - "processed_data_dir": "data/processed", - "augmented_data_dir": "data/augmented", - "evaluation_data_dir": "data/evaluation" - }, - "model": { - "checkpoint_dir": "checkpoints", - "model_dir": "models" - }, - "training": { - "batch_size": 16, - "learning_rate": 5e-5, - "num_epochs": 3 - } - } - - config_path = Path("configs/config.yaml") - config_path.parent.mkdir(parents=True, exist_ok=True) - - import yaml - with open(config_path, "w") as f: - yaml.dump(config, f, default_flow_style=False) - - print(f"โœ“ Configuration saved to {config_path}") - return True - -def print_summary(): - """Print setup summary""" - print("\n" + "="*60) - print("ํพ‰ Environment Setup Complete!") - print("="*60) - print("\nํณ‹ Next Steps:") - print("1. Run: python scripts/download_datasets.py") - print("2. Run: python scripts/preprocess_data.py") - print("3. Start development!") - print("\nํณš Resources:") - print(f"- GCP Project: {GCP_PROJECT_ID}") - print(f"- Dataset bucket: {GCS_BUCKETS['datasets']}") - print(f"- Model bucket: {GCS_BUCKETS['models']}") - print("\nํฒก Tip: Use 'python scripts/verify_setup.py' to check your setup anytime") - -def main(): - """Main setup routine""" - print("="*60) - print("ํบ€ STT Agentic AI Environment Setup") - print("="*60) - - steps = [ - ("Python version", check_python_version), - ("Dependencies", install_dependencies), - ("GPU access", verify_gpu_access), - ("GCP credentials", setup_gcp_credentials), - ("GCS buckets", verify_gcs_buckets), - ("Local directories", create_local_directories), - ("Configuration file", create_config_file), - ] - - results = [] - for step_name, step_func in steps: - try: - success = step_func() - results.append((step_name, success)) - except Exception as e: - print(f"โŒ Error in {step_name}: {e}") - results.append((step_name, False)) - - print("\n" + "="*60) - print("ํณŠ Setup Summary") - print("="*60) - for step_name, success in results: - status = "โœ“" if success else "โŒ" - print(f"{status} {step_name}") - - if all(success for _, success in results): - print_summary() - return 0 - else: - print("\nโš ๏ธ Some steps failed. Please review errors above.") - return 1 - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/setup_gcp_gpu.sh b/scripts/setup_gcp_gpu.sh old mode 100755 new mode 100644 diff --git a/scripts/setup_path.sh b/scripts/setup_path.sh deleted file mode 100755 index be3a006..0000000 --- a/scripts/setup_path.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# Add gcloud to PATH (run this before using gcloud commands) - -export PATH=/usr/local/share/google-cloud-sdk/bin:"$PATH" - -echo "โœ… Added gcloud to PATH" -echo "Run this before using gcloud commands, or add to your ~/.zshrc:" -echo "" -echo "export PATH=/usr/local/share/google-cloud-sdk/bin:\"\$PATH\"" -echo "" - diff --git a/scripts/verify_setup.py b/scripts/verify_setup.py deleted file mode 100644 index 1f7703b..0000000 --- a/scripts/verify_setup.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python3 -""" -Verification script to check complete setup. -""" - -import sys -from pathlib import Path - -sys.path.append(str(Path(__file__).parent.parent)) -from src.utils.gcs_utils import get_gcs_manager - -def verify_local_data(): - """Verify local data directories""" - print("ํณ Checking local data directories...") - - required_dirs = [ - "data/raw", - "data/processed", - "data/evaluation" - ] - - all_exist = True - for dir_path in required_dirs: - exists = Path(dir_path).exists() - status = "โœ“" if exists else "โŒ" - print(f" {status} {dir_path}") - all_exist = all_exist and exists - - return all_exist - -def verify_gcs_data(): - """Verify data in GCS""" - print("\nโ˜๏ธ Checking GCS buckets...") - - try: - gcs_manager = get_gcs_manager("datasets") - - prefixes = ["raw/", "processed/", "evaluation/"] - - for prefix in prefixes: - files = gcs_manager.list_files(prefix) - status = "โœ“" if files else "โš ๏ธ " - print(f" {status} {prefix}: {len(files)} files") - - return True - except Exception as e: - print(f" โŒ Error accessing GCS: {e}") - return False - -def main(): - print("="*60) - print("ํด Setup Verification") - print("="*60) - - local_ok = verify_local_data() - gcs_ok = verify_gcs_data() - - print("\n" + "="*60) - if local_ok and gcs_ok: - print("โœ“ Setup verified successfully!") - else: - print("โš ๏ธ Some issues detected. Review output above.") - print("="*60) - - return 0 if (local_ok and gcs_ok) else 1 - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/__pycache__/__init__.cpython-313.pyc b/src/__pycache__/__init__.cpython-313.pyc index 6981baa..3f69095 100644 Binary files a/src/__pycache__/__init__.cpython-313.pyc and b/src/__pycache__/__init__.cpython-313.pyc differ diff --git a/src/agent_evaluation/__init__.py b/src/agent_evaluation/__init__.py deleted file mode 100644 index 497e17f..0000000 --- a/src/agent_evaluation/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -""" -Agent Evaluation Framework - Week 2 - -Comprehensive evaluation of agent correction accuracy, false positives, -ablation testing, and latency benchmarking. -""" - -from .agent_evaluator import AgentEvaluator -from .ablation_tester import AblationTester -from .agent_benchmark import AgentBenchmark -from .false_positive_detector import FalsePositiveDetector - -__all__ = [ - 'AgentEvaluator', - 'AblationTester', - 'AgentBenchmark', - 'FalsePositiveDetector' -] \ No newline at end of file diff --git a/src/agent_evaluation/ablation_tester.py b/src/agent_evaluation/ablation_tester.py deleted file mode 100644 index 1a99c12..0000000 --- a/src/agent_evaluation/ablation_tester.py +++ /dev/null @@ -1,272 +0,0 @@ -""" -Ablation Testing Framework - Isolate Agent Impact - -Tests agent components individually to understand their contribution. -""" - -import logging -from typing import Dict, List, Optional -from dataclasses import dataclass -import json -from pathlib import Path -from datetime import datetime - -from jiwer import wer, cer -import numpy as np - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -@dataclass -class AblationResult: - """Result of an ablation test""" - test_name: str - description: str - - # What was tested - baseline_enabled: bool - agent_enabled: bool - error_detection_enabled: bool - auto_correction_enabled: bool - - # Results - mean_wer: float - mean_cer: float - mean_inference_time: float - samples_evaluated: int - - def to_dict(self) -> Dict: - return { - 'test_name': self.test_name, - 'description': self.description, - 'baseline_enabled': self.baseline_enabled, - 'agent_enabled': self.agent_enabled, - 'error_detection_enabled': self.error_detection_enabled, - 'auto_correction_enabled': self.auto_correction_enabled, - 'mean_wer': self.mean_wer, - 'mean_cer': self.mean_cer, - 'mean_inference_time': self.mean_inference_time, - 'samples_evaluated': self.samples_evaluated - } - - -class AblationTester: - """ - Performs ablation testing to isolate agent impact. - - Tests: - 1. Baseline only (no agent) - 2. Baseline + Error detection (no correction) - 3. Baseline + Error detection + Auto-correction (full agent) - 4. Error detection only (no baseline) - - Metrics: - - WER/CER improvement at each stage - - Latency overhead at each stage - - Consistency across configurations - """ - - def __init__(self, - agent=None, - baseline_model=None, - output_dir: str = "experiments/evaluation_outputs"): - """Initialize ablation tester""" - self.agent = agent - self.baseline_model = baseline_model - self.output_dir = Path(output_dir) - self.output_dir.mkdir(parents=True, exist_ok=True) - - self.results: Dict[str, AblationResult] = {} - - logger.info("Ablation Tester initialized") - - def test_baseline_only(self, audio_paths: List[str]) -> AblationResult: - """Test baseline model without agent""" - - logger.info("Testing baseline only (no agent)...") - - wers = [] - cers = [] - times = [] - - for audio_path in audio_paths: - import time - start = time.time() - result = self.baseline_model.transcribe(audio_path) - times.append(time.time() - start) - - # Can't calculate WER/CER without reference, use error score - wers.append(0.0) # Placeholder - cers.append(0.0) - - result = AblationResult( - test_name="baseline_only", - description="Baseline model without agent", - baseline_enabled=True, - agent_enabled=False, - error_detection_enabled=False, - auto_correction_enabled=False, - mean_wer=np.mean(wers) if wers else 0.0, - mean_cer=np.mean(cers) if cers else 0.0, - mean_inference_time=np.mean(times), - samples_evaluated=len(audio_paths) - ) - - self.results['baseline_only'] = result - return result - - def test_with_error_detection(self, audio_paths: List[str]) -> AblationResult: - """Test with error detection but no auto-correction""" - - logger.info("Testing with error detection (no auto-correction)...") - - times = [] - errors_detected = 0 - - for audio_path in audio_paths: - import time - start = time.time() - result = self.agent.transcribe_with_agent( - audio_path, - enable_auto_correction=False - ) - times.append(time.time() - start) - - if result['error_detection']['has_errors']: - errors_detected += 1 - - result = AblationResult( - test_name="with_error_detection", - description="Error detection enabled, auto-correction disabled", - baseline_enabled=True, - agent_enabled=True, - error_detection_enabled=True, - auto_correction_enabled=False, - mean_wer=0.0, # Would need reference for WER - mean_cer=0.0, - mean_inference_time=np.mean(times), - samples_evaluated=len(audio_paths) - ) - - self.results['with_error_detection'] = result - - print(f" Errors detected in {errors_detected}/{len(audio_paths)} samples") - - return result - - def test_full_agent(self, audio_paths: List[str]) -> AblationResult: - """Test full agent with error detection and auto-correction""" - - logger.info("Testing full agent (detection + correction)...") - - times = [] - errors_detected = 0 - corrections_applied = 0 - - for audio_path in audio_paths: - import time - start = time.time() - result = self.agent.transcribe_with_agent( - audio_path, - enable_auto_correction=True - ) - times.append(time.time() - start) - - if result['error_detection']['has_errors']: - errors_detected += 1 - if result['corrections']['applied']: - corrections_applied += 1 - - result = AblationResult( - test_name="full_agent", - description="Full agent with error detection and auto-correction", - baseline_enabled=True, - agent_enabled=True, - error_detection_enabled=True, - auto_correction_enabled=True, - mean_wer=0.0, - mean_cer=0.0, - mean_inference_time=np.mean(times), - samples_evaluated=len(audio_paths) - ) - - self.results['full_agent'] = result - - print(f" Errors detected in {errors_detected}/{len(audio_paths)} samples") - print(f" Corrections applied in {corrections_applied}/{len(audio_paths)} samples") - - return result - - def run_full_ablation(self, audio_paths: List[str]) -> Dict: - """Run complete ablation study""" - - logger.info(f"Running full ablation study on {len(audio_paths)} samples...") - - self.test_baseline_only(audio_paths) - self.test_with_error_detection(audio_paths) - self.test_full_agent(audio_paths) - - return self.compare_configurations() - - def compare_configurations(self) -> Dict: - """Compare all configurations""" - - if not self.results: - return {"error": "No ablation results"} - - comparison = { - "configurations": [r.to_dict() for r in self.results.values()], - "latency_analysis": {}, - "impact_analysis": {} - } - - # Latency comparison - if 'baseline_only' in self.results: - baseline_time = self.results['baseline_only'].mean_inference_time - - for config_name, result in self.results.items(): - if config_name != 'baseline_only': - overhead = (result.mean_inference_time - baseline_time) / baseline_time * 100 - comparison['latency_analysis'][config_name] = { - 'baseline_time_ms': baseline_time * 1000, - 'config_time_ms': result.mean_inference_time * 1000, - 'overhead_percent': overhead - } - - return comparison - - def save_ablation_report(self, filename: str = "ablation_study.json"): - """Save ablation study report""" - - report = { - "timestamp": datetime.now().isoformat(), - "configurations": self.compare_configurations() - } - - output_path = self.output_dir / filename - with open(output_path, 'w') as f: - json.dump(report, f, indent=2) - - logger.info(f"โœ… Ablation report saved to {output_path}") - return output_path - - def print_summary(self): - """Print ablation summary""" - - print("\n" + "="*70) - print("ABLATION TESTING REPORT") - print("="*70) - - for config_name, result in self.results.items(): - print(f"\n{config_name.upper().replace('_', ' ')}:") - print(f" Mean inference time: {result.mean_inference_time*1000:.2f} ms") - print(f" Samples: {result.samples_evaluated}") - - comparison = self.compare_configurations() - if 'latency_analysis' in comparison: - print("\nLatency overhead:") - for config, stats in comparison['latency_analysis'].items(): - print(f" {config}: +{stats['overhead_percent']:.2f}%") - - print("="*70 + "\n") diff --git a/src/agent_evaluation/agent_benchmark.py b/src/agent_evaluation/agent_benchmark.py deleted file mode 100644 index 7a30333..0000000 --- a/src/agent_evaluation/agent_benchmark.py +++ /dev/null @@ -1,219 +0,0 @@ -""" -Agent Benchmarking - Latency and Runtime Performance - -Measures agent performance overhead and scalability. -""" - -import logging -from typing import Dict, List, Optional -import time -from dataclasses import dataclass -import json -from pathlib import Path -from datetime import datetime - -import numpy as np - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -@dataclass -class BenchmarkResult: - """Single benchmark result""" - audio_path: str - baseline_latency_ms: float - agent_latency_ms: float - error_detection_latency_ms: float - correction_latency_ms: float - overhead_percent: float - errors_detected: int - corrections_applied: int - - -class AgentBenchmark: - """ - Benchmark agent performance metrics. - - Metrics: - - Baseline inference latency - - Agent overhead (error detection + correction) - - Breakdown: error detection time vs correction time - - Throughput: samples per second - - Scalability: latency vs audio length - """ - - def __init__(self, - agent=None, - baseline_model=None, - output_dir: str = "experiments/evaluation_outputs"): - """Initialize benchmark""" - self.agent = agent - self.baseline_model = baseline_model - self.output_dir = Path(output_dir) - self.output_dir.mkdir(parents=True, exist_ok=True) - - self.results: List[BenchmarkResult] = [] - - logger.info("Agent Benchmark initialized") - - def benchmark_single(self, audio_path: str) -> BenchmarkResult: - """Benchmark a single audio file""" - - # Baseline latency - start = time.time() - baseline_result = self.baseline_model.transcribe(audio_path) - baseline_latency = (time.time() - start) * 1000 # ms - - # Agent latency (full pipeline) - start = time.time() - agent_result = self.agent.transcribe_with_agent( - audio_path, - enable_auto_correction=True - ) - agent_latency = (time.time() - start) * 1000 # ms - - # Estimate breakdown (rough approximation) - error_detection_latency = agent_latency * 0.3 # Error detection ~30% - correction_latency = agent_latency * 0.7 # Correction ~70% - - overhead = ((agent_latency - baseline_latency) / baseline_latency * 100) \ - if baseline_latency > 0 else 0 - - result = BenchmarkResult( - audio_path=str(audio_path), - baseline_latency_ms=baseline_latency, - agent_latency_ms=agent_latency, - error_detection_latency_ms=error_detection_latency, - correction_latency_ms=correction_latency, - overhead_percent=overhead, - errors_detected=agent_result['error_detection']['error_count'], - corrections_applied=agent_result['corrections']['count'] - ) - - self.results.append(result) - return result - - def benchmark_batch(self, audio_paths: List[str], - verbose: bool = True) -> Dict: - """Benchmark multiple audio files""" - - logger.info(f"Benchmarking {len(audio_paths)} audio files...") - - for idx, audio_path in enumerate(audio_paths): - try: - result = self.benchmark_single(audio_path) - if verbose and (idx + 1) % 10 == 0: - logger.info(f" Completed {idx+1}/{len(audio_paths)}") - except Exception as e: - logger.error(f"Error benchmarking {audio_path}: {e}") - continue - - return self.get_benchmark_summary() - - def get_benchmark_summary(self) -> Dict: - """Get benchmark summary statistics""" - - if not self.results: - return {"error": "No benchmark results"} - - baseline_latencies = [r.baseline_latency_ms for r in self.results] - agent_latencies = [r.agent_latency_ms for r in self.results] - overheads = [r.overhead_percent for r in self.results] - - return { - "total_samples": len(self.results), - "baseline_latency": { - "mean_ms": np.mean(baseline_latencies), - "std_ms": np.std(baseline_latencies), - "min_ms": np.min(baseline_latencies), - "max_ms": np.max(baseline_latencies), - "p95_ms": np.percentile(baseline_latencies, 95), - "p99_ms": np.percentile(baseline_latencies, 99) - }, - "agent_latency": { - "mean_ms": np.mean(agent_latencies), - "std_ms": np.std(agent_latencies), - "min_ms": np.min(agent_latencies), - "max_ms": np.max(agent_latencies), - "p95_ms": np.percentile(agent_latencies, 95), - "p99_ms": np.percentile(agent_latencies, 99) - }, - "overhead": { - "mean_percent": np.mean(overheads), - "std_percent": np.std(overheads), - "min_percent": np.min(overheads), - "max_percent": np.max(overheads) - }, - "throughput": { - "baseline_samples_per_sec": 1000 / np.mean(baseline_latencies), - "agent_samples_per_sec": 1000 / np.mean(agent_latencies) - }, - "detailed_results": [ - { - 'audio_path': r.audio_path, - 'baseline_latency_ms': r.baseline_latency_ms, - 'agent_latency_ms': r.agent_latency_ms, - 'overhead_percent': r.overhead_percent, - 'errors_detected': r.errors_detected, - 'corrections_applied': r.corrections_applied - } - for r in self.results - ] - } - - def save_benchmark_report(self, filename: str = "agent_benchmark.json"): - """Save benchmark report""" - - report = { - "timestamp": datetime.now().isoformat(), - "benchmark_results": self.get_benchmark_summary() - } - - output_path = self.output_dir / filename - with open(output_path, 'w') as f: - json.dump(report, f, indent=2) - - logger.info(f"โœ… Benchmark report saved to {output_path}") - return output_path - - def print_summary(self): - """Print benchmark summary""" - - summary = self.get_benchmark_summary() - - if "error" in summary: - print(f"โŒ {summary['error']}") - return - - print("\n" + "="*70) - print("AGENT LATENCY BENCHMARK REPORT") - print("="*70) - - print(f"\nTotal samples: {summary['total_samples']}") - - print("\nBaseline latency:") - baseline = summary['baseline_latency'] - print(f" Mean: {baseline['mean_ms']:.2f} ms") - print(f" Std: {baseline['std_ms']:.2f} ms") - print(f" P95: {baseline['p95_ms']:.2f} ms") - print(f" P99: {baseline['p99_ms']:.2f} ms") - - print("\nAgent latency (with error detection + correction):") - agent = summary['agent_latency'] - print(f" Mean: {agent['mean_ms']:.2f} ms") - print(f" Std: {agent['std_ms']:.2f} ms") - print(f" P95: {agent['p95_ms']:.2f} ms") - print(f" P99: {agent['p99_ms']:.2f} ms") - - print("\nAgent overhead:") - overhead = summary['overhead'] - print(f" Mean: +{overhead['mean_percent']:.2f}%") - print(f" Range: {overhead['min_percent']:.2f}% to {overhead['max_percent']:.2f}%") - - print("\nThroughput:") - throughput = summary['throughput'] - print(f" Baseline: {throughput['baseline_samples_per_sec']:.2f} samples/sec") - print(f" Agent: {throughput['agent_samples_per_sec']:.2f} samples/sec") - - print("="*70 + "\n") diff --git a/src/agent_evaluation/agent_evaluator.py b/src/agent_evaluation/agent_evaluator.py deleted file mode 100644 index c086d54..0000000 --- a/src/agent_evaluation/agent_evaluator.py +++ /dev/null @@ -1,282 +0,0 @@ -""" -Agent Evaluator - Correction Accuracy and Consistency Metrics - -Evaluates how well the agent corrects transcription errors. -""" - -import logging -from typing import Dict, List, Tuple, Optional -from dataclasses import dataclass, asdict -from pathlib import Path -import json -import time -from datetime import datetime - -import numpy as np -from src.evaluation.metrics import STTEvaluator - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -@dataclass -class CorrectionResult: - """Result of a single correction evaluation""" - audio_path: str - original_transcript: str - corrected_transcript: str - reference_transcript: Optional[str] - - # Error metrics - original_wer: float - corrected_wer: float - original_cer: float - corrected_cer: float - - # Correction impact - wer_improvement: float # negative means worse - cer_improvement: float - was_improved: bool # True if correction helped - - # Agent metadata - errors_detected: int - error_score: float - agent_confidence: Optional[float] - - timestamp: str = "" - - def to_dict(self) -> Dict: - """Convert to dictionary for serialization""" - return asdict(self) - - -class AgentEvaluator: - """ - Evaluates agent correction accuracy and consistency. - - Metrics calculated: - - Correction accuracy: % of corrections that improve WER/CER - - Correction consistency: Std dev of improvements - - Average improvement: Mean WER/CER reduction - - Worsening rate: % of corrections that hurt - """ - - def __init__(self, - agent=None, - baseline_model=None, - output_dir: str = "experiments/evaluation_outputs"): - """ - Initialize evaluator. - - Args: - agent: STTAgent instance - baseline_model: BaselineSTTModel instance - output_dir: Directory to save evaluation results - """ - self.agent = agent - self.baseline_model = baseline_model - self.output_dir = Path(output_dir) - self.output_dir.mkdir(parents=True, exist_ok=True) - - self.results: List[CorrectionResult] = [] - self.evaluator = STTEvaluator() - - logger.info("Agent Evaluator initialized") - - def evaluate_correction(self, - audio_path: str, - reference_transcript: Optional[str] = None, - enable_correction: bool = True) -> CorrectionResult: - """ - Evaluate a single correction. - - Args: - audio_path: Path to audio file - reference_transcript: Ground truth transcription (optional) - enable_correction: Whether to apply agent corrections - - Returns: - CorrectionResult with metrics - """ - from datetime import datetime - - # Get baseline transcription - baseline_result = self.baseline_model.transcribe(audio_path) - original_transcript = baseline_result['transcript'] - - # Get agent transcription - agent_result = self.agent.transcribe_with_agent( - audio_path, - enable_auto_correction=enable_correction - ) - corrected_transcript = agent_result['transcript'] - - # Calculate WER/CER if reference available - original_wer = 0.0 - original_cer = 0.0 - corrected_wer = 0.0 - corrected_cer = 0.0 - - if reference_transcript: - original_wer = self.evaluator.calculate_wer(reference_transcript, original_transcript) - original_cer = self.evaluator.calculate_cer(reference_transcript, original_transcript) - corrected_wer = self.evaluator.calculate_wer(reference_transcript, corrected_transcript) - corrected_cer = self.evaluator.calculate_cer(reference_transcript, corrected_transcript) - - # Calculate improvements (negative means correction made it worse) - wer_improvement = original_wer - corrected_wer - cer_improvement = original_cer - corrected_cer - was_improved = (wer_improvement > 0.01) # 0.01 threshold for significance - - result = CorrectionResult( - audio_path=str(audio_path), - original_transcript=original_transcript, - corrected_transcript=corrected_transcript, - reference_transcript=reference_transcript, - original_wer=original_wer, - corrected_wer=corrected_wer, - original_cer=original_cer, - corrected_cer=corrected_cer, - wer_improvement=wer_improvement, - cer_improvement=cer_improvement, - was_improved=was_improved, - errors_detected=agent_result['error_detection']['error_count'], - error_score=agent_result['error_detection']['error_score'], - agent_confidence=None, - timestamp=datetime.now().isoformat() - ) - - self.results.append(result) - return result - - def evaluate_batch(self, - audio_paths: List[str], - reference_transcripts: Optional[List[str]] = None, - enable_correction: bool = True) -> Dict: - """ - Evaluate a batch of corrections. - - Args: - audio_paths: List of audio file paths - reference_transcripts: List of ground truth transcriptions (optional) - enable_correction: Whether to apply corrections - - Returns: - Dictionary with batch evaluation metrics - """ - logger.info(f"Evaluating {len(audio_paths)} audio files...") - - if reference_transcripts: - assert len(audio_paths) == len(reference_transcripts), \ - "Audio paths and reference transcripts must have same length" - - results = [] - for idx, audio_path in enumerate(audio_paths): - ref = reference_transcripts[idx] if reference_transcripts else None - try: - result = self.evaluate_correction( - audio_path=audio_path, - reference_transcript=ref, - enable_correction=enable_correction - ) - results.append(result) - except Exception as e: - logger.error(f"Error evaluating {audio_path}: {e}") - continue - - return self._calculate_batch_metrics(results) - - def _calculate_batch_metrics(self, results: List[CorrectionResult]) -> Dict: - """Calculate aggregate metrics from batch results""" - - if not results: - return {"error": "No valid results"} - - improved_count = sum(1 for r in results if r.was_improved) - worsened_count = sum(1 for r in results if not r.was_improved and r.wer_improvement < -0.01) - neutral_count = len(results) - improved_count - worsened_count - - wer_improvements = [r.wer_improvement for r in results] - cer_improvements = [r.cer_improvement for r in results] - - return { - "total_samples": len(results), - "improvement_metrics": { - "improved_count": improved_count, - "improvement_rate": improved_count / len(results), - "worsened_count": worsened_count, - "worsening_rate": worsened_count / len(results), - "neutral_count": neutral_count, - "neutral_rate": neutral_count / len(results) - }, - "wer_metrics": { - "mean_improvement": np.mean(wer_improvements), - "std_improvement": np.std(wer_improvements), - "min_improvement": np.min(wer_improvements), - "max_improvement": np.max(wer_improvements), - "median_improvement": np.median(wer_improvements) - }, - "cer_metrics": { - "mean_improvement": np.mean(cer_improvements), - "std_improvement": np.std(cer_improvements), - "min_improvement": np.min(cer_improvements), - "max_improvement": np.max(cer_improvements), - "median_improvement": np.median(cer_improvements) - }, - "consistency": { - "wer_improvement_std": np.std(wer_improvements), - "cer_improvement_std": np.std(cer_improvements) - }, - "raw_results": [r.to_dict() for r in results] - } - - def get_correction_accuracy(self) -> Dict: - """Get overall correction accuracy statistics""" - - if not self.results: - return {"error": "No evaluation results"} - - improved = sum(1 for r in self.results if r.was_improved) - - return { - "total_corrections": len(self.results), - "successful_corrections": improved, - "accuracy_rate": improved / len(self.results), - "mean_wer_improvement": np.mean([r.wer_improvement for r in self.results]), - "mean_cer_improvement": np.mean([r.cer_improvement for r in self.results]), - "consistency_wer_std": np.std([r.wer_improvement for r in self.results]), - "consistency_cer_std": np.std([r.cer_improvement for r in self.results]) - } - - def save_results(self, filename: str = "agent_evaluation_results.json"): - """Save evaluation results to JSON""" - - results_data = { - "accuracy": self.get_correction_accuracy(), - "detailed_results": [r.to_dict() for r in self.results], - "timestamp": datetime.now().isoformat() - } - - output_path = self.output_dir / filename - with open(output_path, 'w') as f: - json.dump(results_data, f, indent=2) - - logger.info(f"โœ… Results saved to {output_path}") - return output_path - - def print_summary(self): - """Print evaluation summary""" - - accuracy = self.get_correction_accuracy() - - print("\n" + "="*70) - print("AGENT EVALUATOR - CORRECTION ACCURACY REPORT") - print("="*70) - print(f"\nTotal corrections evaluated: {accuracy['total_corrections']}") - print(f"Successful corrections: {accuracy['successful_corrections']}") - print(f"Accuracy rate: {accuracy['accuracy_rate']:.2%}") - print(f"\nMean WER improvement: {accuracy['mean_wer_improvement']:.4f}") - print(f"Mean CER improvement: {accuracy['mean_cer_improvement']:.4f}") - print(f"\nWER improvement consistency (std): {accuracy['consistency_wer_std']:.4f}") - print(f"CER improvement consistency (std): {accuracy['consistency_cer_std']:.4f}") - print("="*70 + "\n") diff --git a/src/agent_evaluation/false_positive_detector.py b/src/agent_evaluation/false_positive_detector.py deleted file mode 100644 index b5db2c2..0000000 --- a/src/agent_evaluation/false_positive_detector.py +++ /dev/null @@ -1,212 +0,0 @@ -""" -False Positive Detector - Identify Harmful Corrections - -Detects when agent corrections make transcriptions worse. -""" - -import logging -from typing import Dict, List, Optional, Tuple -from dataclasses import dataclass -import json -from pathlib import Path - -from jiwer import wer, cer -import numpy as np - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -@dataclass -class FalsePositive: - """Represents a false positive correction""" - audio_path: str - original_transcript: str - corrected_transcript: str - reference_transcript: Optional[str] - - original_wer: float - corrected_wer: float - wer_degradation: float # positive = worse - - original_cer: float - corrected_cer: float - cer_degradation: float - - error_type: str # What kind of error was detected - error_confidence: float - - def to_dict(self) -> Dict: - return { - 'audio_path': self.audio_path, - 'original_transcript': self.original_transcript, - 'corrected_transcript': self.corrected_transcript, - 'reference_transcript': self.reference_transcript, - 'original_wer': self.original_wer, - 'corrected_wer': self.corrected_wer, - 'wer_degradation': self.wer_degradation, - 'original_cer': self.original_cer, - 'corrected_cer': self.corrected_cer, - 'cer_degradation': self.cer_degradation, - 'error_type': self.error_type, - 'error_confidence': self.error_confidence - } - - -class FalsePositiveDetector: - """ - Detects false positive corrections (corrections that hurt accuracy). - - A false positive occurs when: - - Agent detects an "error" and "corrects" it - - But the correction actually makes WER/CER worse - - Key metrics: - - False positive rate: % of corrections that hurt - - Average degradation: How much worse corrections made things - - Error type breakdown: Which error types cause false positives most - """ - - def __init__(self, output_dir: str = "experiments/evaluation_outputs"): - """Initialize detector""" - self.output_dir = Path(output_dir) - self.output_dir.mkdir(parents=True, exist_ok=True) - - self.false_positives: List[FalsePositive] = [] - - logger.info("False Positive Detector initialized") - - def detect_false_positive(self, - original_transcript: str, - corrected_transcript: str, - reference_transcript: str, - error_type: str, - error_confidence: float, - audio_path: str = "") -> Optional[FalsePositive]: - """ - Detect if a correction is a false positive. - - Args: - original_transcript: Baseline output - corrected_transcript: Agent-corrected output - reference_transcript: Ground truth - error_type: Type of error detected - error_confidence: Confidence of error detection - audio_path: Path to audio file - - Returns: - FalsePositive object if correction hurt, None otherwise - """ - - original_wer = wer(reference_transcript, original_transcript) - original_cer = cer(reference_transcript, original_transcript) - - corrected_wer = wer(reference_transcript, corrected_transcript) - corrected_cer = cer(reference_transcript, corrected_transcript) - - wer_degradation = corrected_wer - original_wer - cer_degradation = corrected_cer - original_cer - - # False positive if correction made things worse (threshold: 0.01) - if wer_degradation > 0.01 or cer_degradation > 0.01: - fp = FalsePositive( - audio_path=audio_path, - original_transcript=original_transcript, - corrected_transcript=corrected_transcript, - reference_transcript=reference_transcript, - original_wer=original_wer, - corrected_wer=corrected_wer, - wer_degradation=wer_degradation, - original_cer=original_cer, - corrected_cer=corrected_cer, - cer_degradation=cer_degradation, - error_type=error_type, - error_confidence=error_confidence - ) - - self.false_positives.append(fp) - return fp - - return None - - def analyze_false_positives(self) -> Dict: - """Analyze all detected false positives""" - - if not self.false_positives: - return { - "total_false_positives": 0, - "false_positive_rate": 0.0 - } - - # Error type breakdown - error_type_counts = {} - for fp in self.false_positives: - error_type_counts[fp.error_type] = \ - error_type_counts.get(fp.error_type, 0) + 1 - - # WER degradation stats - wer_degradations = [fp.wer_degradation for fp in self.false_positives] - cer_degradations = [fp.cer_degradation for fp in self.false_positives] - - return { - "total_false_positives": len(self.false_positives), - "error_type_breakdown": error_type_counts, - "wer_degradation": { - "mean": np.mean(wer_degradations), - "std": np.std(wer_degradations), - "max": np.max(wer_degradations), - "min": np.min(wer_degradations) - }, - "cer_degradation": { - "mean": np.mean(cer_degradations), - "std": np.std(cer_degradations), - "max": np.max(cer_degradations), - "min": np.min(cer_degradations) - }, - "error_type_analysis": { - error_type: { - "count": count, - "average_degradation": np.mean([ - fp.wer_degradation for fp in self.false_positives - if fp.error_type == error_type - ]) - } - for error_type, count in error_type_counts.items() - } - } - - def save_analysis(self, filename: str = "false_positives_analysis.json"): - """Save false positive analysis""" - - analysis = self.analyze_false_positives() - analysis['detailed_false_positives'] = [ - fp.to_dict() for fp in self.false_positives - ] - - output_path = self.output_dir / filename - with open(output_path, 'w') as f: - json.dump(analysis, f, indent=2) - - logger.info(f"โœ… Analysis saved to {output_path}") - return output_path - - def print_summary(self): - """Print false positive summary""" - - analysis = self.analyze_false_positives() - - print("\n" + "="*70) - print("FALSE POSITIVE DETECTION REPORT") - print("="*70) - print(f"\nTotal false positives: {analysis['total_false_positives']}") - - if analysis['total_false_positives'] > 0: - print(f"\nWER degradation (mean): {analysis['wer_degradation']['mean']:.4f}") - print(f"CER degradation (mean): {analysis['cer_degradation']['mean']:.4f}") - - print("\nError types causing false positives:") - for error_type, stats in analysis['error_type_analysis'].items(): - print(f" {error_type}: {stats['count']} cases " + - f"(avg degradation: {stats['average_degradation']:.4f})") - - print("="*70 + "\n") diff --git a/src/data/augmentation.py b/src/data/augmentation.py deleted file mode 100644 index a0e2265..0000000 --- a/src/data/augmentation.py +++ /dev/null @@ -1,117 +0,0 @@ -""" -Noise augmentation for creating robust training data. -""" - -import librosa -import soundfile as sf -import numpy as np -from pathlib import Path -import random -from typing import List, Optional -import logging - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -class NoiseAugmentor: - """Add background noise to audio samples""" - - def __init__(self, noise_samples: List[str], sr: int = 16000): - """ - Initialize augmentor. - - Args: - noise_samples: List of paths to noise audio files - sr: Sample rate - """ - self.noise_samples = noise_samples - self.sr = sr - self.noise_cache = {} - - logger.info(f"Initialized with {len(noise_samples)} noise samples") - - def _load_noise(self, noise_path: str) -> np.ndarray: - """Load and cache noise sample""" - if noise_path not in self.noise_cache: - noise, _ = librosa.load(noise_path, sr=self.sr) - self.noise_cache[noise_path] = noise - return self.noise_cache[noise_path] - - def add_noise( - self, - audio: np.ndarray, - snr_db: float = 10.0, - noise_path: Optional[str] = None - ) -> np.ndarray: - """ - Add noise to audio at specified SNR. - - Args: - audio: Clean audio array - snr_db: Signal-to-noise ratio in decibels - noise_path: Specific noise file to use (random if None) - - Returns: - Noisy audio array - """ - # Select noise - if noise_path is None: - noise_path = random.choice(self.noise_samples) - - noise = self._load_noise(noise_path) - - # Match noise length to audio - if len(noise) < len(audio): - # Repeat noise if too short - repetitions = int(np.ceil(len(audio) / len(noise))) - noise = np.tile(noise, repetitions)[:len(audio)] - else: - # Random crop if too long - start = random.randint(0, len(noise) - len(audio)) - noise = noise[start:start + len(audio)] - - # Calculate scaling factor for target SNR - signal_power = np.mean(audio ** 2) - noise_power = np.mean(noise ** 2) - - snr_linear = 10 ** (snr_db / 10) - scale = np.sqrt(signal_power / (snr_linear * noise_power)) - - # Add scaled noise - noisy_audio = audio + scale * noise - - return noisy_audio - - def augment_file( - self, - input_path: str, - output_path: str, - snr_db: float = 10.0 - ) -> dict: - """ - Augment single audio file. - - Args: - input_path: Input audio file - output_path: Output audio file - snr_db: Target SNR in dB - - Returns: - Metadata dictionary - """ - # Load audio - audio, sr = librosa.load(input_path, sr=self.sr) - - # Add noise - noisy_audio = self.add_noise(audio, snr_db) - - # Save - Path(output_path).parent.mkdir(parents=True, exist_ok=True) - sf.write(output_path, noisy_audio, sr) - - return { - "input_path": input_path, - "output_path": output_path, - "snr_db": snr_db, - "duration": len(audio) / sr - } diff --git a/src/data/wandb_sweeps.py b/src/data/wandb_sweeps.py deleted file mode 100644 index ef02e31..0000000 --- a/src/data/wandb_sweeps.py +++ /dev/null @@ -1,540 +0,0 @@ -""" -W&B Sweeps Integration for Hyperparameter Optimization - -Automatically finds the best hyperparameters for fine-tuning using various search strategies. -""" - -import logging -from typing import Dict, List, Optional, Callable, Any -from pathlib import Path -import json - -try: - import wandb - WANDB_AVAILABLE = True -except ImportError: - WANDB_AVAILABLE = False - logging.warning("wandb not available. Install with: pip install wandb") - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -class SweepConfig: - """Configuration for W&B hyperparameter sweeps.""" - - # Search strategies - RANDOM = "random" - BAYES = "bayes" - GRID = "grid" - - # Optimization goals - MINIMIZE = "minimize" - MAXIMIZE = "maximize" - - @staticmethod - def create_finetuning_sweep( - metric_name: str = "validation/model_wer", - goal: str = "minimize", - method: str = "random", - num_trials: int = 20 - ) -> Dict: - """ - Create sweep configuration for STT fine-tuning. - - Args: - metric_name: Metric to optimize (e.g., 'validation/model_wer') - goal: 'minimize' or 'maximize' - method: 'random', 'bayes', or 'grid' - num_trials: Number of trials to run - - Returns: - Sweep configuration dictionary - """ - config = { - 'method': method, - 'metric': { - 'name': metric_name, - 'goal': goal - }, - 'parameters': { - # Learning rate - most critical parameter - 'learning_rate': { - 'distribution': 'log_uniform_values', - 'min': 1e-6, - 'max': 1e-4 - }, - - # Batch size - 'batch_size': { - 'values': [4, 8, 16, 32] - }, - - # Training epochs - 'epochs': { - 'values': [3, 5, 10, 15] - }, - - # Warmup steps - 'warmup_steps': { - 'values': [100, 500, 1000] - }, - - # Weight decay - 'weight_decay': { - 'distribution': 'uniform', - 'min': 0.0, - 'max': 0.1 - }, - - # Gradient accumulation - 'gradient_accumulation_steps': { - 'values': [1, 2, 4] - }, - - # Dropout - 'dropout': { - 'distribution': 'uniform', - 'min': 0.0, - 'max': 0.3 - } - } - } - - # Add early termination for Bayesian optimization - if method == 'bayes': - config['early_terminate'] = { - 'type': 'hyperband', - 'min_iter': 3 - } - - return config - - @staticmethod - def create_minimal_sweep( - metric_name: str = "validation/model_wer", - num_trials: int = 10 - ) -> Dict: - """ - Create minimal sweep config for quick testing. - - Args: - metric_name: Metric to optimize - num_trials: Number of trials - - Returns: - Minimal sweep configuration - """ - return { - 'method': 'random', - 'metric': { - 'name': metric_name, - 'goal': 'minimize' - }, - 'parameters': { - 'learning_rate': { - 'values': [1e-5, 5e-5, 1e-4] - }, - 'batch_size': { - 'values': [8, 16] - }, - 'epochs': { - 'values': [5, 10] - } - } - } - - @staticmethod - def create_custom_sweep( - parameters: Dict[str, Any], - metric_name: str = "validation/model_wer", - goal: str = "minimize", - method: str = "random" - ) -> Dict: - """ - Create custom sweep configuration. - - Args: - parameters: Custom parameter definitions - metric_name: Metric to optimize - goal: 'minimize' or 'maximize' - method: Search method - - Returns: - Custom sweep configuration - """ - return { - 'method': method, - 'metric': { - 'name': metric_name, - 'goal': goal - }, - 'parameters': parameters - } - - -class WandbSweepOrchestrator: - """ - Orchestrates hyperparameter optimization sweeps for fine-tuning. - - Features: - - Multiple search strategies (random, Bayesian, grid) - - Automatic optimization of WER/CER - - Integration with fine-tuning pipeline - - Best hyperparameters selection - - Parallel sweep execution support - """ - - def __init__( - self, - project_name: str = "stt-finetuning-sweeps", - entity: Optional[str] = None, - enabled: bool = True - ): - """ - Initialize sweep orchestrator. - - Args: - project_name: W&B project name for sweeps - entity: W&B entity (username/team) - enabled: Whether sweeps are enabled - """ - self.project_name = project_name - self.entity = entity - self.enabled = enabled and WANDB_AVAILABLE - self.sweep_id = None - - if not WANDB_AVAILABLE and enabled: - logger.warning("W&B not available. Sweeps disabled.") - self.enabled = False - - if self.enabled: - logger.info(f"W&B Sweep Orchestrator initialized for project: {project_name}") - - def create_sweep( - self, - sweep_config: Dict, - sweep_name: Optional[str] = None - ) -> Optional[str]: - """ - Create a new hyperparameter sweep. - - Args: - sweep_config: Sweep configuration dictionary - sweep_name: Optional name for the sweep - - Returns: - Sweep ID if successful, None otherwise - """ - if not self.enabled: - logger.warning("Sweeps not enabled") - return None - - try: - # Add sweep name if provided - if sweep_name: - sweep_config['name'] = sweep_name - - # Create sweep - self.sweep_id = wandb.sweep( - sweep_config, - project=self.project_name, - entity=self.entity - ) - - logger.info(f"Created sweep: {self.sweep_id}") - logger.info(f"View at: https://wandb.ai/{self.entity or 'your-username'}/{self.project_name}/sweeps/{self.sweep_id}") - - return self.sweep_id - - except Exception as e: - logger.error(f"Failed to create sweep: {e}") - return None - - def run_sweep_agent( - self, - train_function: Callable, - sweep_id: Optional[str] = None, - count: Optional[int] = None - ): - """ - Run sweep agent to execute hyperparameter trials. - - Args: - train_function: Training function that uses wandb.config for hyperparameters - sweep_id: Sweep ID to run (uses self.sweep_id if not provided) - count: Number of trials to run (None for unlimited) - """ - if not self.enabled: - logger.warning("Sweeps not enabled") - return - - sweep_id = sweep_id or self.sweep_id - if not sweep_id: - logger.error("No sweep ID provided") - return - - try: - logger.info(f"Starting sweep agent for sweep: {sweep_id}") - - wandb.agent( - sweep_id, - function=train_function, - count=count, - project=self.project_name, - entity=self.entity - ) - - logger.info("Sweep agent completed") - - except Exception as e: - logger.error(f"Sweep agent failed: {e}") - - def get_best_run( - self, - sweep_id: Optional[str] = None, - metric_name: str = "validation/model_wer", - minimize: bool = True - ) -> Optional[Dict]: - """ - Get best run from a completed sweep. - - Args: - sweep_id: Sweep ID to analyze - metric_name: Metric to use for comparison - minimize: Whether to minimize (True) or maximize (False) the metric - - Returns: - Dictionary with best run information - """ - if not self.enabled: - return None - - sweep_id = sweep_id or self.sweep_id - if not sweep_id: - logger.error("No sweep ID provided") - return None - - try: - api = wandb.Api() - sweep = api.sweep(f"{self.entity or api.viewer()['entity']}/{self.project_name}/{sweep_id}") - - # Get all runs from sweep - runs = sweep.runs - - if not runs: - logger.warning("No runs found in sweep") - return None - - # Find best run - best_run = None - best_metric = float('inf') if minimize else float('-inf') - - for run in runs: - if metric_name in run.summary: - metric_value = run.summary[metric_name] - - if minimize and metric_value < best_metric: - best_metric = metric_value - best_run = run - elif not minimize and metric_value > best_metric: - best_metric = metric_value - best_run = run - - if not best_run: - logger.warning(f"No runs with metric {metric_name} found") - return None - - # Extract best hyperparameters - best_config = { - 'run_id': best_run.id, - 'run_name': best_run.name, - 'metric_value': best_metric, - 'hyperparameters': dict(best_run.config), - 'summary': dict(best_run.summary) - } - - logger.info(f"Best run: {best_run.name}") - logger.info(f"Best {metric_name}: {best_metric:.4f}") - logger.info(f"Best hyperparameters: {best_config['hyperparameters']}") - - return best_config - - except Exception as e: - logger.error(f"Failed to get best run: {e}") - return None - - def save_best_config( - self, - output_path: str, - sweep_id: Optional[str] = None, - metric_name: str = "validation/model_wer" - ) -> bool: - """ - Save best hyperparameters to file. - - Args: - output_path: Path to save configuration - sweep_id: Sweep ID to analyze - metric_name: Metric to optimize - - Returns: - True if successful - """ - best_config = self.get_best_run(sweep_id, metric_name) - - if not best_config: - return False - - try: - output_path = Path(output_path) - output_path.parent.mkdir(parents=True, exist_ok=True) - - with open(output_path, 'w') as f: - json.dump(best_config, f, indent=2) - - logger.info(f"Saved best configuration to: {output_path}") - return True - - except Exception as e: - logger.error(f"Failed to save configuration: {e}") - return False - - def create_and_run_sweep( - self, - train_function: Callable, - sweep_config: Optional[Dict] = None, - sweep_name: Optional[str] = None, - num_trials: int = 20 - ) -> Optional[Dict]: - """ - Create and run a complete sweep, returning best hyperparameters. - - Args: - train_function: Training function - sweep_config: Sweep configuration (uses default if not provided) - sweep_name: Sweep name - num_trials: Number of trials to run - - Returns: - Best hyperparameters dictionary - """ - # Use default config if not provided - if sweep_config is None: - sweep_config = SweepConfig.create_finetuning_sweep( - method='random', - num_trials=num_trials - ) - - # Create sweep - sweep_id = self.create_sweep(sweep_config, sweep_name) - - if not sweep_id: - return None - - # Run sweep - self.run_sweep_agent(train_function, sweep_id, count=num_trials) - - # Get best configuration - best_config = self.get_best_run(sweep_id) - - return best_config - - -def create_sweep_training_wrapper( - base_train_function: Callable, - data_manager, - orchestrator -) -> Callable: - """ - Create a training wrapper function compatible with W&B sweeps. - - Args: - base_train_function: Your actual training function - data_manager: DataManager instance - orchestrator: FinetuningOrchestrator instance - - Returns: - Wrapped function that uses wandb.config for hyperparameters - """ - def train(): - """Training function for sweep.""" - # Initialize W&B run (sweep agent does this automatically) - config = wandb.config - - logger.info(f"Running trial with config: {dict(config)}") - - # Trigger fine-tuning with sweep hyperparameters - job = orchestrator.trigger_finetuning(force=True) - - if not job: - logger.error("Failed to trigger fine-tuning") - return - - # Train with hyperparameters from sweep - training_params = { - 'learning_rate': config.learning_rate, - 'batch_size': config.batch_size, - 'epochs': config.epochs, - 'warmup_steps': config.get('warmup_steps', 500), - 'weight_decay': config.get('weight_decay', 0.01), - 'gradient_accumulation_steps': config.get('gradient_accumulation_steps', 1) - } - - # Call your actual training function - result = base_train_function(job, training_params) - - # Log final metrics - if result and 'validation' in result: - wandb.log({ - 'validation/model_wer': result['validation']['wer'], - 'validation/model_cer': result['validation']['cer'], - 'validation/wer_improvement': result['validation']['wer_improvement'] - }) - - return train - - -# Example usage -def example_sweep(): - """Example of running a hyperparameter sweep.""" - - # 1. Create sweep orchestrator - sweep_orch = WandbSweepOrchestrator( - project_name="stt-finetuning-sweeps" - ) - - # 2. Create sweep configuration - sweep_config = SweepConfig.create_finetuning_sweep( - metric_name="validation/model_wer", - goal="minimize", - method="random", - num_trials=20 - ) - - # 3. Define your training function - def train(): - # Your training code here - # Use wandb.config for hyperparameters - config = wandb.config - - # ... training logic ... - - # Log metrics - wandb.log({ - 'validation/model_wer': 0.15, # Your actual WER - 'validation/model_cer': 0.08 # Your actual CER - }) - - # 4. Create and run sweep - sweep_id = sweep_orch.create_sweep(sweep_config, "my_sweep") - sweep_orch.run_sweep_agent(train, count=20) - - # 5. Get best hyperparameters - best_config = sweep_orch.get_best_run(sweep_id) - - # 6. Save best config - sweep_orch.save_best_config("best_hyperparameters.json", sweep_id) - - return best_config - diff --git a/src/integration/__init__.py b/src/integration/__init__.py deleted file mode 100644 index eb58647..0000000 --- a/src/integration/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -Integration Module - Week 4 -Unified system integration and testing framework -""" - -from .unified_system import UnifiedSTTSystem -from .end_to_end_testing import EndToEndTester -from .statistical_analysis import StatisticalAnalyzer -from .ablation_studies import AblationStudy - -__all__ = [ - 'UnifiedSTTSystem', - 'EndToEndTester', - 'StatisticalAnalyzer', - 'AblationStudy' -] diff --git a/src/integration/ablation_studies.py b/src/integration/ablation_studies.py deleted file mode 100644 index fbb0ee5..0000000 --- a/src/integration/ablation_studies.py +++ /dev/null @@ -1,396 +0,0 @@ -""" -Ablation Studies Framework - Week 4 -Evaluate individual component contributions through systematic ablation. -""" - -import logging -from typing import Dict, List, Optional, Tuple -from pathlib import Path -import time -import json -from datetime import datetime -import numpy as np - -from .unified_system import UnifiedSTTSystem -from .statistical_analysis import StatisticalAnalyzer -from ..evaluation.metrics import STTEvaluator - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -class AblationStudy: - """ - Ablation study framework for evaluating component contributions. - Systematically removes components to measure their individual impact. - """ - - def __init__(self, base_config: Optional[Dict] = None): - """ - Initialize ablation study framework. - - Args: - base_config: Base configuration for system initialization - """ - self.base_config = base_config or {} - self.study_results = [] - self.analyzer = StatisticalAnalyzer() - self.evaluator = STTEvaluator() - - def run_ablation_study( - self, - audio_files: List[str], - reference_transcripts: List[str], - model_name: str = "whisper" - ) -> Dict: - """ - Run complete ablation study. - - Args: - audio_files: List of audio file paths - reference_transcripts: List of reference transcripts - model_name: Model name to use - - Returns: - Dictionary with ablation study results - """ - logger.info("="*60) - logger.info("Running Ablation Study") - logger.info("="*60) - logger.info(f"Testing {len(audio_files)} files") - logger.info("") - - # Define all system configurations to test - configurations = self._define_configurations() - - all_results = {} - - for config_name, config in configurations.items(): - logger.info(f"\n{'='*60}") - logger.info(f"Configuration: {config_name}") - logger.info(f"{'='*60}") - logger.info(f"Components enabled: {config}") - - # Initialize system with this configuration - system = UnifiedSTTSystem( - model_name=model_name, - enable_error_detection=config.get('error_detection', False), - enable_llm_correction=config.get('llm_correction', False), - enable_adaptive_fine_tuning=config.get('adaptive_fine_tuning', False), - **self.base_config - ) - - # Evaluate this configuration - results = self._evaluate_configuration( - system=system, - audio_files=audio_files, - reference_transcripts=reference_transcripts, - config_name=config_name - ) - - all_results[config_name] = results - - logger.info(f" Average WER: {results['average_wer']:.4f}") - logger.info(f" Average CER: {results['average_cer']:.4f}") - logger.info(f" Processing time: {results['total_time']:.2f}s") - - # Analyze component contributions - contribution_analysis = self._analyze_contributions(all_results) - - return { - 'study_type': 'ablation', - 'timestamp': datetime.now().isoformat(), - 'num_files': len(audio_files), - 'configurations_tested': list(all_results.keys()), - 'results': all_results, - 'contribution_analysis': contribution_analysis, - 'summary': self._generate_ablation_summary(all_results, contribution_analysis) - } - - def _define_configurations(self) -> Dict[str, Dict]: - """ - Define all system configurations for ablation study. - Each configuration represents a different combination of components. - """ - return { - # Baseline: Only baseline model - 'baseline_only': { - 'error_detection': False, - 'llm_correction': False, - 'adaptive_fine_tuning': False, - 'description': 'Baseline STT model only' - }, - - # Baseline + Error Detection - 'baseline_error_detection': { - 'error_detection': True, - 'llm_correction': False, - 'adaptive_fine_tuning': False, - 'description': 'Baseline + Error Detection' - }, - - # Baseline + Error Detection + Self-Learning - 'baseline_error_self_learning': { - 'error_detection': True, - 'llm_correction': False, - 'adaptive_fine_tuning': False, - 'description': 'Baseline + Error Detection + Self-Learning' - }, - - # Baseline + Error Detection + LLM Correction - 'baseline_error_llm': { - 'error_detection': True, - 'llm_correction': True, - 'adaptive_fine_tuning': False, - 'description': 'Baseline + Error Detection + LLM Correction' - }, - - # Full system without fine-tuning - 'full_no_finetuning': { - 'error_detection': True, - 'llm_correction': True, - 'adaptive_fine_tuning': False, - 'description': 'Full system without adaptive fine-tuning' - }, - - # Full system with all components - 'full_system': { - 'error_detection': True, - 'llm_correction': True, - 'adaptive_fine_tuning': True, - 'description': 'Full system with all components' - } - } - - def _evaluate_configuration( - self, - system: UnifiedSTTSystem, - audio_files: List[str], - reference_transcripts: List[str], - config_name: str - ) -> Dict: - """ - Evaluate a specific system configuration. - - Args: - system: UnifiedSTTSystem instance - audio_files: List of audio file paths - reference_transcripts: List of reference transcripts - config_name: Name of configuration - - Returns: - Dictionary with evaluation results - """ - start_time = time.time() - - wers = [] - cers = [] - error_counts = [] - correction_counts = [] - - for audio_path, reference in zip(audio_files, reference_transcripts): - result = system.transcribe( - audio_path=audio_path, - reference_transcript=reference, - enable_auto_correction=True - ) - - if 'evaluation' in result: - wers.append(result['evaluation']['wer']) - cers.append(result['evaluation']['cer']) - - error_counts.append(result.get('error_detection', {}).get('error_count', 0)) - correction_counts.append(result.get('corrections', {}).get('count', 0)) - - total_time = time.time() - start_time - - return { - 'config_name': config_name, - 'average_wer': np.mean(wers) if wers else None, - 'average_cer': np.mean(cers) if cers else None, - 'std_wer': np.std(wers) if wers else None, - 'std_cer': np.std(cers) if cers else None, - 'total_errors_detected': sum(error_counts), - 'total_corrections_applied': sum(correction_counts), - 'total_time': total_time, - 'average_time_per_file': total_time / len(audio_files) if audio_files else 0, - 'wer_scores': wers, - 'cer_scores': cers - } - - def _analyze_contributions(self, all_results: Dict[str, Dict]) -> Dict: - """ - Analyze contribution of each component. - - Args: - all_results: Dictionary mapping config names to results - - Returns: - Dictionary with contribution analysis - """ - baseline_results = all_results.get('baseline_only', {}) - baseline_wers = baseline_results.get('wer_scores', []) - - if not baseline_wers: - return {'status': 'insufficient_baseline_data'} - - contributions = {} - - # Error Detection contribution - error_detection_results = all_results.get('baseline_error_detection', {}) - if error_detection_results.get('wer_scores'): - error_detection_contribution = self.analyzer.paired_t_test( - baseline_scores=baseline_wers, - treatment_scores=error_detection_results['wer_scores'] - ) - contributions['error_detection'] = { - 'improvement': error_detection_contribution['mean_difference'], - 'p_value': error_detection_contribution['p_value'], - 'is_significant': error_detection_contribution['is_significant'], - 'effect_size': error_detection_contribution['cohens_d'] - } - - # LLM Correction contribution - llm_results = all_results.get('baseline_error_llm', {}) - error_only_results = all_results.get('baseline_error_detection', {}) - if llm_results.get('wer_scores') and error_only_results.get('wer_scores'): - llm_contribution = self.analyzer.paired_t_test( - baseline_scores=error_only_results['wer_scores'], - treatment_scores=llm_results['wer_scores'] - ) - contributions['llm_correction'] = { - 'improvement': llm_contribution['mean_difference'], - 'p_value': llm_contribution['p_value'], - 'is_significant': llm_contribution['is_significant'], - 'effect_size': llm_contribution['cohens_d'] - } - - # Adaptive Fine-Tuning contribution - full_results = all_results.get('full_system', {}) - no_finetuning_results = all_results.get('full_no_finetuning', {}) - if full_results.get('wer_scores') and no_finetuning_results.get('wer_scores'): - finetuning_contribution = self.analyzer.paired_t_test( - baseline_scores=no_finetuning_results['wer_scores'], - treatment_scores=full_results['wer_scores'] - ) - contributions['adaptive_fine_tuning'] = { - 'improvement': finetuning_contribution['mean_difference'], - 'p_value': finetuning_contribution['p_value'], - 'is_significant': finetuning_contribution['is_significant'], - 'effect_size': finetuning_contribution['cohens_d'] - } - - # Overall system improvement - if full_results.get('wer_scores'): - overall_improvement = self.analyzer.paired_t_test( - baseline_scores=baseline_wers, - treatment_scores=full_results['wer_scores'] - ) - contributions['overall_system'] = { - 'improvement': overall_improvement['mean_difference'], - 'p_value': overall_improvement['p_value'], - 'is_significant': overall_improvement['is_significant'], - 'effect_size': overall_improvement['cohens_d'], - 'relative_improvement': ( - overall_improvement['mean_difference'] / np.mean(baseline_wers) * 100 - if np.mean(baseline_wers) > 0 else 0 - ) - } - - return { - 'baseline_wer': np.mean(baseline_wers), - 'component_contributions': contributions, - 'significant_components': [ - name for name, contrib in contributions.items() - if contrib.get('is_significant', False) - ] - } - - def _generate_ablation_summary( - self, - all_results: Dict[str, Dict], - contribution_analysis: Dict - ) -> Dict: - """Generate summary of ablation study.""" - baseline_wer = all_results.get('baseline_only', {}).get('average_wer') - full_system_wer = all_results.get('full_system', {}).get('average_wer') - - summary = { - 'baseline_performance': baseline_wer, - 'full_system_performance': full_system_wer, - 'overall_improvement': baseline_wer - full_system_wer if (baseline_wer and full_system_wer) else None, - 'configurations_tested': len(all_results), - 'component_contributions': {} - } - - if 'component_contributions' in contribution_analysis: - for component, contrib in contribution_analysis['component_contributions'].items(): - summary['component_contributions'][component] = { - 'improvement': contrib.get('improvement', 0), - 'is_significant': contrib.get('is_significant', False), - 'effect_size': contrib.get('effect_size', 0) - } - - return summary - - def generate_ablation_report( - self, - study_results: Dict, - output_path: Optional[str] = None - ) -> str: - """ - Generate detailed ablation study report. - - Args: - study_results: Results from ablation study - output_path: Optional path to save report - - Returns: - Report string - """ - report_lines = [] - report_lines.append("="*60) - report_lines.append("Ablation Study Report") - report_lines.append("="*60) - report_lines.append("") - - # Summary - summary = study_results.get('summary', {}) - report_lines.append("SUMMARY") - report_lines.append("-"*60) - report_lines.append(f"Baseline WER: {summary.get('baseline_performance', 'N/A'):.4f}") - report_lines.append(f"Full System WER: {summary.get('full_system_performance', 'N/A'):.4f}") - report_lines.append(f"Overall Improvement: {summary.get('overall_improvement', 'N/A'):.4f}") - report_lines.append("") - - # Component Contributions - report_lines.append("COMPONENT CONTRIBUTIONS") - report_lines.append("-"*60) - contributions = summary.get('component_contributions', {}) - for component, contrib in contributions.items(): - report_lines.append(f"{component}:") - report_lines.append(f" Improvement: {contrib.get('improvement', 0):.4f}") - report_lines.append(f" Significant: {contrib.get('is_significant', False)}") - report_lines.append(f" Effect Size: {contrib.get('effect_size', 0):.4f}") - report_lines.append("") - - # Configuration Results - report_lines.append("CONFIGURATION RESULTS") - report_lines.append("-"*60) - results = study_results.get('results', {}) - for config_name, config_results in results.items(): - report_lines.append(f"{config_name}:") - report_lines.append(f" WER: {config_results.get('average_wer', 'N/A'):.4f}") - report_lines.append(f" CER: {config_results.get('average_cer', 'N/A'):.4f}") - report_lines.append(f" Time: {config_results.get('total_time', 0):.2f}s") - report_lines.append("") - - report = "\n".join(report_lines) - - if output_path: - Path(output_path).parent.mkdir(parents=True, exist_ok=True) - with open(output_path, 'w') as f: - f.write(report) - logger.info(f"Ablation report saved to {output_path}") - - return report diff --git a/src/integration/end_to_end_testing.py b/src/integration/end_to_end_testing.py deleted file mode 100644 index cdd86f1..0000000 --- a/src/integration/end_to_end_testing.py +++ /dev/null @@ -1,427 +0,0 @@ -""" -End-to-End Testing Framework - Week 4 -Tests the complete feedback loop from transcription to fine-tuning. -""" - -import logging -from typing import Dict, List, Optional, Tuple -from pathlib import Path -import time -import json -from datetime import datetime -import numpy as np - -from .unified_system import UnifiedSTTSystem -from ..evaluation.metrics import STTEvaluator - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -class EndToEndTester: - """ - End-to-end testing framework for the complete feedback loop. - Tests the full pipeline: transcription โ†’ error detection โ†’ correction โ†’ learning โ†’ fine-tuning - """ - - def __init__(self, system: UnifiedSTTSystem): - """ - Initialize end-to-end tester. - - Args: - system: UnifiedSTTSystem instance to test - """ - self.system = system - self.evaluator = STTEvaluator() - self.test_results = [] - - def test_feedback_loop( - self, - audio_files: List[str], - reference_transcripts: List[str], - num_iterations: int = 3, - enable_corrections: bool = True - ) -> Dict: - """ - Test the complete feedback loop over multiple iterations. - - Args: - audio_files: List of audio file paths - reference_transcripts: List of reference transcripts - num_iterations: Number of feedback loop iterations - enable_corrections: Whether to enable corrections - - Returns: - Dictionary with feedback loop test results - """ - logger.info(f"Starting feedback loop test: {len(audio_files)} files, {num_iterations} iterations") - - iteration_results = [] - - for iteration in range(num_iterations): - logger.info(f"\n{'='*60}") - logger.info(f"Iteration {iteration + 1}/{num_iterations}") - logger.info(f"{'='*60}") - - iteration_start = time.time() - - # Transcribe all files - batch_results = [] - for i, (audio_path, reference) in enumerate(zip(audio_files, reference_transcripts)): - logger.info(f" Processing {i+1}/{len(audio_files)}: {Path(audio_path).name}") - - result = self.system.transcribe( - audio_path=audio_path, - reference_transcript=reference, - enable_auto_correction=enable_corrections - ) - - batch_results.append(result) - - # Calculate metrics for this iteration - wers = [r['evaluation']['wer'] for r in batch_results if 'evaluation' in r] - cers = [r['evaluation']['cer'] for r in batch_results if 'evaluation' in r] - - iteration_metrics = { - 'iteration': iteration + 1, - 'average_wer': np.mean(wers) if wers else None, - 'average_cer': np.mean(cers) if cers else None, - 'total_errors_detected': sum(r.get('error_detection', {}).get('error_count', 0) for r in batch_results), - 'total_corrections_applied': sum(r.get('corrections', {}).get('count', 0) for r in batch_results), - 'fine_tuning_triggered': any(r.get('agent_metadata', {}).get('fine_tuning_triggered', False) for r in batch_results), - 'processing_time': time.time() - iteration_start, - 'detailed_results': batch_results - } - - iteration_results.append(iteration_metrics) - - # Get system status after iteration - system_status = self.system.get_system_status() - iteration_metrics['system_status'] = system_status - - logger.info(f" Average WER: {iteration_metrics['average_wer']:.4f}" if iteration_metrics['average_wer'] else " Average WER: N/A") - logger.info(f" Errors detected: {iteration_metrics['total_errors_detected']}") - logger.info(f" Corrections applied: {iteration_metrics['total_corrections_applied']}") - logger.info(f" Fine-tuning triggered: {iteration_metrics['fine_tuning_triggered']}") - - # Analyze feedback loop effectiveness - feedback_analysis = self._analyze_feedback_loop(iteration_results) - - return { - 'test_type': 'feedback_loop', - 'num_files': len(audio_files), - 'num_iterations': num_iterations, - 'iteration_results': iteration_results, - 'feedback_analysis': feedback_analysis, - 'summary': self._generate_summary(iteration_results, feedback_analysis) - } - - def test_error_detection_accuracy( - self, - audio_files: List[str], - reference_transcripts: List[str], - known_errors: Optional[List[List[Dict]]] = None - ) -> Dict: - """ - Test accuracy of error detection component. - - Args: - audio_files: List of audio file paths - reference_transcripts: List of reference transcripts - known_errors: Optional list of known errors per file - - Returns: - Dictionary with error detection test results - """ - logger.info(f"Testing error detection accuracy: {len(audio_files)} files") - - detection_results = [] - - for audio_path, reference in zip(audio_files, reference_transcripts): - result = self.system.transcribe( - audio_path=audio_path, - reference_transcript=reference, - enable_auto_correction=False # Don't correct, just detect - ) - - errors_detected = result.get('error_detection', {}) - - detection_result = { - 'audio_file': str(audio_path), - 'errors_detected': errors_detected.get('error_count', 0), - 'error_types': errors_detected.get('error_types', {}), - 'has_errors': errors_detected.get('has_errors', False), - 'error_score': errors_detected.get('error_score', 0.0) - } - - # Compare with known errors if provided - if known_errors: - file_index = audio_files.index(audio_path) - known = known_errors[file_index] if file_index < len(known_errors) else [] - detection_result['known_errors'] = len(known) - detection_result['detection_accuracy'] = self._calculate_detection_accuracy( - errors_detected, known - ) - - detection_results.append(detection_result) - - return { - 'test_type': 'error_detection_accuracy', - 'num_files': len(audio_files), - 'detection_results': detection_results, - 'summary': { - 'total_errors_detected': sum(r['errors_detected'] for r in detection_results), - 'files_with_errors': sum(1 for r in detection_results if r['has_errors']), - 'average_error_score': np.mean([r['error_score'] for r in detection_results]) - } - } - - def test_correction_effectiveness( - self, - audio_files: List[str], - reference_transcripts: List[str] - ) -> Dict: - """ - Test effectiveness of correction mechanisms. - - Args: - audio_files: List of audio file paths - reference_transcripts: List of reference transcripts - - Returns: - Dictionary with correction effectiveness results - """ - logger.info(f"Testing correction effectiveness: {len(audio_files)} files") - - # Test without corrections - results_no_correction = [] - for audio_path, reference in zip(audio_files, reference_transcripts): - result = self.system.transcribe( - audio_path=audio_path, - reference_transcript=reference, - enable_auto_correction=False - ) - results_no_correction.append(result) - - # Test with corrections - results_with_correction = [] - for audio_path, reference in zip(audio_files, reference_transcripts): - result = self.system.transcribe( - audio_path=audio_path, - reference_transcript=reference, - enable_auto_correction=True - ) - results_with_correction.append(result) - - # Compare results - comparison = [] - for no_corr, with_corr in zip(results_no_correction, results_with_correction): - wer_no = no_corr.get('evaluation', {}).get('wer', float('inf')) - wer_with = with_corr.get('evaluation', {}).get('wer', float('inf')) - - improvement = wer_no - wer_with # Positive means improvement - - comparison.append({ - 'wer_without_correction': wer_no, - 'wer_with_correction': wer_with, - 'wer_improvement': improvement, - 'relative_improvement': (improvement / wer_no * 100) if wer_no > 0 else 0, - 'corrections_applied': with_corr.get('corrections', {}).get('count', 0) - }) - - return { - 'test_type': 'correction_effectiveness', - 'num_files': len(audio_files), - 'comparison': comparison, - 'summary': { - 'average_wer_without': np.mean([c['wer_without_correction'] for c in comparison]), - 'average_wer_with': np.mean([c['wer_with_correction'] for c in comparison]), - 'average_improvement': np.mean([c['wer_improvement'] for c in comparison]), - 'average_relative_improvement': np.mean([c['relative_improvement'] for c in comparison]), - 'total_corrections': sum(c['corrections_applied'] for c in comparison) - } - } - - def test_fine_tuning_impact( - self, - audio_files: List[str], - reference_transcripts: List[str], - trigger_fine_tuning: bool = True - ) -> Dict: - """ - Test impact of fine-tuning on system performance. - - Args: - audio_files: List of audio file paths - reference_transcripts: List of reference transcripts - trigger_fine_tuning: Whether to trigger fine-tuning during test - - Returns: - Dictionary with fine-tuning impact results - """ - logger.info(f"Testing fine-tuning impact: {len(audio_files)} files") - - # Baseline performance (before fine-tuning) - baseline_results = [] - for audio_path, reference in zip(audio_files, reference_transcripts): - result = self.system.transcribe( - audio_path=audio_path, - reference_transcript=reference - ) - baseline_results.append(result) - - baseline_wer = np.mean([r['evaluation']['wer'] for r in baseline_results if 'evaluation' in r]) - - # Trigger fine-tuning if enabled - if trigger_fine_tuning and self.system.enable_adaptive_fine_tuning: - logger.info("Triggering fine-tuning...") - fine_tuning_result = self.system.agent.manually_trigger_fine_tuning() - logger.info(f"Fine-tuning result: {fine_tuning_result.get('success', False)}") - - # Performance after fine-tuning - post_fine_tuning_results = [] - for audio_path, reference in zip(audio_files, reference_transcripts): - result = self.system.transcribe( - audio_path=audio_path, - reference_transcript=reference - ) - post_fine_tuning_results.append(result) - - post_wer = np.mean([r['evaluation']['wer'] for r in post_fine_tuning_results if 'evaluation' in r]) - - return { - 'test_type': 'fine_tuning_impact', - 'num_files': len(audio_files), - 'baseline_wer': baseline_wer, - 'post_fine_tuning_wer': post_wer, - 'wer_improvement': baseline_wer - post_wer, - 'relative_improvement': ((baseline_wer - post_wer) / baseline_wer * 100) if baseline_wer > 0 else 0, - 'fine_tuning_triggered': trigger_fine_tuning - } - - def _analyze_feedback_loop(self, iteration_results: List[Dict]) -> Dict: - """Analyze feedback loop effectiveness.""" - wers = [r['average_wer'] for r in iteration_results if r['average_wer'] is not None] - - if len(wers) < 2: - return {'status': 'insufficient_data'} - - # Check if performance is improving - improvement_trend = wers[-1] < wers[0] # Lower WER is better - - # Calculate improvement rate - if len(wers) >= 2: - total_improvement = wers[0] - wers[-1] - improvement_rate = total_improvement / len(wers) if len(wers) > 1 else 0 - else: - improvement_rate = 0 - - return { - 'initial_wer': wers[0], - 'final_wer': wers[-1], - 'improvement_trend': improvement_trend, - 'total_improvement': total_improvement, - 'improvement_rate': improvement_rate, - 'wer_trajectory': wers - } - - def _calculate_detection_accuracy( - self, - detected_errors: Dict, - known_errors: List[Dict] - ) -> Dict: - """Calculate error detection accuracy metrics.""" - detected_count = detected_errors.get('error_count', 0) - known_count = len(known_errors) - - if known_count == 0: - return { - 'precision': 1.0 if detected_count == 0 else 0.0, - 'recall': 1.0, - 'f1_score': 1.0 if detected_count == 0 else 0.0 - } - - # Simplified: assume detected errors match known errors - # In practice, would need more sophisticated matching - true_positives = min(detected_count, known_count) - precision = true_positives / detected_count if detected_count > 0 else 0.0 - recall = true_positives / known_count if known_count > 0 else 0.0 - f1_score = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0 - - return { - 'precision': precision, - 'recall': recall, - 'f1_score': f1_score, - 'true_positives': true_positives, - 'false_positives': max(0, detected_count - known_count), - 'false_negatives': max(0, known_count - detected_count) - } - - def _generate_summary( - self, - iteration_results: List[Dict], - feedback_analysis: Dict - ) -> Dict: - """Generate summary of feedback loop test.""" - return { - 'num_iterations': len(iteration_results), - 'average_wer_per_iteration': [r['average_wer'] for r in iteration_results if r['average_wer'] is not None], - 'total_errors_detected': sum(r['total_errors_detected'] for r in iteration_results), - 'total_corrections_applied': sum(r['total_corrections_applied'] for r in iteration_results), - 'fine_tuning_events': sum(1 for r in iteration_results if r['fine_tuning_triggered']), - 'feedback_effectiveness': feedback_analysis.get('improvement_trend', False), - 'total_improvement': feedback_analysis.get('total_improvement', 0) - } - - def run_full_test_suite( - self, - audio_files: List[str], - reference_transcripts: List[str] - ) -> Dict: - """ - Run complete end-to-end test suite. - - Args: - audio_files: List of audio file paths - reference_transcripts: List of reference transcripts - - Returns: - Dictionary with all test results - """ - logger.info("="*60) - logger.info("Running Full End-to-End Test Suite") - logger.info("="*60) - - all_results = {} - - # Test 1: Feedback loop - logger.info("\n1. Testing feedback loop...") - all_results['feedback_loop'] = self.test_feedback_loop( - audio_files, reference_transcripts, num_iterations=3 - ) - - # Test 2: Error detection accuracy - logger.info("\n2. Testing error detection accuracy...") - all_results['error_detection'] = self.test_error_detection_accuracy( - audio_files, reference_transcripts - ) - - # Test 3: Correction effectiveness - logger.info("\n3. Testing correction effectiveness...") - all_results['correction_effectiveness'] = self.test_correction_effectiveness( - audio_files, reference_transcripts - ) - - # Test 4: Fine-tuning impact - logger.info("\n4. Testing fine-tuning impact...") - all_results['fine_tuning_impact'] = self.test_fine_tuning_impact( - audio_files, reference_transcripts - ) - - return { - 'test_suite': 'end_to_end', - 'timestamp': datetime.now().isoformat(), - 'num_files': len(audio_files), - 'results': all_results, - 'system_status': self.system.get_system_status() - } diff --git a/src/integration/statistical_analysis.py b/src/integration/statistical_analysis.py deleted file mode 100644 index a126661..0000000 --- a/src/integration/statistical_analysis.py +++ /dev/null @@ -1,294 +0,0 @@ -""" -Statistical Analysis Module - Week 4 -Quantitative analysis with paired t-tests for statistical significance. -""" - -import logging -from typing import Dict, List, Tuple, Optional -import numpy as np -from scipy import stats -import pandas as pd -from pathlib import Path -import json - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -class StatisticalAnalyzer: - """ - Statistical analysis module for evaluating system performance. - Implements paired t-tests and other statistical methods. - """ - - def __init__(self): - """Initialize statistical analyzer.""" - self.analysis_results = [] - - def paired_t_test( - self, - baseline_scores: List[float], - treatment_scores: List[float], - alpha: float = 0.05, - alternative: str = 'two-sided' - ) -> Dict: - """ - Perform paired t-test to compare baseline vs treatment. - - Args: - baseline_scores: List of baseline performance scores - treatment_scores: List of treatment performance scores - alpha: Significance level (default: 0.05) - alternative: 'two-sided', 'less', or 'greater' - - Returns: - Dictionary with t-test results - """ - assert len(baseline_scores) == len(treatment_scores), \ - "Baseline and treatment scores must have same length" - - baseline_array = np.array(baseline_scores) - treatment_array = np.array(treatment_scores) - - # Calculate differences - differences = treatment_array - baseline_array - mean_diff = np.mean(differences) - std_diff = np.std(differences, ddof=1) - n = len(differences) - - # Perform paired t-test - t_statistic, p_value = stats.ttest_rel(baseline_array, treatment_array, alternative=alternative) - - # Calculate effect size (Cohen's d for paired samples) - cohens_d = mean_diff / std_diff if std_diff > 0 else 0 - - # Determine significance - is_significant = p_value < alpha - - # Calculate confidence interval - se_diff = std_diff / np.sqrt(n) - t_critical = stats.t.ppf(1 - alpha/2, df=n-1) if alternative == 'two-sided' else stats.t.ppf(1 - alpha, df=n-1) - ci_lower = mean_diff - t_critical * se_diff - ci_upper = mean_diff + t_critical * se_diff - - result = { - 'test_type': 'paired_t_test', - 'n_samples': n, - 'mean_baseline': np.mean(baseline_array), - 'mean_treatment': np.mean(treatment_array), - 'mean_difference': mean_diff, - 'std_difference': std_diff, - 't_statistic': t_statistic, - 'p_value': p_value, - 'alpha': alpha, - 'is_significant': is_significant, - 'cohens_d': cohens_d, - 'confidence_interval': (ci_lower, ci_upper), - 'alternative': alternative, - 'interpretation': self._interpret_t_test_result(mean_diff, p_value, alpha, alternative) - } - - return result - - def compare_systems( - self, - system_a_scores: List[float], - system_b_scores: List[float], - system_a_name: str = "System A", - system_b_name: str = "System B", - alpha: float = 0.05 - ) -> Dict: - """ - Compare two systems using paired t-test. - - Args: - system_a_scores: Performance scores for system A - system_b_scores: Performance scores for system B - system_a_name: Name of system A - system_b_name: Name of system B - alpha: Significance level - - Returns: - Dictionary with comparison results - """ - result = self.paired_t_test( - baseline_scores=system_a_scores, - treatment_scores=system_b_scores, - alpha=alpha - ) - - result['system_a_name'] = system_a_name - result['system_b_name'] = system_b_name - - # Determine which system is better (assuming lower scores are better, e.g., WER) - if result['mean_difference'] < 0: - better_system = system_b_name - improvement = abs(result['mean_difference']) - else: - better_system = system_a_name - improvement = result['mean_difference'] - - result['better_system'] = better_system - result['improvement'] = improvement - - return result - - def analyze_component_contributions( - self, - baseline_scores: List[float], - component_scores: Dict[str, List[float]], - alpha: float = 0.05 - ) -> Dict: - """ - Analyze contribution of individual components. - - Args: - baseline_scores: Baseline performance scores - component_scores: Dictionary mapping component names to their scores - alpha: Significance level - - Returns: - Dictionary with component contribution analysis - """ - contributions = {} - - for component_name, scores in component_scores.items(): - comparison = self.paired_t_test( - baseline_scores=baseline_scores, - treatment_scores=scores, - alpha=alpha - ) - - contributions[component_name] = { - 'mean_improvement': comparison['mean_difference'], - 'p_value': comparison['p_value'], - 'is_significant': comparison['is_significant'], - 'effect_size': comparison['cohens_d'], - 'interpretation': comparison['interpretation'] - } - - return { - 'baseline_mean': np.mean(baseline_scores), - 'component_contributions': contributions, - 'significant_components': [ - name for name, contrib in contributions.items() - if contrib['is_significant'] - ] - } - - def analyze_trajectory( - self, - iteration_scores: List[List[float]], - alpha: float = 0.05 - ) -> Dict: - """ - Analyze performance trajectory across iterations. - - Args: - iteration_scores: List of score lists, one per iteration - alpha: Significance level - - Returns: - Dictionary with trajectory analysis - """ - if len(iteration_scores) < 2: - return {'status': 'insufficient_data'} - - # Calculate mean scores per iteration - mean_scores = [np.mean(scores) for scores in iteration_scores] - - # Test if final iteration is significantly better than first - first_iteration = iteration_scores[0] - final_iteration = iteration_scores[-1] - - improvement_test = self.paired_t_test( - baseline_scores=first_iteration, - treatment_scores=final_iteration, - alpha=alpha, - alternative='less' # Testing if final is better (lower scores) - ) - - # Calculate trend - x = np.arange(len(mean_scores)) - slope, intercept, r_value, p_value_trend, std_err = stats.linregress(x, mean_scores) - - return { - 'mean_scores_per_iteration': mean_scores, - 'total_improvement': mean_scores[0] - mean_scores[-1], - 'improvement_test': improvement_test, - 'trend_slope': slope, - 'trend_p_value': p_value_trend, - 'trend_r_squared': r_value ** 2, - 'is_improving': slope < 0 and improvement_test['is_significant'] - } - - def _interpret_t_test_result( - self, - mean_diff: float, - p_value: float, - alpha: float, - alternative: str - ) -> str: - """Interpret t-test result in plain language.""" - if p_value < alpha: - if alternative == 'two-sided': - if mean_diff < 0: - return f"Treatment is significantly better (p={p_value:.4f} < {alpha})" - else: - return f"Treatment is significantly worse (p={p_value:.4f} < {alpha})" - elif alternative == 'less': - return f"Treatment is significantly better (p={p_value:.4f} < {alpha})" - else: # greater - return f"Treatment is significantly worse (p={p_value:.4f} < {alpha})" - else: - return f"No significant difference (p={p_value:.4f} >= {alpha})" - - def generate_report( - self, - analysis_results: List[Dict], - output_path: Optional[str] = None - ) -> str: - """ - Generate statistical analysis report. - - Args: - analysis_results: List of analysis result dictionaries - output_path: Optional path to save report - - Returns: - Report string - """ - report_lines = [] - report_lines.append("="*60) - report_lines.append("Statistical Analysis Report") - report_lines.append("="*60) - report_lines.append("") - - for i, result in enumerate(analysis_results, 1): - report_lines.append(f"Analysis {i}: {result.get('test_type', 'unknown')}") - report_lines.append("-"*60) - - if 'mean_baseline' in result: - report_lines.append(f" Baseline Mean: {result['mean_baseline']:.4f}") - report_lines.append(f" Treatment Mean: {result['mean_treatment']:.4f}") - report_lines.append(f" Mean Difference: {result['mean_difference']:.4f}") - - if 'p_value' in result: - report_lines.append(f" p-value: {result['p_value']:.4f}") - report_lines.append(f" Significant: {result['is_significant']}") - report_lines.append(f" Effect Size (Cohen's d): {result.get('cohens_d', 0):.4f}") - - if 'interpretation' in result: - report_lines.append(f" Interpretation: {result['interpretation']}") - - report_lines.append("") - - report = "\n".join(report_lines) - - if output_path: - Path(output_path).parent.mkdir(parents=True, exist_ok=True) - with open(output_path, 'w') as f: - f.write(report) - logger.info(f"Report saved to {output_path}") - - return report diff --git a/src/integration/unified_system.py b/src/integration/unified_system.py deleted file mode 100644 index 3f3ffa1..0000000 --- a/src/integration/unified_system.py +++ /dev/null @@ -1,285 +0,0 @@ -""" -Unified System Architecture - Week 4 -Integrates all components into a single cohesive system. -""" - -import logging -from typing import Dict, Optional, List, Tuple, Any -from pathlib import Path -import time -import json -from datetime import datetime - -from ..baseline_model import BaselineSTTModel -from ..agent import STTAgent -from ..evaluation.metrics import STTEvaluator - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -class UnifiedSTTSystem: - """ - Unified system that integrates all components: - - Baseline STT Model - - Agent (Error Detection, Self-Learning, LLM Correction) - - Adaptive Scheduler & Fine-Tuner - - Evaluation Metrics - """ - - def __init__( - self, - model_name: str = "whisper", - enable_error_detection: bool = True, - enable_llm_correction: bool = True, - enable_adaptive_fine_tuning: bool = True, - error_threshold: float = 0.3, - scheduler_history_path: Optional[str] = None, - config: Optional[Dict] = None - ): - """ - Initialize unified STT system with all components. - - Args: - model_name: Baseline model name ("whisper") - enable_error_detection: Enable error detection component - enable_llm_correction: Enable LLM-based correction - enable_adaptive_fine_tuning: Enable adaptive fine-tuning (Week 3) - error_threshold: Error detection threshold - scheduler_history_path: Path for scheduler history - config: Additional configuration dictionary - """ - self.config = config or {} - self.model_name = model_name - - # Component flags - self.enable_error_detection = enable_error_detection - self.enable_llm_correction = enable_llm_correction - self.enable_adaptive_fine_tuning = enable_adaptive_fine_tuning - - # Initialize baseline model - logger.info(f"Initializing baseline model: {model_name}") - self.baseline_model = BaselineSTTModel(model_name=model_name) - - # Initialize agent with all components - logger.info("Initializing STT Agent with integrated components...") - self.agent = STTAgent( - baseline_model=self.baseline_model, - error_threshold=error_threshold, - use_llm_correction=enable_llm_correction, - enable_adaptive_fine_tuning=enable_adaptive_fine_tuning, - scheduler_history_path=scheduler_history_path - ) - - # Initialize evaluator - self.evaluator = STTEvaluator() - - # System statistics - self.system_stats = { - 'initialization_time': datetime.now().isoformat(), - 'total_transcriptions': 0, - 'total_errors_detected': 0, - 'total_corrections_applied': 0, - 'fine_tuning_events': 0, - 'component_status': self._get_component_status() - } - - logger.info("โœ… Unified STT System initialized successfully") - - def _get_component_status(self) -> Dict[str, bool]: - """Get status of all system components.""" - return { - 'baseline_model': self.baseline_model is not None, - 'error_detection': self.enable_error_detection and self.agent.error_detector is not None, - 'self_learning': self.agent.self_learner is not None, - 'llm_correction': self.enable_llm_correction and self.agent.llm_corrector is not None, - 'adaptive_scheduler': self.enable_adaptive_fine_tuning and self.agent.adaptive_scheduler is not None, - 'fine_tuner': self.enable_adaptive_fine_tuning and self.agent.fine_tuner is not None - } - - def transcribe( - self, - audio_path: str, - reference_transcript: Optional[str] = None, - enable_auto_correction: bool = True - ) -> Dict: - """ - Transcribe audio with full system pipeline. - - Args: - audio_path: Path to audio file - reference_transcript: Ground truth transcript (for evaluation) - enable_auto_correction: Whether to apply automatic corrections - - Returns: - Dictionary with transcription results and metadata - """ - start_time = time.time() - - # Get audio length for error detection - import librosa - audio_length = librosa.get_duration(filename=audio_path) - - # Transcribe with agent (includes error detection, correction, learning) - result = self.agent.transcribe_with_agent( - audio_path=audio_path, - audio_length_seconds=audio_length, - enable_auto_correction=enable_auto_correction - ) - - # Evaluate if reference provided - evaluation_results = None - if reference_transcript: - transcript = result.get('transcript', '') - wer = self.evaluator.calculate_wer(reference_transcript, transcript) - cer = self.evaluator.calculate_cer(reference_transcript, transcript) - - evaluation_results = { - 'wer': wer, - 'cer': cer, - 'reference': reference_transcript, - 'hypothesis': transcript - } - - # Update system statistics - self.system_stats['total_transcriptions'] += 1 - if result.get('error_detection', {}).get('has_errors', False): - self.system_stats['total_errors_detected'] += result['error_detection']['error_count'] - if result.get('corrections', {}).get('applied', False): - self.system_stats['total_corrections_applied'] += result['corrections']['count'] - if result.get('agent_metadata', {}).get('fine_tuning_triggered', False): - self.system_stats['fine_tuning_events'] += 1 - - processing_time = time.time() - start_time - - # Compile full result - full_result = { - **result, - 'system_metadata': { - 'processing_time': processing_time, - 'components_enabled': self._get_component_status(), - 'system_stats': self.system_stats.copy() - } - } - - if evaluation_results: - full_result['evaluation'] = evaluation_results - - return full_result - - def evaluate_batch( - self, - audio_files: List[str], - reference_transcripts: List[str], - enable_auto_correction: bool = True - ) -> Dict: - """ - Evaluate system on a batch of audio files. - - Args: - audio_files: List of audio file paths - reference_transcripts: List of reference transcripts - enable_auto_correction: Whether to apply corrections - - Returns: - Dictionary with batch evaluation results - """ - assert len(audio_files) == len(reference_transcripts), \ - "Audio files and reference transcripts must have same length" - - logger.info(f"Evaluating batch of {len(audio_files)} files...") - - results = [] - total_time = 0.0 - - for i, (audio_path, reference) in enumerate(zip(audio_files, reference_transcripts)): - logger.info(f"Processing {i+1}/{len(audio_files)}: {Path(audio_path).name}") - - result = self.transcribe( - audio_path=audio_path, - reference_transcript=reference, - enable_auto_correction=enable_auto_correction - ) - - results.append(result) - total_time += result['system_metadata']['processing_time'] - - # Calculate aggregate metrics - wers = [r['evaluation']['wer'] for r in results if 'evaluation' in r] - cers = [r['evaluation']['cer'] for r in results if 'evaluation' in r] - - batch_results = { - 'num_samples': len(results), - 'average_wer': sum(wers) / len(wers) if wers else None, - 'average_cer': sum(cers) / len(cers) if cers else None, - 'total_processing_time': total_time, - 'average_processing_time': total_time / len(results) if results else 0, - 'total_errors_detected': sum(r.get('error_detection', {}).get('error_count', 0) for r in results), - 'total_corrections_applied': sum(r.get('corrections', {}).get('count', 0) for r in results), - 'fine_tuning_events': sum(1 for r in results if r.get('agent_metadata', {}).get('fine_tuning_triggered', False)), - 'detailed_results': results - } - - return batch_results - - def get_system_status(self) -> Dict: - """Get comprehensive system status.""" - agent_stats = self.agent.get_agent_stats() - scheduler_stats = self.agent.get_adaptive_scheduler_stats() - - return { - 'component_status': self._get_component_status(), - 'system_statistics': self.system_stats, - 'agent_statistics': agent_stats, - 'scheduler_statistics': scheduler_stats, - 'model_info': self.baseline_model.get_model_info() - } - - def get_component_contributions(self) -> Dict: - """ - Get contribution of each component to system performance. - Useful for ablation studies. - """ - return { - 'baseline_model': { - 'enabled': True, - 'description': 'Core STT transcription' - }, - 'error_detection': { - 'enabled': self.enable_error_detection, - 'description': 'Multi-heuristic error detection' - }, - 'self_learning': { - 'enabled': True, - 'description': 'Error pattern tracking and learning' - }, - 'llm_correction': { - 'enabled': self.enable_llm_correction, - 'description': 'LLM-based intelligent correction' - }, - 'adaptive_scheduler': { - 'enabled': self.enable_adaptive_fine_tuning, - 'description': 'Adaptive fine-tuning scheduling' - }, - 'fine_tuner': { - 'enabled': self.enable_adaptive_fine_tuning, - 'description': 'Automated model fine-tuning' - } - } - - def save_system_state(self, output_path: str): - """Save system state for later restoration.""" - state = { - 'config': self.config, - 'system_stats': self.system_stats, - 'component_status': self._get_component_status(), - 'timestamp': datetime.now().isoformat() - } - - output_path = Path(output_path) - output_path.parent.mkdir(parents=True, exist_ok=True) - - with open(output_path, 'w') as f: - json.dump(state, f, indent=2) - - logger.info(f"System state saved to {output_path}") diff --git a/src/model_selector.py b/src/model_selector.py deleted file mode 100644 index 182e65a..0000000 --- a/src/model_selector.py +++ /dev/null @@ -1,116 +0,0 @@ -""" -Task 1: Evaluate Whisper vs Wav2Vec2 for cost-effectiveness and performance -Run this first in a notebook to compare models and decide which to deploy -""" - -import torch -import time -from transformers import ( - WhisperProcessor, WhisperForConditionalGeneration, - Wav2Vec2Processor, Wav2Vec2ForCTC -) -import librosa -import numpy as np - -class STTModelEvaluator: - """Compare STT models on key metrics""" - - def __init__(self, device="cuda" if torch.cuda.is_available() else "cpu"): - self.device = device - self.results = {} - - def load_whisper_base(self): - """Load Whisper base model (~140M parameters)""" - processor = WhisperProcessor.from_pretrained("openai/whisper-base") - model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-base") - model.to(self.device) - return processor, model - - def load_wav2vec2_base(self): - """Load Wav2Vec2 base model (~95M parameters, lighter)""" - processor = Wav2Vec2Processor.from_pretrained("facebook/wav2vec2-base-960h") - model = Wav2Vec2ForCTC.from_pretrained("facebook/wav2vec2-base-960h") - model.to(self.device) - return processor, model - - def benchmark_inference(self, audio_path, processor, model, model_name): - """Measure latency and memory for a single inference""" - # Load audio - audio, sr = librosa.load(audio_path, sr=16000) - - # Warm up - _ = self._run_inference(audio, sr, processor, model, model_name) - - # Measure latency - start = time.time() - transcript = self._run_inference(audio, sr, processor, model, model_name) - latency = time.time() - start - - # Memory estimate - param_count = sum(p.numel() for p in model.parameters()) - - return { - "transcript": transcript, - "latency_seconds": latency, - "parameters": param_count, - "model_size_mb": param_count * 4 / (1024**2) # approximate - } - - def _run_inference(self, audio, sr, processor, model, model_name): - """Internal inference wrapper""" - if "whisper" in model_name.lower(): - inputs = processor(audio, sampling_rate=sr, return_tensors="pt") - with torch.no_grad(): - predicted_ids = model.generate( - inputs["input_features"].to(self.device), - max_new_tokens=128 - ) - return processor.batch_decode(predicted_ids, skip_special_tokens=True)[0] - else: - inputs = processor(audio, sampling_rate=sr, return_tensors="pt") - with torch.no_grad(): - logits = model(inputs["input_values"].to(self.device)).logits - predicted_ids = torch.argmax(logits, dim=-1) - return processor.batch_decode(predicted_ids)[0] - - def compare_models(self, test_audio_paths): - """Run comparison and return summary""" - print("๐Ÿ” Loading models...") - whisper_proc, whisper_model = self.load_whisper_base() - wav2vec_proc, wav2vec_model = self.load_wav2vec2_base() - - print("\n๐Ÿ“Š Benchmarking on", len(test_audio_paths), "samples...") - - whisper_results = [] - wav2vec_results = [] - - for audio_path in test_audio_paths: - try: - w_result = self.benchmark_inference(audio_path, whisper_proc, whisper_model, "whisper") - whisper_results.append(w_result) - except Exception as e: - print(f"โš ๏ธ Whisper failed on {audio_path}: {e}") - - try: - w2_result = self.benchmark_inference(audio_path, wav2vec_proc, wav2vec_model, "wav2vec2") - wav2vec_results.append(w2_result) - except Exception as e: - print(f"โš ๏ธ Wav2Vec2 failed on {audio_path}: {e}") - - # Aggregate metrics - summary = { - "whisper": { - "avg_latency": np.mean([r["latency_seconds"] for r in whisper_results]), - "model_size_mb": whisper_results[0]["model_size_mb"] if whisper_results else None, - "parameters": whisper_results[0]["parameters"] if whisper_results else None, - "samples_processed": len(whisper_results) - }, - "wav2vec2": { - "avg_latency": np.mean([r["latency_seconds"] for r in wav2vec_results]), - "model_size_mb": wav2vec_results[0]["model_size_mb"] if wav2vec_results else None, - "parameters": wav2vec_results[0]["parameters"] if wav2vec_results else None, - "samples_processed": len(wav2vec_results) - } - } - - return summary diff --git a/tests/run_all_tests.py b/tests/run_all_tests.py deleted file mode 100644 index 68e75c9..0000000 --- a/tests/run_all_tests.py +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env python -""" -Master test runner for all test suites -Runs unit tests, integration tests, and API tests with comprehensive reporting -""" - -import sys -from pathlib import Path -sys.path.insert(0, str(Path(__file__).parent.parent)) - -import subprocess -import argparse -from datetime import datetime -import json - - -class TestRunner: - """Master test runner""" - - def __init__(self): - self.results = { - "timestamp": datetime.now().isoformat(), - "test_suites": {}, - "summary": { - "total_passed": 0, - "total_failed": 0, - "total_skipped": 0 - } - } - - def run_pytest(self, test_path, description, markers=None): - """Run pytest on a specific path""" - print(f"\n{'=' * 70}") - print(f"Running: {description}") - print(f"{'=' * 70}\n") - - cmd = ["pytest", str(test_path), "-v", "--tb=short"] - - if markers: - cmd.extend(["-m", markers]) - - try: - result = subprocess.run(cmd, capture_output=True, text=True) - - # Parse output for pass/fail counts (simple parsing) - output = result.stdout - - passed = output.count(" PASSED") - failed = output.count(" FAILED") - skipped = output.count(" SKIPPED") - - self.results["test_suites"][description] = { - "passed": passed, - "failed": failed, - "skipped": skipped, - "exit_code": result.returncode - } - - self.results["summary"]["total_passed"] += passed - self.results["summary"]["total_failed"] += failed - self.results["summary"]["total_skipped"] += skipped - - print(output) - - if result.returncode == 0: - print(f"\nโœ… {description}: PASSED") - else: - print(f"\nโŒ {description}: FAILED") - - return result.returncode == 0 - - except Exception as e: - print(f"โŒ Error running {description}: {e}") - return False - - def print_summary(self): - """Print test summary""" - print("\n" + "=" * 70) - print("TEST SUMMARY") - print("=" * 70) - - for suite, results in self.results["test_suites"].items(): - status = "โœ… PASSED" if results["exit_code"] == 0 else "โŒ FAILED" - print(f"\n{suite}: {status}") - print(f" Passed: {results['passed']}") - print(f" Failed: {results['failed']}") - print(f" Skipped: {results['skipped']}") - - print(f"\n{'=' * 70}") - print("OVERALL SUMMARY") - print(f"{'=' * 70}") - print(f"Total Passed: {self.results['summary']['total_passed']}") - print(f"Total Failed: {self.results['summary']['total_failed']}") - print(f"Total Skipped: {self.results['summary']['total_skipped']}") - - total_tests = ( - self.results['summary']['total_passed'] + - self.results['summary']['total_failed'] - ) - - if total_tests > 0: - pass_rate = (self.results['summary']['total_passed'] / total_tests) * 100 - print(f"Pass Rate: {pass_rate:.1f}%") - - all_passed = all( - suite["exit_code"] == 0 - for suite in self.results["test_suites"].values() - ) - - if all_passed and self.results['summary']['total_failed'] == 0: - print(f"\n{'๐ŸŽ‰' * 35}") - print("ALL TESTS PASSED!") - print(f"{'๐ŸŽ‰' * 35}") - else: - print(f"\nโš ๏ธ Some tests failed. Please review the output above.") - - def save_results(self, output_path="test_results.json"): - """Save test results to JSON""" - with open(output_path, 'w') as f: - json.dump(self.results, f, indent=2) - print(f"\n๐Ÿ“Š Test results saved to: {output_path}") - - -def main(): - """Main test runner""" - parser = argparse.ArgumentParser(description="Run test suites") - parser.add_argument( - "--suite", - choices=["all", "unit", "integration", "api", "quick"], - default="all", - help="Test suite to run" - ) - parser.add_argument( - "--save-results", - action="store_true", - help="Save test results to JSON" - ) - - args = parser.parse_args() - - runner = TestRunner() - tests_dir = Path(__file__).parent - - print("=" * 70) - print("ADAPTIVE STT SYSTEM - TEST SUITE RUNNER") - print("=" * 70) - print(f"Test Suite: {args.suite.upper()}") - print(f"Timestamp: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") - print("=" * 70) - - all_passed = True - - if args.suite in ["all", "unit", "quick"]: - # Run unit tests - all_passed &= runner.run_pytest( - tests_dir / "test_metrics.py", - "Unit Tests: Metrics (WER/CER)" - ) - - all_passed &= runner.run_pytest( - tests_dir / "test_error_detector.py", - "Unit Tests: Error Detector" - ) - - all_passed &= runner.run_pytest( - tests_dir / "test_benchmark.py", - "Unit Tests: Benchmark" - ) - - if args.suite in ["all", "integration"]: - # Run integration tests - all_passed &= runner.run_pytest( - tests_dir / "test_integration.py", - "Integration Tests: Complete Workflow" - ) - - if args.suite in ["all", "api"]: - # Run API tests - print("\nโš ๏ธ Note: API tests require the server to be running:") - print(" uvicorn src.agent_api:app --port 8000\n") - - all_passed &= runner.run_pytest( - tests_dir / "test_api_comprehensive.py", - "API Tests: Comprehensive Endpoint Testing" - ) - - # Print summary - runner.print_summary() - - # Save results if requested - if args.save_results: - runner.save_results() - - # Exit with appropriate code - sys.exit(0 if all_passed else 1) - - -if __name__ == "__main__": - main() - diff --git a/tests/run_orchestration_tests.py b/tests/run_orchestration_tests.py deleted file mode 100644 index 61fd2e5..0000000 --- a/tests/run_orchestration_tests.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 -""" -Run Fine-Tuning Orchestration Tests - -Convenience script to run all fine-tuning orchestration tests. -""" - -import sys -import subprocess -from pathlib import Path - -# Test files for fine-tuning orchestration -ORCHESTRATION_TESTS = [ - "test_finetuning_orchestrator.py", - "test_model_validator.py", - "test_model_deployer.py", - "test_regression_tester.py", - "test_finetuning_coordinator.py" -] - - -def run_tests(verbose=True, coverage=False): - """Run orchestration tests.""" - test_dir = Path(__file__).parent - - print("="*80) - print("FINE-TUNING ORCHESTRATION TESTS") - print("="*80) - print() - - # Build pytest command - cmd = ["pytest"] - - # Add test files - for test_file in ORCHESTRATION_TESTS: - cmd.append(str(test_dir / test_file)) - - # Add options - if verbose: - cmd.append("-v") - - cmd.append("--tb=short") - cmd.append("-m") - cmd.append("unit") - - if coverage: - cmd.extend([ - "--cov=src/data", - "--cov-report=term-missing", - "--cov-report=html" - ]) - - print(f"Running: {' '.join(cmd)}") - print() - - # Run tests - result = subprocess.run(cmd) - - print() - print("="*80) - if result.returncode == 0: - print("โœ… ALL TESTS PASSED") - else: - print("โŒ SOME TESTS FAILED") - print("="*80) - - return result.returncode - - -def main(): - """Main entry point.""" - import argparse - - parser = argparse.ArgumentParser( - description="Run fine-tuning orchestration tests" - ) - parser.add_argument( - "--coverage", - action="store_true", - help="Generate coverage report" - ) - parser.add_argument( - "--quiet", - action="store_true", - help="Quiet mode (less verbose)" - ) - - args = parser.parse_args() - - returncode = run_tests( - verbose=not args.quiet, - coverage=args.coverage - ) - - sys.exit(returncode) - - -if __name__ == "__main__": - main() -