run.py

#!/usr/bin/env python3
"""
AgentLead - Main Application Entry Point and CLI Interface

A comprehensive lead extraction and research platform that integrates
Kimi K2 extraction, AI-powered research, personalized outreach, and cost tracking.

Features:
- Command-line interface for all operations
- Web dashboard for monitoring (optional)
- Batch processing capabilities
- Configuration validation
- Health checks and status reporting
- Integration of all components

Usage:
    python run.py extract --url https://company.com
    python run.py outreach --leads-file leads.csv --output outreach_campaigns
    python run.py monitor --dashboard
    python run.py health --full
    python run.py batch process --config batch_config.json
"""

import argparse
import asyncio
import csv
import json
import sys
import os
import logging
import signal
import time
import glob
from pathlib import Path
from datetime import datetime
from typing import Dict, List, Optional, Any, Union
from dataclasses import dataclass, asdict
from decimal import Decimal
import threading
import multiprocessing
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor

# Add project root to path
project_root = Path(__file__).parent
sys.path.append(str(project_root))

# Import core components with error handling
try:
    import sys
    sys.path.insert(0, str(project_root))
    from src.core.kimi_extractor import KimiK2Extractor, IndustryType, LeadQuality
    HAS_KIMI_EXTRACTOR = True
except ImportError as e:
    print(f"Warning: Could not import Kimi extractor: {e}")
    HAS_KIMI_EXTRACTOR = False
    # Define dummy classes
    class IndustryType:
        TECHNOLOGY = "technology"
    class LeadQuality:
        HOT = "hot"

# Cost tracking removed - was causing complexity without proper integration

try:
    from src.core.crawler_manager import CrawlerManager
    HAS_CRAWLER_MANAGER = True
except ImportError as e:
    print(f"Warning: Could not import crawler manager: {e}")
    HAS_CRAWLER_MANAGER = False

# SmartLead client removed - not in use

# Clay enrichment functionality removed - not in use

# Import multi-agent components
try:
    from src.agents import (
        OutreachCrew, LeadProfile, CompanyResearcher,
        set_progress_callback, print_progress_update
    )
    from src.core.research_exporter import ResearchExporter, quick_export_research
    HAS_MULTI_AGENT = True
except ImportError as e:
    print(f"Warning: Could not import multi-agent components: {e}")
    HAS_MULTI_AGENT = False

try:
    from config.kimi_config import get_config, validate_config, get_industry_config
    HAS_KIMI_CONFIG = True
except ImportError as e:
    print(f"Warning: Could not import Kimi config: {e}")
    HAS_KIMI_CONFIG = False

# Import utilities with fallbacks
from dotenv import load_dotenv

try:
    import structlog
    HAS_STRUCTLOG = True
except ImportError:
    import logging
    structlog = logging
    HAS_STRUCTLOG = False

try:
    from rich.console import Console
    from rich.table import Table
    from rich.progress import Progress, TaskID
    from rich.panel import Panel
    from rich.text import Text
    from rich import print as rprint
    HAS_RICH = True
    console = Console()
except ImportError:
    HAS_RICH = False
    console = None
    def rprint(text):
        print(text)

try:
    import click
    HAS_CLICK = True
except ImportError:
    HAS_CLICK = False
    print("Error: Click is required for CLI functionality. Please install with: pip install click")

# Load environment variables
load_dotenv()

# Configure structured logging
if HAS_STRUCTLOG:
    structlog.configure(
        processors=[
            structlog.stdlib.filter_by_level,
            structlog.stdlib.add_logger_name,
            structlog.stdlib.add_log_level,
            structlog.stdlib.PositionalArgumentsFormatter(),
            structlog.processors.TimeStamper(fmt="iso"),
            structlog.processors.StackInfoRenderer(),
            structlog.processors.format_exc_info,
            structlog.processors.UnicodeDecoder(),
            structlog.processors.JSONRenderer()
        ],
        context_class=dict,
        logger_factory=structlog.stdlib.LoggerFactory(),
        wrapper_class=structlog.stdlib.BoundLogger,
        cache_logger_on_first_use=True,
    )
    logger = structlog.get_logger(__name__)
else:
    logger = logging.getLogger(__name__)


@dataclass
class AppConfig:
    """Main application configuration."""
    debug: bool = False
    log_level: str = "INFO"
    max_workers: int = 4
    batch_size: int = 10
    enable_dashboard: bool = False
    dashboard_port: int = 8080
    health_check_interval: int = 300  # 5 minutes
# Cost tracking removed
    auto_save_interval: int = 60  # seconds
    
    # Clay enrichment configuration removed - not in use
    
    def __post_init__(self):
        """Validate configuration."""
        if self.max_workers <= 0:
            self.max_workers = multiprocessing.cpu_count()
        if self.batch_size <= 0:
            self.batch_size = 10
        # Clay configuration removed


class AgentLeadApp:
    """Main AgentLead application class."""
    
    def __init__(self, config: Optional[AppConfig] = None):
        """Initialize the AgentLead application."""
        self.config = config or AppConfig()
        
        # Initialize config if available
        if HAS_KIMI_CONFIG:
            self.kimi_config = get_config()
        else:
            self.kimi_config = None
            
        self.running = False
        self.health_status = {}
        
        # Initialize components
        self.extractor = None
# Cost tracking removed
        # SmartLead client removed
        self.crawler_manager = None
        # Clay client removed
        
        # Background tasks
        self.health_check_task = None
        self.auto_save_task = None
        
        # Setup logging
        self._setup_logging()
        
        # Signal handlers
        signal.signal(signal.SIGINT, self._signal_handler)
        signal.signal(signal.SIGTERM, self._signal_handler)
    
    def _setup_logging(self):
        """Setup application logging."""
        level = getattr(logging, self.config.log_level.upper(), logging.INFO)
        logging.basicConfig(
            level=level,
            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
        )
        
        if self.config.debug:
            logging.getLogger().setLevel(logging.DEBUG)
    
    def _signal_handler(self, signum, frame):
        """Handle shutdown signals."""
        if HAS_STRUCTLOG:
            logger.info("Received shutdown signal", signal=signum)
        else:
            logger.info(f"Received shutdown signal: {signum}")
        self.shutdown()
    
    async def initialize(self) -> bool:
        """Initialize all application components."""
        print("Initializing AgentLead application")
        
        try:
            # Validate configuration if available
            if HAS_KIMI_CONFIG and self.kimi_config:
                validate_config(self.kimi_config)
                print("✓ Configuration validated")
            
            # Cost tracking removed - was causing complexity without proper integration
            
            # Initialize Kimi K2 extractor
            if HAS_KIMI_EXTRACTOR:
                # Get API key from environment
                api_key = os.getenv('OPENROUTER_API_KEY') or os.getenv('KIMI_API_KEY')
                if api_key:
                    self.extractor = KimiK2Extractor(
                        api_key=api_key,
                        base_url="https://openrouter.ai/api/v1" if os.getenv('OPENROUTER_API_KEY') else "https://api.moonshot.cn/v1"
                    )
                    print("✓ Kimi K2 extractor initialized")
                else:
                    print("⚠️ No API key found for Kimi K2 extractor")
                    self.extractor = None
            else:
                print("⚠️ Kimi K2 extractor not available")
                self.extractor = None
            
            # SmartLead client initialization removed
            
            # Initialize crawler manager
            if HAS_CRAWLER_MANAGER:
                self.crawler_manager = CrawlerManager()
                print("✓ Crawler manager initialized")
            else:
                print("⚠️ Crawler manager not available")
            
            # Clay integration removed - not in use
            
            # Update health status
            await self.check_health()
            
            self.running = True
            print("✅ AgentLead application initialized successfully")
            return True
            
        except Exception as e:
            print(f"❌ Failed to initialize application: {e}")
            return False
    
    async def shutdown(self):
        """Shutdown the application gracefully."""
        if HAS_STRUCTLOG:
            logger.info("Shutting down AgentLead application")
        else:
            logger.info("Shutting down AgentLead application")
        self.running = False
        
        # Cancel background tasks
        if self.health_check_task:
            self.health_check_task.cancel()
        if self.auto_save_task:
            self.auto_save_task.cancel()
        
        # Cleanup components
# Cost tracking removed
        
        if self.extractor:
            await self.extractor.close()
        
        # Clay client shutdown removed
        
        print("✅ Application shutdown complete")
    
    async def check_health(self) -> Dict[str, Any]:
        """Perform comprehensive health checks."""
        health_status = {
            "timestamp": datetime.now().isoformat(),
            "overall_status": "healthy",
            "components": {}
        }
        
        # Check Kimi K2 extractor
        try:
            if self.extractor:
                # Simple API connectivity test
                test_result = await self.extractor.test_connection()
                health_status["components"]["kimi_extractor"] = {
                    "status": "healthy" if test_result else "unhealthy",
                    "last_check": datetime.now().isoformat()
                }
            else:
                health_status["components"]["kimi_extractor"] = {
                    "status": "not_initialized",
                    "last_check": datetime.now().isoformat()
                }
        except Exception as e:
            health_status["components"]["kimi_extractor"] = {
                "status": "error",
                "error": str(e),
                "last_check": datetime.now().isoformat()
            }
        
        # Cost tracking removed from health checks
        
        # SmartLead client health check removed
        
        # Clay client health check removed
        
        # Check system resources
        try:
            import psutil
            health_status["system"] = {
                "cpu_percent": psutil.cpu_percent(),
                "memory_percent": psutil.virtual_memory().percent,
                "disk_usage": psutil.disk_usage('/').percent
            }
        except ImportError:
            health_status["system"] = {"status": "monitoring_unavailable"}
        
        # Determine overall status
        component_statuses = [
            comp["status"] for comp in health_status["components"].values()
        ]
        if "error" in component_statuses or "unhealthy" in component_statuses:
            health_status["overall_status"] = "degraded"
        elif all(status in ["healthy", "disabled", "not_configured"] for status in component_statuses):
            health_status["overall_status"] = "healthy"
        else:
            health_status["overall_status"] = "unknown"
        
        self.health_status = health_status
        return health_status
    
    async def extract_leads(self, 
                          content: str = None,
                          url: str = None,
                          file_path: str = None,
                          industry: str = "technology",
                          output_format: str = "json",
                          output_file: str = None) -> Dict[str, Any]:
        """Extract leads from various sources."""
        if not self.extractor:
            raise RuntimeError("Extractor not initialized")
        
        # Extract lead using appropriate method
        industry_type = IndustryType(industry.upper()) if hasattr(IndustryType, industry.upper()) else IndustryType.TECHNOLOGY
        
        if url:
            # Direct URL extraction using integrated Firecrawl crawler
            result = await self.extractor.extract_lead_from_url(
                url=url,
                industry=industry_type
            )
        else:
            # Content-based extraction
            if file_path:
                with open(file_path, 'r', encoding='utf-8') as f:
                    content = f.read()
            elif not content:
                raise ValueError("Must provide content, URL, or file path")
            
            result = await self.extractor.extract_lead(
                content=content,
                industry=industry_type,
                source_url=url
            )
        
        # Format output
        if result.success:
            output_data = {
                "extraction_timestamp": datetime.now().isoformat(),
                "source": url or file_path or "direct_input",
                "industry": industry,
                "lead_data": result.lead_data.model_dump() if result.lead_data else None,
                "metadata": {
                    "extractor_version": "1.0.0",
                    "processing_time": result.processing_time,
                    "token_usage": result.token_usage,
                    "extraction_id": result.extraction_id,
                    "retry_count": result.retry_count
                }
            }
        else:
            raise RuntimeError(f"Lead extraction failed: {result.error_message}")
        
        # Save output if requested
        if output_file:
            if output_format.lower() == "json":
                with open(output_file, 'w', encoding='utf-8') as f:
                    json.dump(output_data, f, indent=2, default=str)
            elif output_format.lower() == "csv":
                import pandas as pd
                df = pd.json_normalize(output_data["lead_data"])
                df.to_csv(output_file, index=False)
        
        return output_data
    
    async def batch_process_leads(self, 
                                config_file: str,
                                progress_callback=None,
                                resume_from: int = 0) -> Dict[str, Any]:
        """Process multiple leads in batch with incremental updates and resume capability."""
        with open(config_file, 'r') as f:
            batch_config = json.load(f)
        
        inputs = batch_config.get("inputs", [])
        industry = batch_config.get("industry", "technology")
        output_dir = batch_config.get("output_directory", "batch_results")
        
        # Create output directory
        os.makedirs(output_dir, exist_ok=True)
        
        # Check for existing progress
        summary_file = f"{output_dir}/batch_summary.json"
        if resume_from == 0 and os.path.exists(summary_file):
            try:
                with open(summary_file, 'r') as f:
                    existing_summary = json.load(f)
                    resume_from = existing_summary.get("last_processed_index", 0)
                    if resume_from > 0:
                        logger.info(f"Resuming from index {resume_from}")
            except Exception as e:
                logger.warning(f"Could not load existing summary: {e}")
                resume_from = 0
        
        results = []
        errors = []
        
        # Load existing results if resuming
        if resume_from > 0:
            existing_files = glob.glob(f"{output_dir}/lead_*.json")
            results = [{"output_file": f} for f in existing_files]
            logger.info(f"Found {len(results)} existing lead files")
        
        # Skip already processed inputs
        inputs_to_process = inputs[resume_from:]
        
        if HAS_RICH:
            with Progress() as progress:
                task = progress.add_task("Processing leads...", total=len(inputs_to_process))
                
                # Process in batches with incremental updates
                for i in range(0, len(inputs_to_process), self.config.batch_size):
                    batch = inputs_to_process[i:i + self.config.batch_size]
                    batch_results = await self._process_batch(batch, industry, output_dir)
                    
                    # Process results and update summary incrementally
                    for j, result in enumerate(batch_results):
                        if isinstance(result, Exception):
                            error_result = {
                                "error": str(result),
                                "input": batch[j],
                                "timestamp": datetime.now().isoformat(),
                                "index": resume_from + i + j
                            }
                            errors.append(error_result)
                        elif "error" in result:
                            result["index"] = resume_from + i + j
                            errors.append(result)
                        else:
                            result["index"] = resume_from + i + j
                            results.append(result)
                    
                    # Update summary after each batch
                    current_summary = self._generate_incremental_summary(
                        inputs, results, errors, output_dir, resume_from + i + len(batch)
                    )
                    
                    # Save incremental summary
                    with open(summary_file, 'w') as f:
                        json.dump(current_summary, f, indent=2, default=str)
                    
                    progress.update(task, advance=len(batch))
                    
                    if progress_callback:
                        progress_callback(len(results), len(errors), len(inputs))
        else:
            # Fallback without rich progress bar
            for i in range(0, len(inputs_to_process), self.config.batch_size):
                batch = inputs_to_process[i:i + self.config.batch_size]
                batch_results = await self._process_batch(batch, industry, output_dir)
                
                for j, result in enumerate(batch_results):
                    if isinstance(result, Exception):
                        error_result = {
                            "error": str(result),
                            "input": batch[j],
                            "timestamp": datetime.now().isoformat(),
                            "index": resume_from + i + j
                        }
                        errors.append(error_result)
                    elif "error" in result:
                        result["index"] = resume_from + i + j
                        errors.append(result)
                    else:
                        result["index"] = resume_from + i + j
                        results.append(result)
                
                # Update summary after each batch
                current_summary = self._generate_incremental_summary(
                    inputs, results, errors, output_dir, resume_from + i + len(batch)
                )
                
                with open(summary_file, 'w') as f:
                    json.dump(current_summary, f, indent=2, default=str)
        
        # Generate final summary with quality metrics
        final_summary = self._generate_final_summary(inputs, results, errors, output_dir)
        
        # Save final summary
        with open(summary_file, 'w') as f:
            json.dump(final_summary, f, indent=2, default=str)
        
        return final_summary
    
    def _generate_incremental_summary(self, inputs: List[Dict], results: List[Dict], 
                                    errors: List[Dict], output_dir: str, 
                                    last_processed_index: int) -> Dict[str, Any]:
        """Generate incremental summary during batch processing."""
        total_processed = len(results) + len(errors)
        success_rate = (len(results) / total_processed * 100) if total_processed > 0 else 0
        
        return {
            "timestamp": datetime.now().isoformat(),
            "total_inputs": len(inputs),
            "last_processed_index": last_processed_index,
            "total_processed": total_processed,
            "successful": len(results),
            "failed": len(errors),
            "success_rate": success_rate,
            "output_directory": output_dir,
            "processing_status": "in_progress",
            "estimated_completion": f"{total_processed}/{len(inputs)} ({(total_processed/len(inputs)*100):.1f}%)",
            "errors": errors[:5] if errors else []  # Keep only first 5 errors for size
        }
    
    def _generate_final_summary(self, inputs: List[Dict], results: List[Dict], 
                              errors: List[Dict], output_dir: str) -> Dict[str, Any]:
        """Generate comprehensive final summary with quality metrics."""
        # Count actual lead files
        lead_files = glob.glob(f"{output_dir}/lead_*.json")
        actual_leads_generated = len(lead_files)
        
        # Analyze lead quality by sampling files
        quality_metrics = self._analyze_lead_quality(lead_files[:10])  # Sample first 10
        
        # Calculate costs
        total_cost = 0
        total_tokens = 0
        for lead_file in lead_files[:20]:  # Sample 20 files for cost analysis
            try:
                with open(lead_file, 'r') as f:
                    lead_data = json.load(f)
                    token_usage = lead_data.get("metadata", {}).get("token_usage", {})
                    if "estimated_cost" in token_usage:
                        total_cost += float(token_usage["estimated_cost"])
                    if "total_tokens" in token_usage:
                        total_tokens += int(token_usage["total_tokens"])
            except Exception:
                continue
        
        # Extrapolate costs
        if total_cost > 0:
            avg_cost_per_lead = total_cost / min(20, len(lead_files))
            estimated_total_cost = avg_cost_per_lead * actual_leads_generated
        else:
            estimated_total_cost = 0
        
        success_rate = (len(results) / len(inputs) * 100) if len(inputs) > 0 else 0
        
        return {
            "timestamp": datetime.now().isoformat(),
            "total_processed": len(inputs),
            "successful": len(results),
            "failed": len(errors),
            "success_rate": success_rate,
            "output_directory": output_dir,
            "processing_status": "completed",
            "campaign_summary": {
                "total_leads_generated": actual_leads_generated,
                "target_met": actual_leads_generated >= 80,  # 80% of 100 target
                "estimated_total_cost": round(estimated_total_cost, 3),
                "average_cost_per_lead": round(estimated_total_cost / actual_leads_generated, 4) if actual_leads_generated > 0 else 0,
                "total_tokens_estimated": total_tokens * (actual_leads_generated / min(20, len(lead_files))) if total_tokens > 0 else 0
            },
            "quality_metrics": quality_metrics,
            "results": results[:10],  # First 10 successful results
            "errors": errors[:10]     # First 10 errors
        }
    
    def _analyze_lead_quality(self, sample_files: List[str]) -> Dict[str, Any]:
        """Analyze quality metrics from sample lead files."""
        if not sample_files:
            return {"error": "No lead files to analyze"}
        
        hot_leads = 0
        warm_leads = 0
        cold_leads = 0
        total_confidence = 0
        total_fit = 0
        decision_makers_count = 0
        companies_with_ceo = 0
        companies_with_cto = 0
        valid_samples = 0
        
        for file_path in sample_files:
            try:
                with open(file_path, 'r') as f:
                    lead_data = json.load(f)
                    lead_info = lead_data.get("lead_data", {})
                    
                    # Quality analysis
                    quality = lead_info.get("lead_quality", "").lower()
                    if "hot" in quality:
                        hot_leads += 1
                    elif "warm" in quality:
                        warm_leads += 1
                    else:
                        cold_leads += 1
                    
                    # Confidence and fit scores
                    total_confidence += lead_info.get("confidence_score", 0)
                    total_fit += lead_info.get("fit_score", 0)
                    
                    # Decision makers analysis
                    decision_makers = lead_info.get("decision_makers", [])
                    decision_makers_count += len(decision_makers)
                    
                    # Executive analysis
                    for dm in decision_makers:
                        title = dm.get("title", "").lower()
                        if "ceo" in title or "chief executive" in title:
                            companies_with_ceo += 1
                            break
                    
                    for dm in decision_makers:
                        title = dm.get("title", "").lower()
                        if "cto" in title or "chief technology" in title or "chief information" in title:
                            companies_with_cto += 1
                            break
                    
                    valid_samples += 1
                    
            except Exception as e:
                continue
        
        if valid_samples == 0:
            return {"error": "No valid lead files found"}
        
        return {
            "sample_size": valid_samples,
            "hot_leads": hot_leads,
            "warm_leads": warm_leads,
            "cold_leads": cold_leads,
            "average_confidence_score": round(total_confidence / valid_samples, 2),
            "average_fit_score": round(total_fit / valid_samples, 2),
            "total_decision_makers": decision_makers_count,
            "avg_decision_makers_per_company": round(decision_makers_count / valid_samples, 1),
            "companies_with_ceo_contact": companies_with_ceo,
            "companies_with_cto_contact": companies_with_cto,
            "executive_coverage": round((companies_with_ceo + companies_with_cto) / (valid_samples * 2) * 100, 1)
        }
    
    async def _process_batch(self, batch: List[Dict], industry: str, output_dir: str) -> List[Dict]:
        """Process a batch of leads concurrently."""
        tasks = []
        for item in batch:
            task = self._process_single_item(item, industry, output_dir)
            tasks.append(task)
        
        return await asyncio.gather(*tasks, return_exceptions=True)
    
    async def _process_single_item(self, item: Dict, industry: str, output_dir: str) -> Dict:
        """Process a single lead extraction item."""
        try:
            result = await self.extract_leads(
                content=item.get("content"),
                url=item.get("url"),
                file_path=item.get("file_path"),
                industry=industry
            )
            
            # Save individual result
            filename = f"lead_{hash(str(item))}.json"
            filepath = os.path.join(output_dir, filename)
            with open(filepath, 'w') as f:
                json.dump(result, f, indent=2, default=str)
            
            result["output_file"] = filepath
            return result
            
        except Exception as e:
            return {
                "error": str(e),
                "input": item,
                "timestamp": datetime.now().isoformat()
            }
    
    # SmartLead campaign creation method removed
    
    # Cost tracking report method removed - was broken and added complexity
    
    def start_dashboard(self):
        """Start the web dashboard (if implemented)."""
        # This would start a web server for monitoring
        # Implementation would depend on chosen web framework (FastAPI, Flask, etc.)
        logger.info("Dashboard feature not yet implemented")
        pass


# CLI Commands using Click
@click.group()
@click.option('--debug', is_flag=True, help='Enable debug mode')
@click.option('--log-level', default='INFO', help='Set log level')
@click.pass_context
def cli(ctx, debug, log_level):
    """AgentLead - Lead Extraction, Enrichment, and Campaign Management Platform
    
    Features:
    - Lead extraction from URLs, files, and content using Kimi K2
    - Multi-source research with AI-powered intelligence gathering
    - Personalized outreach generation using CrewAI agents
    - Cost tracking and usage analytics
    - Real-time progress tracking
    - Export capabilities and batch processing
    - Health monitoring and configuration validation
    
    Examples:
        # Traditional lead extraction
        python run.py extract --url https://company.com --enrich
        python run.py batch --config batch_config.json --enrich
        
        # Multi-agent research and outreach (NEW!)
        python run.py research --leads-file leads.csv --output results --progress
        python run.py outreach --leads-file leads.csv --output campaigns --concurrent 3
        python run.py full-pipeline --leads-file leads.csv --output complete --progress
        
        # Clay enrichment
        python run.py enrich leads --leads-file leads.json
        python run.py enrich status <enrichment-id>
        python run.py enrich report --days 7 --detailed
        
        # System monitoring
        python run.py health --full
        python run.py validate
    """
    ctx.ensure_object(dict)
    ctx.obj['debug'] = debug
    ctx.obj['log_level'] = log_level


@cli.command()
@click.option('--content', help='Direct content input')
@click.option('--url', help='URL to extract content from')
@click.option('--file', 'file_path', help='File path to read content from')
@click.option('--industry', default='technology', help='Industry type for specialized extraction')
@click.option('--output-format', default='json', type=click.Choice(['json', 'csv']), help='Output format')
@click.option('--output-file', help='Output file path')
@click.option('--enrich', is_flag=True, help='Automatically enrich extracted leads with Clay')
@click.option('--enrich-types', default='email,phone,linkedin,company', help='Clay enrichment types (if --enrich used)')
@click.option('--enrich-priority', default='normal', type=click.Choice(['low', 'normal', 'high']), help='Clay enrichment priority')
@click.pass_context
async def extract(ctx, content, url, file_path, industry, output_format, output_file, enrich, enrich_types, enrich_priority):
    """Extract leads from content, URL, or file."""
    app_config = AppConfig(
        debug=ctx.obj['debug'],
        log_level=ctx.obj['log_level'],
        clay_enabled=enrich
    )
    
    app = AgentLeadApp(app_config)
    
    try:
        if not await app.initialize():
            click.echo("❌ Failed to initialize application", err=True)
            return
        
        with console.status("[bold green]Extracting leads..."):
            result = await app.extract_leads(
                content=content,
                url=url,
                file_path=file_path,
                industry=industry,
                output_format=output_format,
                output_file=output_file
            )
        
        # Enrich extracted lead if requested
        if enrich and result.get("lead_data"):
            try:
                enrichment_types = [t.strip() for t in enrich_types.split(',')]
                
                click.echo("🔄 Enriching extracted lead with Clay...")
                
                with console.status("[bold green]Enriching lead..."):
                    enrichment_result = await app.enrich_lead_with_clay(
                        lead_data=result["lead_data"],
                        enrichment_types=enrichment_types,
                        priority=enrich_priority
                    )
                
                # Add enrichment info to result
                result["enrichment"] = enrichment_result
                
                click.echo(f"✅ Lead enrichment initiated:")
                click.echo(f"   Enrichment ID: {enrichment_result['enrichment_id']}")
                click.echo(f"   Status: {enrichment_result['status']}")
                if enrichment_result.get('cost_estimate'):
                    click.echo(f"   Estimated cost: ${enrichment_result['cost_estimate']:.2f}")
                
                # Save updated result if output file specified
                if output_file:
                    if output_format.lower() == "json":
                        with open(output_file, 'w', encoding='utf-8') as f:
                            json.dump(result, f, indent=2, default=str)
                    
            except Exception as e:
                click.echo(f"⚠️ Enrichment failed: {e}")
                click.echo("✅ Lead extraction completed without enrichment")
        
        # Display results
        if output_file:
            click.echo(f"✅ Results saved to: {output_file}")
        else:
            click.echo(json.dumps(result, indent=2, default=str))
        
    except Exception as e:
        click.echo(f"❌ Error: {e}", err=True)
    finally:
        await app.shutdown()


@cli.command()
@click.option('--config', required=True, help='Batch configuration file')
@click.option('--progress/--no-progress', default=True, help='Show progress bar (default: enabled)')
@click.option('--resume', is_flag=True, help='Resume from last processed index')
@click.option('--resume-from', default=0, help='Resume from specific index')
@click.option('--enrich', is_flag=True, help='Automatically enrich processed leads with Clay')
@click.option('--enrich-types', default='email,phone,linkedin,company', help='Clay enrichment types (if --enrich used)')
@click.option('--enrich-priority', default='normal', type=click.Choice(['low', 'normal', 'high']), help='Clay enrichment priority')
@click.pass_context
async def batch(ctx, config, progress, resume, resume_from, enrich, enrich_types, enrich_priority):
    """Process multiple leads in batch."""
    app_config = AppConfig(
        debug=ctx.obj['debug'],
        log_level=ctx.obj['log_level'],
        clay_enabled=enrich
    )
    
    app = AgentLeadApp(app_config)
    
    try:
        if not await app.initialize():
            click.echo("❌ Failed to initialize application", err=True)
            return
        
        def progress_callback(successful, failed, total):
            if progress:
                click.echo(f"Progress: {successful + failed}/{total} processed ({successful} successful, {failed} failed)")
        
        # Determine resume index
        start_index = resume_from if resume_from > 0 else (0 if not resume else 0)
        
        result = await app.batch_process_leads(
            config_file=config,
            progress_callback=progress_callback if progress else None,
            resume_from=start_index
        )
        
        click.echo(f"✅ Batch processing complete:")
        click.echo(f"   Total: {result['total_processed']}")
        click.echo(f"   Successful: {result['successful']}")
        click.echo(f"   Failed: {result['failed']}")
        click.echo(f"   Success Rate: {result['success_rate']:.1f}%")
        click.echo(f"   Results saved to: {result['output_directory']}")
        
        # Auto-enrich if requested and we have successful results
        if enrich and result['successful'] > 0:
            try:
                click.echo("\n🔄 Starting automatic enrichment of extracted leads...")
                
                # Create a file with the batch results for enrichment
                batch_leads_file = f"{result['output_directory']}/batch_leads_for_enrichment.json"
                
                # Load successful lead files and create enrichment input
                import glob
                lead_files = glob.glob(f"{result['output_directory']}/lead_*.json")
                enrichment_leads = []
                
                for lead_file in lead_files[:result['successful']]:  # Only successful ones
                    try:
                        with open(lead_file, 'r') as f:
                            lead_result = json.load(f)
                            if lead_result.get("lead_data"):
                                enrichment_leads.append(lead_result["lead_data"])
                    except Exception:
                        continue
                
                if enrichment_leads:
                    # Save leads for enrichment
                    with open(batch_leads_file, 'w') as f:
                        json.dump(enrichment_leads, f, indent=2, default=str)
                    
                    enrichment_types = [t.strip() for t in enrich_types.split(',')]
                    
                    # Perform batch enrichment
                    enrichment_result = await app.batch_enrich_leads_with_clay(
                        leads_file=batch_leads_file,
                        enrichment_types=enrichment_types,
                        priority=enrich_priority,
                        output_file=f"{result['output_directory']}/enrichment_results.json"
                    )
                    
                    click.echo(f"✅ Batch enrichment completed:")
                    click.echo(f"   Leads sent for enrichment: {enrichment_result['total_leads']}")
                    click.echo(f"   Successful submissions: {enrichment_result['successful_submissions']}")
                    click.echo(f"   Failed submissions: {enrichment_result['failed_submissions']}")
                    click.echo(f"   Estimated cost: ${enrichment_result['total_estimated_cost']:.2f}")
                    click.echo(f"   Enrichment results saved to: {result['output_directory']}/enrichment_results.json")
                    
                else:
                    click.echo("⚠️ No valid leads found for enrichment")
                    
            except Exception as e:
                click.echo(f"⚠️ Auto-enrichment failed: {e}")
                click.echo("✅ Batch processing completed without enrichment")
        
    except Exception as e:
        click.echo(f"❌ Error: {e}", err=True)
    finally:
        await app.shutdown()


# SmartLead campaign CLI commands removed


@cli.command()
@click.option('--full', is_flag=True, help='Full health check including all components')
@click.option('--json-output', is_flag=True, help='Output in JSON format')
@click.pass_context
async def health(ctx, full, json_output):
    """Check system health and status."""
    app_config = AppConfig(
        debug=ctx.obj['debug'],
        log_level=ctx.obj['log_level']
    )
    
    app = AgentLeadApp(app_config)
    
    try:
        # Always initialize for health check
        if not await app.initialize():
            click.echo("❌ Failed to initialize application", err=True)
            return
        
        health_status = await app.check_health()
        
        if json_output:
            click.echo(json.dumps(health_status, indent=2, default=str))
        else:
            # Pretty print health status
            status_color = "green" if health_status["overall_status"] == "healthy" else "red"
            
            table = Table(title="AgentLead Health Status")
            table.add_column("Component", style="cyan")
            table.add_column("Status", style=status_color)
            table.add_column("Last Check", style="dim")
            
            for component, status in health_status["components"].items():
                table.add_row(
                    component.replace("_", " ").title(),
                    status["status"],
                    status["last_check"]
                )
            
            console.print(table)
            
            # Overall status
            status_panel = Panel(
                f"Overall Status: {health_status['overall_status'].upper()}",
                style=status_color
            )
            console.print(status_panel)
        
    except Exception as e:
        click.echo(f"❌ Error: {e}", err=True)
    finally:
        if full:
            await app.shutdown()


# Cost tracking CLI command removed - was broken and added complexity


@cli.command()
@click.option('--port', default=8080, help='Dashboard port')
@click.pass_context
def dashboard(ctx, port):
    """Start the monitoring dashboard."""
    click.echo(f"🚀 Starting dashboard on port {port}...")
    click.echo("Dashboard feature is not yet implemented")
    # This would start the web dashboard


# Add the new Clay enrichment CLI group
@cli.group()
def enrich():
    """Clay enrichment commands for lead data enhancement."""
    pass


@enrich.command('leads')
@click.option('--leads-file', required=True, help='JSON file containing leads to enrich')
@click.option('--types', default='email,phone,linkedin,company', help='Comma-separated enrichment types')
@click.option('--priority', default='normal', type=click.Choice(['low', 'normal', 'high']), help='Processing priority')
@click.option('--batch-size', default=10, help='Batch size for processing')
@click.option('--output', help='Output file for enrichment results')
@click.pass_context
async def enrich_leads(ctx, leads_file, types, priority, batch_size, output):
    """Enrich leads from a file using Clay."""
    app_config = AppConfig(
        debug=ctx.obj['debug'],
        log_level=ctx.obj['log_level'],
        clay_enabled=True
    )
    
    app = AgentLeadApp(app_config)
    
    try:
        if not await app.initialize():
            click.echo("❌ Failed to initialize application", err=True)
            return
        
        enrichment_types = [t.strip() for t in types.split(',')]
        
        click.echo(f"🔄 Starting batch enrichment of leads from {leads_file}")
        click.echo(f"   Types: {', '.join(enrichment_types)}")
        click.echo(f"   Priority: {priority}")
        click.echo(f"   Batch size: {batch_size}")
        
        with console.status("[bold green]Enriching leads..."):
            result = await app.batch_enrich_leads_with_clay(
                leads_file=leads_file,
                enrichment_types=enrichment_types,
                priority=priority,
                batch_size=batch_size,
                output_file=output
            )
        
        # Display results
        click.echo(f"✅ Batch enrichment completed:")
        click.echo(f"   Total leads: {result['total_leads']}")
        click.echo(f"   Successful submissions: {result['successful_submissions']}")
        click.echo(f"   Failed submissions: {result['failed_submissions']}")
        click.echo(f"   Success rate: {result['success_rate']:.1f}%")
        click.echo(f"   Estimated total cost: ${result['total_estimated_cost']:.2f}")
        
        if output:
            click.echo(f"   Results saved to: {output}")
        
        # Show some enrichment IDs for tracking
        successful_results = [r for r in result['results'] if r.get('enrichment_id')]
        if successful_results:
            click.echo("\n📋 Enrichment IDs for tracking:")
            for i, r in enumerate(successful_results[:5]):  # Show first 5
                click.echo(f"   {i+1}. {r['enrichment_id']}")
            if len(successful_results) > 5:
                click.echo(f"   ... and {len(successful_results) - 5} more")
        
    except Exception as e:
        click.echo(f"❌ Error: {e}", err=True)
    finally:
        await app.shutdown()


@enrich.command('status')
@click.argument('enrichment_id')
@click.pass_context
async def enrichment_status(ctx, enrichment_id):
    """Check status of a Clay enrichment."""
    app_config = AppConfig(
        debug=ctx.obj['debug'],
        log_level=ctx.obj['log_level'],
        clay_enabled=True
    )
    
    app = AgentLeadApp(app_config)
    
    try:
        if not await app.initialize():
            click.echo("❌ Failed to initialize application", err=True)
            return
        
        click.echo(f"🔍 Checking status for enrichment: {enrichment_id}")
        
        status_data = await app.get_enrichment_status(enrichment_id)
        
        # Display status information
        click.echo(f"✅ Enrichment Status:")
        click.echo(f"   ID: {enrichment_id}")
        click.echo(f"   Status: {status_data.get('status', 'unknown')}")
        click.echo(f"   Progress: {status_data.get('progress', 0)}%")
        
        if status_data.get('estimated_completion'):
            click.echo(f"   Estimated completion: {status_data['estimated_completion']}")
        
        if status_data.get('credits_used'):
            click.echo(f"   Credits used: {status_data['credits_used']}")
        
        if status_data.get('cost_usd'):
            click.echo(f"   Cost: ${status_data['cost_usd']:.2f}")
        
        if status_data.get('error'):
            click.echo(f"   Error: {status_data['error']}")
        
    except Exception as e:
        click.echo(f"❌ Error: {e}", err=True)
    finally:
        await app.shutdown()


@enrich.command('cancel')
@click.argument('enrichment_id')
@click.confirmation_option(prompt='Are you sure you want to cancel this enrichment?')
@click.pass_context
async def cancel_enrichment(ctx, enrichment_id):
    """Cancel a Clay enrichment."""
    app_config = AppConfig(
        debug=ctx.obj['debug'],
        log_level=ctx.obj['log_level'],
        clay_enabled=True
    )
    
    app = AgentLeadApp(app_config)
    
    try:
        if not await app.initialize():
            click.echo("❌ Failed to initialize application", err=True)
            return
        
        click.echo(f"🛑 Cancelling enrichment: {enrichment_id}")
        
        success = await app.cancel_enrichment(enrichment_id)
        
        if success:
            click.echo(f"✅ Enrichment {enrichment_id} cancelled successfully")
        else:
            click.echo(f"❌ Failed to cancel enrichment {enrichment_id}")
        
    except Exception as e:
        click.echo(f"❌ Error: {e}", err=True)
    finally:
        await app.shutdown()


@enrich.command('report')
@click.option('--days', default=30, help='Number of days for report period')
@click.option('--detailed', is_flag=True, help='Include detailed breakdown')
@click.option('--export', help='Export to file (JSON)')
@click.pass_context
async def enrichment_report(ctx, days, detailed, export):
    """Generate Clay enrichment usage report."""
    app_config = AppConfig(
        debug=ctx.obj['debug'],
        log_level=ctx.obj['log_level'],
# Cost tracking removed,
        clay_enabled=True
    )
    
    app = AgentLeadApp(app_config)
    
    try:
        if not await app.initialize():
            click.echo("❌ Failed to initialize application", err=True)
            return
        
        click.echo(f"📊 Generating enrichment report for last {days} days...")
        
        report = await app.get_enrichment_report(days=days, include_details=detailed)
        
        # Display report summary
        click.echo(f"✅ Enrichment Report:")
        click.echo(f"   Period: {days} days")
        
        if 'total_cost' in report:
            click.echo(f"   Total cost: ${report['total_cost']:.2f}")
        
        if 'total_requests' in report:
            click.echo(f"   Total requests: {report['total_requests']}")
        
        if 'enrichment_metrics' in report:
            metrics = report['enrichment_metrics']
            click.echo(f"   Pending enrichments: {metrics.get('pending_enrichments', 0)}")
        
        # Show cost breakdown if available
        if detailed and 'cost_breakdown' in report:
            click.echo("\n💰 Cost Breakdown:")
            for provider, cost in report['cost_breakdown'].items():
                click.echo(f"   {provider}: ${cost:.2f}")
        
        if export:
            with open(export, 'w') as f:
                json.dump(report, f, indent=2, default=str)
            click.echo(f"   Report exported to: {export}")
        elif detailed:
            click.echo(f"\n📋 Full Report:")
            click.echo(json.dumps(report, indent=2, default=str))
        
    except Exception as e:
        click.echo(f"❌ Error: {e}", err=True)
    finally:
        await app.shutdown()


@enrich.command('export')
@click.option('--input', 'input_file', required=True, help='Batch summary JSON file to export')
@click.option('--output', help='Output CSV file for Clay (auto-generated if not provided)')
@click.option('--format', 'export_format', default='csv', type=click.Choice(['csv', 'json']), help='Export format')
@click.option('--include-metadata', is_flag=True, help='Include processing metadata in export')
@click.pass_context
async def export_for_clay(ctx, input_file, output, export_format, include_metadata):
    """Export leads to Clay-optimized CSV format."""
    try:
        from src.utils.clay_export import export_to_clay_csv, ClayExporter, ClayExportConfig
        from datetime import datetime
        import os
        
        # Validate input file
        if not os.path.exists(input_file):
            click.echo(f"❌ Input file not found: {input_file}", err=True)
            return
        
        # Generate output filename if not provided
        if not output:
            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
            output = f"clay_export_{timestamp}.{export_format}"
        
        click.echo(f"🔄 Exporting leads for Clay enrichment...")
        click.echo(f"   Input: {input_file}")
        click.echo(f"   Output: {output}")
        click.echo(f"   Format: {export_format.upper()}")
        
        # Configure export
        config = ClayExportConfig(
            include_decision_makers=True,
            max_decision_makers_per_row=3,
            include_business_context=True,
            include_talking_points=True,
            export_format=export_format
        )
        
        # Perform export
        if export_format == 'csv':
            result = export_to_clay_csv(
                input_file, 
                output, 
                include_metadata=include_metadata,
                config=config
            )
        else:
            exporter = ClayExporter(config)
            result = exporter.export_from_batch_summary(
                input_file, 
                output, 
                include_metadata=include_metadata
            )
        
        # Display results
        click.echo(f"✅ Export completed successfully!")
        click.echo(f"   📁 Output file: {output}")
        
        # Show appropriate stats based on export format
        stats = result['export_stats']
        if export_format == 'csv':
            click.echo(f"   🏢 Companies: {stats['total_companies']}")
            click.echo(f"   👥 Individual contacts: {stats['total_contacts']}")
            click.echo(f"   📊 Avg contacts/company: {stats['average_contacts_per_company']}")
        else:
            click.echo(f"   📊 Leads exported: {stats.get('total_leads', 0)}")
        
        click.echo(f"   📅 Export time: {result['export_timestamp']}")
        
        if result.get('failed_files'):
            click.echo(f"   ⚠️  Failed files: {len(result['failed_files'])}")
        
        click.echo(f"\n🎯 Next Steps:")
        click.echo(f"   1. Upload {output} to your Clay table")
        click.echo(f"   2. Configure enrichment workflow in Clay UI")
        click.echo(f"   3. Monitor progress in Clay dashboard")
        click.echo(f"   4. Export enriched results when complete")
        
        # Show sample of exported data for CSV
        if export_format == 'csv' and os.path.exists(output):
            click.echo(f"\n📋 Sample of exported data:")
            with open(output, 'r') as f:
                lines = f.readlines()
                for i, line in enumerate(lines[:3]):  # Header + 2 sample rows
                    click.echo(f"   {line.strip()}")
                if len(lines) > 3:
                    click.echo(f"   ... and {len(lines) - 3} more rows")
        
    except Exception as e:
        click.echo(f"❌ Export failed: {str(e)}", err=True)
        if ctx.obj.get('debug'):
            import traceback
            click.echo(traceback.format_exc(), err=True)


@enrich.command('import')
@click.option('--clay-file', required=True, help='CSV file exported from Clay with enriched data')
@click.option('--batch-file', required=True, help='Original AgentLead batch summary JSON file')
@click.option('--output', help='Output file for campaign-ready leads (auto-generated if not provided)')
@click.option('--format', 'output_format', default='json', type=click.Choice(['json', 'csv']), help='Output format for campaign leads')
@click.option('--validate/--no-validate', default=True, help='Validate enriched data quality before import')
@click.pass_context
async def import_clay_data(ctx, clay_file, batch_file, output, output_format, validate):
    """Import enriched data from Clay and merge with original AgentLead data."""
    try:
        from src.utils.clay_import import ClayImporter, ImportValidationConfig
        from datetime import datetime
        import os
        
        # Validate input files
        if not os.path.exists(clay_file):
            click.echo(f"❌ Clay export file not found: {clay_file}", err=True)
            return
        
        if not os.path.exists(batch_file):
            click.echo(f"❌ Batch summary file not found: {batch_file}", err=True)
            return
        
        # Generate output filename if not provided
        if not output:
            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
            output = f"campaign_leads_{timestamp}.{output_format}"
        
        click.echo(f"🔄 Importing enriched data from Clay...")
        click.echo(f"   Clay export: {clay_file}")
        click.echo(f"   Original batch: {batch_file}")
        click.echo(f"   Output: {output}")
        click.echo(f"   Validation: {'✅ Enabled' if validate else '❌ Disabled'}")
        
        # Configure validation
        validation_config = ImportValidationConfig(
            require_email=True,
            require_phone=False,
            require_linkedin=False,
            min_confidence_score=0.7,
            validate_email_format=True,
            validate_phone_format=True,
            validate_linkedin_format=True
        ) if validate else None
        
        # Perform import
        importer = ClayImporter(validation_config)
        result = importer.import_enriched_data(
            clay_export_file=clay_file,
            original_batch_file=batch_file,
            output_file=output,
            validate_quality=validate
        )
        
        # Display results
        click.echo(f"✅ Import completed successfully!")
        click.echo(f"   📁 Campaign leads: {output}")
        
        # Show import statistics
        stats = result['statistics']
        click.echo(f"   📊 Import Statistics:")
        click.echo(f"      Total contacts processed: {stats.total_contacts}")
        click.echo(f"      Successfully enriched: {stats.enriched_contacts}")
        click.echo(f"      Enrichment rate: {stats.enrichment_rate:.1f}%")
        click.echo(f"      Emails found: {stats.emails_found}")
        click.echo(f"      Phones found: {stats.phones_found}")
        click.echo(f"      LinkedIn profiles: {stats.linkedin_found}")
        click.echo(f"      High-quality contacts: {stats.high_quality_contacts}")
        
        if validate and stats.failed_validations > 0:
            click.echo(f"      Failed validations: {stats.failed_validations}")
        
        click.echo(f"   📅 Import time: {result['import_timestamp']}")
        
        # Show next steps
        click.echo(f"\n🎯 Next Steps:")
        click.echo(f"   1. Review campaign leads in {output}")
        click.echo(f"   2. Upload to SmartLead for campaign creation")
        click.echo(f"   3. Configure email sequences and sending schedules")
        click.echo(f"   4. Launch your outreach campaign")
        
        # Show sample of campaign data
        if output_format == 'csv' and os.path.exists(output):
            click.echo(f"\n📋 Sample campaign data:")
            with open(output, 'r') as f:
                lines = f.readlines()
                for i, line in enumerate(lines[:3]):  # Header + 2 sample rows
                    click.echo(f"   {line.strip()}")
                if len(lines) > 3:
                    click.echo(f"   ... and {len(lines) - 3} more leads")
        
    except Exception as e:
        click.echo(f"❌ Import failed: {str(e)}", err=True)
        if ctx.obj.get('debug'):
            import traceback
            click.echo(traceback.format_exc(), err=True)


@enrich.command('config')
@click.option('--show', is_flag=True, help='Show current Clay configuration')
@click.option('--webhook-url', help='Set webhook URL')
@click.option('--types', help='Set default enrichment types (comma-separated)')
@click.option('--test', is_flag=True, help='Test Clay connection')
@click.pass_context
async def clay_config(ctx, show, webhook_url, types, test):
    """Manage Clay configuration and settings."""
    app_config = AppConfig(
        debug=ctx.obj['debug'],
        log_level=ctx.obj['log_level'],
        clay_enabled=True
    )
    
    if show:
        click.echo("🔧 Clay Configuration:")
        click.echo(f"   API Key: {'✅ Set' if os.getenv('CLAY_API_KEY') else '❌ Not set'}")
        click.echo(f"   Webhook Secret: {'✅ Set' if os.getenv('CLAY_WEBHOOK_SECRET') else '❌ Not set'}")
        click.echo(f"   Default Types: {', '.join(app_config.clay_default_enrichment_types)}")
        click.echo(f"   Webhook Host: {app_config.clay_webhook_host}")
        click.echo(f"   Webhook Port: {app_config.clay_webhook_port}")
        click.echo(f"   Webhook Path: {app_config.clay_webhook_path}")
        
        if webhook_url:
            click.echo(f"   Custom Webhook URL: {webhook_url}")
        else:
            default_webhook = f"http://{app_config.clay_webhook_host}:{app_config.clay_webhook_port}{app_config.clay_webhook_path}"
            click.echo(f"   Default Webhook URL: {default_webhook}")
        return
    
    if types:
        app_config.clay_default_enrichment_types = [t.strip() for t in types.split(',')]
        click.echo(f"✅ Default enrichment types updated: {', '.join(app_config.clay_default_enrichment_types)}")
    
    if test:
        app = AgentLeadApp(app_config)
        try:
            if not await app.initialize():
                click.echo("❌ Failed to initialize application", err=True)
                return
            
            click.echo("🔍 Testing Clay connection...")
            
            if app.clay_client:
                # Test connection by checking session
                if app.clay_client.session and not app.clay_client.session.closed:
                    click.echo("✅ Clay connection successful")
                else:
                    click.echo("❌ Clay connection failed")
            else:
                click.echo("❌ Clay client not initialized")
                
        except Exception as e:
            click.echo(f"❌ Connection test failed: {e}")
        finally:
            await app.shutdown()


# New multi-agent research and outreach commands
@cli.command()
@click.option('--leads-file', required=True, help='CSV file containing leads')
@click.option('--output', required=True, help='Output file prefix for results')
@click.option('--progress/--no-progress', default=True, help='Show real-time progress tracking (default: enabled)')
@click.option('--limit', type=int, help='Limit number of leads to process')
@click.option('--delay', type=float, default=2.0, help='Delay between API calls')
@click.option('--export-format', multiple=True, default=['csv', 'json'], 
              type=click.Choice(['csv', 'json', 'excel', 'html']), 
              help='Export formats (can use multiple)')
@click.option('--analytics', is_flag=True, default=True, help='Include analytics report')
@click.option('--charts', is_flag=True, help='Generate visualization charts')
@click.pass_context
async def research(ctx, leads_file, output, progress, limit, delay, export_format, analytics, charts):
    """Research companies using multi-source intelligence gathering."""
    if not HAS_MULTI_AGENT:
        click.echo("❌ Multi-agent components not available", err=True)
        return
    
    click.echo("🔍 AgentLead Multi-Source Research")
    click.echo("=" * 50)
    
    # Set up progress tracking if requested
    if progress:
        set_progress_callback(print_progress_update)
        click.echo("📊 Progress tracking enabled\n")
    
    try:
        # Load leads from CSV
        leads = load_leads_for_multi_agent(leads_file, limit)
        
        if not leads:
            click.echo("❌ No valid leads found in CSV file")
            return
        
        click.echo(f"📋 Loaded {len(leads)} leads")
        
        # Calculate accurate cost estimate using existing system
        try:
            from src.agents.model_config import estimate_api_cost, get_extraction_model
            from src.database.repositories import get_repositories
            
            # Get model config for research
            model_config = get_extraction_model("medium")
            
            # Estimate OpenAI costs per lead (research task)
            # Using 780 tokens (95th percentile from actual pipeline analysis)
            research_prompt = f"Research company intelligence for business outreach targeting."
            openai_cost_per_lead = estimate_api_cost(research_prompt, 780, model_config.model)
            
            # Add external API costs (estimated rates)
            tavily_cost_per_lead = 0.015  # ~3 searches at $0.005 each
            firecrawl_cost_per_lead = 0.002  # ~$0.002 per page scrape
            
            total_cost_per_lead = openai_cost_per_lead + tavily_cost_per_lead + firecrawl_cost_per_lead
            total_estimated_cost = len(leads) * total_cost_per_lead
            
            click.echo(f"💰 Estimated cost: ~${total_estimated_cost:.3f} (${total_cost_per_lead:.3f} per lead)")
            click.echo(f"   OpenAI ({model_config.model}): ${openai_cost_per_lead:.3f} per lead")
            click.echo(f"   External APIs: ${(tavily_cost_per_lead + firecrawl_cost_per_lead):.3f} per lead")
            
            if len(leads) > 10:
                if not click.confirm("⚠️  Continue with research?"):
                    click.echo("Research cancelled.")
                    return
                    
        except ImportError as e:
            click.echo(f"⚠️  Cost estimation unavailable: {e}")
            if len(leads) > 10:
                if not click.confirm("⚠️  Continue with research? (cost estimation failed)"):
                    click.echo("Research cancelled.")
                    return
        
        # Initialize researcher
        researcher = CompanyResearcher()
        
        # Process leads
        results = []
        for i, lead in enumerate(leads, 1):
            click.echo(f"\n{'='*50}")
            click.echo(f"Processing {i}/{len(leads)}: {lead.company_name}")
            
            try:
                intelligence = await researcher.research_company(lead)
                results.append({
                    'lead': lead,
                    'intelligence': intelligence,
                    'success': True
                })
                
                if not progress:  # Show summary if not using progress tracking
                    click.echo(f"✅ Research complete - Confidence: {intelligence.research_confidence:.2f}")
                    
            except Exception as e:
                results.append({
                    'lead': lead,
                    'intelligence': None,
                    'success': False,
                    'error': str(e)
                })
                click.echo(f"❌ Research failed: {str(e)}")
            
            # Delay between requests
            if i < len(leads):
                await asyncio.sleep(delay)
        
        # Export results with enhanced capabilities
        export_research_results_advanced(results, output, list(export_format), analytics, charts)
        
        # Show summary
        successful = sum(1 for r in results if r['success'])
        failed = len(results) - successful
        
        click.echo(f"\n{'='*50}")
        click.echo(f"🎉 Research completed!")
        click.echo(f"📊 Results: {successful} successful, {failed} failed")
            
    except Exception as e:
        click.echo(f"❌ Error: {e}", err=True)


@cli.command()
@click.option('--leads-file', required=True, help='CSV file containing leads')
@click.option('--output', required=True, help='Output file prefix')
@click.option('--limit', type=int, help='Limit number of leads to process')
@click.option('--concurrent', type=int, default=2, help='Max concurrent agents')
@click.option('--format', 'output_format', type=click.Choice(['json', 'csv', 'both']), default='both')
@click.option('--progress/--no-progress', default=True, help='Show real-time progress (default: enabled)')
@click.option('--delay', type=float, default=2.0, help='Delay between batches')
@click.pass_context
async def outreach(ctx, leads_file, output, limit, concurrent, output_format, progress, delay):
    """Generate personalized outreach messages using AI agents."""
    if not HAS_MULTI_AGENT:
        click.echo("❌ Multi-agent components not available", err=True)
        return
    
    click.echo("✉️  AgentLead Personalized Outreach")
    click.echo("=" * 50)
    
    # Set up progress tracking if requested
    if progress:
        set_progress_callback(print_progress_update)
        click.echo("📊 Progress tracking enabled\n")
    
    try:
        # Load leads
        leads = load_leads_for_multi_agent(leads_file, limit)
        
        if not leads:
            click.echo("❌ No valid leads found in CSV file")
            return
        
        click.echo(f"📋 Loaded {len(leads)} leads")
        
        # Calculate accurate cost estimate using existing system
        try:
            from src.agents.model_config import estimate_api_cost, get_extraction_model
            
            # Get model configs for different tasks (cost-optimized)
            research_model = get_extraction_model("medium")    # gpt-4o-mini
            personalization_model = get_extraction_model("medium")  # gpt-4o-mini (not complex!)
            
            # Estimate OpenAI costs per lead
            # Using 780 tokens total (95th percentile from actual pipeline analysis)
            # Split: ~350 research + ~430 personalization
            research_prompt = "Research company intelligence for business outreach targeting."
            research_cost = estimate_api_cost(research_prompt, 350, research_model.model)
            
            personalization_prompt = "Generate personalized outreach email based on company research."
            personalization_cost = estimate_api_cost(personalization_prompt, 430, personalization_model.model)
            
            openai_cost_per_lead = research_cost + personalization_cost
            
            # Add external API costs
            tavily_cost_per_lead = 0.015  # ~3 searches at $0.005 each
            firecrawl_cost_per_lead = 0.002  # ~$0.002 per page scrape
            
            total_cost_per_lead = openai_cost_per_lead + tavily_cost_per_lead + firecrawl_cost_per_lead
            total_estimated_cost = len(leads) * total_cost_per_lead
            
            click.echo(f"💰 Estimated cost: ~${total_estimated_cost:.3f} (${total_cost_per_lead:.3f} per lead)")
            click.echo(f"   OpenAI: ${openai_cost_per_lead:.3f} per lead ({research_model.model}/{personalization_model.model})")
            click.echo(f"   External APIs: ${(tavily_cost_per_lead + firecrawl_cost_per_lead):.3f} per lead")
            
            if len(leads) > 10:
                try:
                    if not click.confirm("⚠️  Continue with outreach generation?"):
                        click.echo("Outreach generation cancelled.")
                        return
                except (EOFError, KeyboardInterrupt, click.Abort):
                    click.echo("⚠️  No input provided, cancelling outreach generation.")
                    return
                    
        except ImportError as e:
            click.echo(f"⚠️  Cost estimation unavailable: {e}")
            if len(leads) > 10:
                try:
                    if not click.confirm("⚠️  Continue with outreach generation? (cost estimation failed)"):
                        click.echo("Outreach generation cancelled.")
                        return
                except (EOFError, KeyboardInterrupt, click.Abort):
                    click.echo("⚠️  No input provided, cancelling outreach generation.")
                    return
        
        click.echo(f"🚀 Processing with {concurrent} concurrent agents...")
        
        # Initialize crew
        crew = OutreachCrew()
        
        # Set up streaming CSV writer if CSV output requested
        streaming_csv_writer = None
        csv_file = None
        if output_format in ['csv', 'both']:
            from src.core.streaming_csv_writer import create_streaming_csv_writer
            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
            csv_file = f"{output}_outreach_{timestamp}.csv"
            streaming_csv_writer = create_streaming_csv_writer(output, timestamp)
            click.echo(f"📊 Streaming to: {csv_file}")
        
        # Process leads with streaming callback
        results = await crew.process_leads_batch(
            leads=leads,
            max_concurrent=concurrent,
            delay_between_batches=delay,
            result_callback=streaming_csv_writer.write_result if streaming_csv_writer else None
        )
        
        # Close streaming CSV writer
        if streaming_csv_writer:
            streaming_csv_writer.close()
        
        # Export JSON results (CSV is already streamed)
        export_timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
        
        if output_format in ['json', 'both']:
            json_file = f"{output}_outreach_{export_timestamp}.json"
            crew.export_results(results, json_file, format='json')
            click.echo(f"📄 JSON export: {json_file}")
        
        # CSV was already streamed, just show completion message
        if output_format in ['csv', 'both']:
            click.echo(f"📊 CSV streaming completed: {csv_file}")
        elif output_format == 'csv':
            # Handle CSV-only case where we need to show the file
            click.echo(f"📊 CSV streaming completed")
        
        # Show summary
        successful = sum(1 for r in results if r.success)
        failed = len(results) - successful
        
        avg_score = sum(
            r.personalized_outreach.personalization_score 
            for r in results 
            if r.success and r.personalized_outreach
        ) / max(successful, 1)
        
        click.echo(f"\n{'='*50}")
        click.echo(f"🎉 Outreach generation completed!")
        click.echo(f"📊 Results: {successful} successful, {failed} failed")
        click.echo(f"⭐ Average personalization score: {avg_score:.2f}")
        
    except Exception as e:
        click.echo(f"❌ Error: {e}", err=True)


@cli.command('full-pipeline')
@click.option('--leads-file', required=True, help='CSV file containing leads')
@click.option('--output', required=True, help='Output file prefix')
@click.option('--limit', type=int, help='Limit number of leads to process')
@click.option('--concurrent', type=int, default=2, help='Max concurrent agents')
@click.option('--format', 'output_format', type=click.Choice(['json', 'csv', 'both']), default='both')
@click.option('--progress/--no-progress', default=True, help='Show real-time progress (default: enabled)')
@click.option('--delay', type=float, default=2.0, help='Delay between batches')
@click.pass_context
async def full_pipeline(ctx, leads_file, output, limit, concurrent, output_format, progress, delay):
    """Run complete research + personalized outreach pipeline."""
    if not HAS_MULTI_AGENT:
        click.echo("❌ Multi-agent components not available", err=True)
        return
    
    click.echo("🚀 AgentLead Full Pipeline (Research + Outreach)")
    click.echo("=" * 50)
    
    # Set up progress tracking if requested
    if progress:
        set_progress_callback(print_progress_update)
        click.echo("📊 Progress tracking enabled\n")
    
    try:
        # Load leads
        leads = load_leads_for_multi_agent(leads_file, limit)
        
        if not leads:
            click.echo("❌ No valid leads found in CSV file")
            return
        
        click.echo(f"📋 Loaded {len(leads)} leads")
        
        # Calculate accurate cost estimate using existing system
        try:
            from src.agents.model_config import estimate_api_cost, get_extraction_model
            
            # Get model configs for full pipeline (cost-optimized)
            research_model = get_extraction_model("medium")     # gpt-4o-mini
            personalization_model = get_extraction_model("medium")  # gpt-4o-mini
            synthesis_model = get_extraction_model("medium")    # gpt-4o-mini
            
            # Estimate OpenAI costs per lead (full pipeline)
            # Using 780 tokens total (95th percentile from actual pipeline analysis)
            # Split: ~280 research + ~200 synthesis + ~300 personalization
            research_cost = estimate_api_cost("Research company intelligence", 280, research_model.model)
            synthesis_cost = estimate_api_cost("Synthesize research data", 200, synthesis_model.model) 
            personalization_cost = estimate_api_cost("Generate personalized outreach", 300, personalization_model.model)
            
            openai_cost_per_lead = research_cost + synthesis_cost + personalization_cost
            
            # Add external API costs (higher for full pipeline)
            tavily_cost_per_lead = 0.020  # More searches for comprehensive research
            firecrawl_cost_per_lead = 0.003  # More scraping
            grok_cost_per_lead = 0.005  # Grok deep search
            
            external_cost_per_lead = tavily_cost_per_lead + firecrawl_cost_per_lead + grok_cost_per_lead
            total_cost_per_lead = openai_cost_per_lead + external_cost_per_lead
            total_estimated_cost = len(leads) * total_cost_per_lead
            
            click.echo(f"💰 Estimated cost: ~${total_estimated_cost:.3f} (${total_cost_per_lead:.3f} per lead)")
            click.echo(f"   OpenAI: ${openai_cost_per_lead:.3f} per lead")
            click.echo(f"   External APIs: ${external_cost_per_lead:.3f} per lead")
            
            if len(leads) > 10:
                if not click.confirm("⚠️  Continue with full pipeline?"):
                    click.echo("Full pipeline cancelled.")
                    return
                    
        except ImportError as e:
            click.echo(f"⚠️  Cost estimation unavailable: {e}")
            if len(leads) > 10:
                if not click.confirm("⚠️  Continue with full pipeline? (cost estimation failed)"):
                    click.echo("Full pipeline cancelled.")
                    return
        
        click.echo(f"🔄 Running full enrichment + personalization pipeline...")
        
        # Initialize crew with enhanced research
        crew = OutreachCrew()
        
        # Process leads with full pipeline
        results = await crew.process_leads_batch(
            leads=leads,
            max_concurrent=concurrent,
            delay_between_batches=delay
        )
        
        # Export comprehensive results
        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
        
        # Export full pipeline results
        full_results = []
        for result in results:
            if result.success:
                full_results.append({
                    'company_name': result.lead_profile.company_name,
                    'contact_name': result.lead_profile.contact_name,
                    'contact_title': result.lead_profile.contact_title,
                    'company_domain': result.lead_profile.company_domain,
                    'research_confidence': result.company_intelligence.research_confidence,
                    'sources_used': result.company_intelligence.sources_used,
                    'growth_signals': result.company_intelligence.growth_signals,
                    'challenges': result.company_intelligence.operational_challenges,
                    'personalization_score': result.personalized_outreach.personalization_score,
                    'subject_line': result.personalized_outreach.subject_line,
                    'opening_line': result.personalized_outreach.opening_line,
                    'pain_point_identified': result.personalized_outreach.pain_point_identified,
                    'value_connection': result.personalized_outreach.value_connection,
                    'curiosity_hook': result.personalized_outreach.curiosity_hook,
                    'email_body': result.personalized_outreach.email_body
                })
        
        # Export to CSV
        if output_format in ['csv', 'both']:
            csv_file = f"{output}_full_pipeline_{timestamp}.csv"
            import csv as csv_module
            with open(csv_file, 'w', newline='', encoding='utf-8') as f:
                if full_results:
                    writer = csv_module.DictWriter(f, fieldnames=full_results[0].keys())
                    writer.writeheader()
                    writer.writerows(full_results)
            click.echo(f"📊 CSV export: {csv_file}")
        
        # Export to JSON
        if output_format in ['json', 'both']:
            json_file = f"{output}_full_pipeline_{timestamp}.json"
            with open(json_file, 'w', encoding='utf-8') as f:
                json.dump({
                    'metadata': {
                        'timestamp': timestamp,
                        'total_leads': len(leads),
                        'successful': sum(1 for r in results if r.success),
                        'failed': sum(1 for r in results if not r.success)
                    },
                    'results': full_results
                }, f, indent=2)
            click.echo(f"📄 JSON export: {json_file}")
        
        # Show summary
        successful = sum(1 for r in results if r.success)
        failed = len(results) - successful
        
        click.echo(f"\n{'='*50}")
        click.echo(f"🎉 Full pipeline completed!")
        click.echo(f"📊 Results: {successful} successful, {failed} failed")
            
    except Exception as e:
        click.echo(f"❌ Error: {e}", err=True)


# Helper functions for multi-agent commands
def load_leads_for_multi_agent(csv_file: str, limit: int = None) -> List[LeadProfile]:
    """Load leads from CSV and convert to LeadProfile objects."""
    leads = []
    
    try:
        with open(csv_file, 'r', encoding='utf-8') as f:
            reader = csv.DictReader(f)
            
            for row in reader:
                # Skip leads without basic info
                if not row.get('companyName') or not row.get('firstName'):
                    continue
                
                # Clean domain
                domain = row.get('companyDomain', '').replace('https://', '').replace('http://', '').replace('www.', '')
                
                lead = LeadProfile(
                    company_name=row.get('companyName', ''),
                    company_domain=domain,
                    company_industry=row.get('companyIndustry', ''),
                    company_location=row.get('companyLocation', ''),
                    contact_name=f"{row.get('firstName', '')} {row.get('lastName', '')}".strip(),
                    contact_title=row.get('title', ''),
                    contact_email=row.get('email', ''),
                    contact_linkedin=row.get('linkedinUrl', ''),
                    raw_data=row
                )
                
                leads.append(lead)
                
                # Check limit after adding valid lead
                if limit and len(leads) >= limit:
                    break
                
        return leads
        
    except Exception as e:
        logger.error("Failed to load leads from CSV", error=str(e), file=csv_file)
        raise


def export_research_results(results: List[Dict[str, Any]], output_prefix: str):
    """Export research results to JSON format."""
    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
    json_file = f"{output_prefix}_research_{timestamp}.json"
    
    export_data = {
        'metadata': {
            'timestamp': timestamp,
            'total_leads': len(results),
            'successful': sum(1 for r in results if r['success']),
            'failed': sum(1 for r in results if not r['success'])
        },
        'results': []
    }
    
    for result in results:
        if result['success'] and result['intelligence']:
            export_data['results'].append({
                'company_name': result['lead'].company_name,
                'contact_name': result['lead'].contact_name,
                'research_confidence': result['intelligence'].research_confidence,
                'sources_used': result['intelligence'].sources_used,
                'employee_count': result['intelligence'].employee_count,
                'growth_signals': result['intelligence'].growth_signals,
                'operational_challenges': result['intelligence'].operational_challenges,
                'tech_stack': result['intelligence'].tech_stack,
                'recent_news': result['intelligence'].recent_news
            })
    
    with open(json_file, 'w', encoding='utf-8') as f:
        json.dump(export_data, f, indent=2)
    
    click.echo(f"📄 Research results exported: {json_file}")


def export_research_results_advanced(
    results: List[Dict[str, Any]], 
    output_prefix: str, 
    formats: List[str], 
    include_analytics: bool, 
    include_charts: bool
):
    """Export research results using advanced exporter with multiple formats."""
    if not HAS_MULTI_AGENT:
        # Fallback to basic export
        export_research_results(results, output_prefix)
        return
    
    try:
        # Convert results to format expected by ResearchExporter
        formatted_results = []
        for result in results:
            if result['success'] and result['intelligence']:
                intel = result['intelligence']
                lead = result['lead']
                
                formatted_results.append({
                    'company_name': lead.company_name,
                    'contact_name': lead.contact_name,
                    'contact_title': lead.contact_title,
                    'company_domain': lead.company_domain,
                    'company_industry': lead.company_industry,
                    'company_location': lead.company_location,
                    'research_confidence': intel.research_confidence,
                    'sources_used': intel.sources_used,
                    'employee_count': intel.employee_count,
                    'growth_signals': intel.growth_signals,
                    'operational_challenges': intel.operational_challenges,
                    'tech_stack': intel.tech_stack,
                    'recent_news': intel.recent_news,
                    'research_time': getattr(intel, 'research_time', 0),
                    'success': True
                })
            else:
                lead = result['lead']
                formatted_results.append({
                    'company_name': lead.company_name,
                    'contact_name': lead.contact_name,
                    'success': False,
                    'error': result.get('error', 'Unknown error')
                })
        
        # Use advanced exporter
        exporter = ResearchExporter()
        exported_files = exporter.export_research_results(
            results=formatted_results,
            filename_prefix=output_prefix,
            formats=formats,
            include_analytics=include_analytics,
            include_charts=include_charts
        )
        
        # Show export summary
        click.echo(f"\n📊 Advanced Export Complete!")
        click.echo(f"   📁 Files generated: {len(exported_files)}")
        
        for format_type, file_path in exported_files.items():
            file_size = "Unknown"
            if os.path.exists(file_path):
                size_bytes = os.path.getsize(file_path)
                if size_bytes < 1024:
                    file_size = f"{size_bytes} B"
                elif size_bytes < 1024 * 1024:
                    file_size = f"{size_bytes / 1024:.1f} KB"
                else:
                    file_size = f"{size_bytes / (1024 * 1024):.1f} MB"
            
            click.echo(f"   📄 {format_type.upper()}: {file_path} ({file_size})")
        
        if include_analytics:
            click.echo(f"   📈 Analytics report included")
        if include_charts and 'charts' in exported_files:
            click.echo(f"   📊 Visualization charts generated")
            
    except Exception as e:
        click.echo(f"⚠️  Advanced export failed, using basic export: {str(e)}")
        export_research_results(results, output_prefix)


@cli.command()
@click.option('--input-file', required=True, help='JSON file containing research results')
@click.option('--output', required=True, help='Output file prefix for exports')
@click.option('--format', 'export_formats', multiple=True, 
              default=['csv', 'json'], 
              type=click.Choice(['csv', 'json', 'excel', 'html']),
              help='Export formats (can specify multiple)')
@click.option('--analytics', is_flag=True, default=True, help='Include analytics report')
@click.option('--charts', is_flag=True, help='Generate visualization charts')
@click.option('--output-dir', default='exports', help='Output directory for exports')
@click.pass_context
async def export_research(ctx, input_file, output, export_formats, analytics, charts, output_dir):
    """Export existing research results in multiple formats with analytics."""
    if not HAS_MULTI_AGENT:
        click.echo("❌ Multi-agent components not available", err=True)
        return
    
    click.echo("📊 AgentLead Research Export")
    click.echo("=" * 50)
    
    try:
        # Load research results
        if not os.path.exists(input_file):
            click.echo(f"❌ Input file not found: {input_file}")
            return
        
        with open(input_file, 'r', encoding='utf-8') as f:
            data = json.load(f)
        
        # Extract results from different possible formats
        if 'results' in data:
            results = data['results']
        elif isinstance(data, list):
            results = data
        else:
            click.echo("❌ Invalid input file format")
            return
        
        if not results:
            click.echo("❌ No results found in input file")
            return
        
        click.echo(f"📋 Loaded {len(results)} research results")
        click.echo(f"📁 Output directory: {output_dir}")
        click.echo(f"📄 Export formats: {', '.join(export_formats)}")
        
        # Create exporter with custom output directory
        exporter = ResearchExporter(output_dir=output_dir)
        
        # Export with advanced features
        exported_files = exporter.export_research_results(
            results=results,
            filename_prefix=output,
            formats=list(export_formats),
            include_analytics=analytics,
            include_charts=charts
        )
        
        # Show summary
        click.echo(f"\n📊 Export Complete!")
        click.echo(f"   📁 Files generated: {len(exported_files)}")
        
        for format_type, file_path in exported_files.items():
            file_size = "Unknown"
            if os.path.exists(file_path):
                size_bytes = os.path.getsize(file_path)
                if size_bytes < 1024:
                    file_size = f"{size_bytes} B"
                elif size_bytes < 1024 * 1024:
                    file_size = f"{size_bytes / 1024:.1f} KB"
                else:
                    file_size = f"{size_bytes / (1024 * 1024):.1f} MB"
            
            click.echo(f"   📄 {format_type.upper()}: {file_path} ({file_size})")
        
        if analytics:
            click.echo(f"   📈 Analytics report included")
        if charts and 'charts' in exported_files:
            click.echo(f"   📊 Visualization charts generated")
        
        # Show quick stats if analytics available
        if analytics and 'analytics' in exported_files:
            analytics_file = exported_files['analytics']
            if os.path.exists(analytics_file):
                with open(analytics_file, 'r') as f:
                    analytics_data = json.load(f)
                    analytics_info = analytics_data.get('analytics', {})
                    
                    click.echo(f"\n📈 Quick Analytics:")
                    click.echo(f"   🏢 Total companies: {analytics_info.get('total_companies', 0)}")
                    click.echo(f"   ✅ Success rate: {analytics_info.get('success_rate', 0):.1f}%")
                    click.echo(f"   📊 Avg confidence: {analytics_info.get('avg_research_confidence', 0):.2f}")
                    click.echo(f"   🔍 Avg sources/company: {analytics_info.get('avg_sources_per_company', 0):.1f}")
                    
                    cost_info = analytics_info.get('cost_analysis', {})
                    if cost_info.get('total_estimated_cost', 0) > 0:
                        click.echo(f"   💰 Total cost: ${cost_info['total_estimated_cost']:.4f}")
        
    except Exception as e:
        click.echo(f"❌ Export failed: {str(e)}", err=True)
        if ctx.obj.get('debug'):
            import traceback
            click.echo(traceback.format_exc(), err=True)


@cli.command()
@click.pass_context
async def validate(ctx):
    """Validate configuration and system setup."""
    click.echo("🔍 Validating AgentLead configuration...")
    
    try:
        # Check environment variables
        required_vars = ["OPENROUTER_API_KEY"]
        optional_vars = ["CRAWL4AI_API_KEY", "TAVILY_API_KEY"]
        
        missing_required = []
        missing_optional = []
        
        for var in required_vars:
            if not os.getenv(var):
                missing_required.append(var)
        
        for var in optional_vars:
            if not os.getenv(var):
                missing_optional.append(var)
        
        if missing_required:
            click.echo("❌ Missing required environment variables:")
            for var in missing_required:
                click.echo(f"   - {var}")
            return
        
        if missing_optional:
            click.echo("⚠️  Missing optional environment variables:")
            for var in missing_optional:
                click.echo(f"   - {var}")
        
        # Validate Kimi config
        config = get_config()
        validate_config(config)
        click.echo("✅ Kimi K2 configuration valid")
        
        # Test component initialization
        app_config = AppConfig(
            debug=ctx.obj['debug'],
            log_level=ctx.obj['log_level']
        )
        
        app = AgentLeadApp(app_config)
        if await app.initialize():
            click.echo("✅ All components initialized successfully")
            
            # Run health check
            health = await app.check_health()
            if health["overall_status"] == "healthy":
                click.echo("✅ System health check passed")
            else:
                click.echo(f"⚠️  System health check: {health['overall_status']}")
            
            await app.shutdown()
        else:
            click.echo("❌ Component initialization failed")
        
    except Exception as e:
        click.echo(f"❌ Validation error: {e}")


def main():
    """Main entry point."""
    # Check for required dependencies
    if not HAS_CLICK:
        print("❌ Error: Click is required for CLI functionality.")
        print("Please install with: pip install click")
        sys.exit(1)
    
    # Add psutil to requirements for system monitoring
    try:
        import psutil
    except ImportError:
        print("Warning: psutil not installed - system monitoring unavailable")
    
    # Check for core dependencies
    missing_deps = []
    if not HAS_KIMI_CONFIG:
        missing_deps.append("Kimi configuration")
    if not HAS_KIMI_EXTRACTOR:
        missing_deps.append("Kimi extractor")
    if missing_deps:
        print("⚠️ Warning: Some components are not available:")
        for dep in missing_deps:
            print(f"   - {dep}")
        print("   Some functionality may be limited.")
        print()
    
    # Run CLI
    import asyncio
    
    # Wrap click commands for async support
    def async_command(f):
        def wrapper(*args, **kwargs):
            return asyncio.run(f(*args, **kwargs))
        return wrapper
    
    # Apply async wrapper to async commands
    extract.callback = async_command(extract.callback)
    batch.callback = async_command(batch.callback)
    health.callback = async_command(health.callback)
    # costs command removed
    validate.callback = async_command(validate.callback)
    
    # Apply async wrapper to enrichment commands
    enrich_leads.callback = async_command(enrich_leads.callback)
    enrichment_status.callback = async_command(enrichment_status.callback)
    cancel_enrichment.callback = async_command(cancel_enrichment.callback)
    enrichment_report.callback = async_command(enrichment_report.callback)
    export_for_clay.callback = async_command(export_for_clay.callback)
    import_clay_data.callback = async_command(import_clay_data.callback)
    clay_config.callback = async_command(clay_config.callback)
    
    # Apply async wrapper to multi-agent commands
    if HAS_MULTI_AGENT:
        research.callback = async_command(research.callback)
        outreach.callback = async_command(outreach.callback)
        full_pipeline.callback = async_command(full_pipeline.callback)
    
    cli()


if __name__ == "__main__":
    main()