Issue #6 Enhance Verdict Writer

.env-example

@@ -5,7 +5,8 @@
 
 # ===== MODEL ACCESS AND SELECTION (OPENAI) =====
 
-OPENAI_API_KEY=your-openai-api-key-here           # For embeddings and optional model access 
+# OpenAI API Configuration
+OPENAI_API_KEY=your_openai_api_key_here
 
 # Default model for general processing and fallback
 DEFAULT_MODEL=gpt-4o
@@ -32,6 +33,9 @@ SERPER_API_KEY=your_serper_api_key_here           # Only needed if USE_SERPER=tr
 
 # ===== DATABASE CONFIGURATION =====
 
+# Database Configuration
+DATABASE_URL=postgresql://username:password@localhost:5432/verifact
+
 # Supabase Configuration
 SUPABASE_URL=https://your-project.supabase.co     # [REQUIRED] URL from Supabase dashboard
 SUPABASE_KEY=your-supabase-anon-key               # [REQUIRED] Public anon key
@@ -66,9 +70,23 @@ PORT=8000
 API_KEY_ENABLED=true                              # Enable API key authentication
 API_KEY_HEADER_NAME=X-API-Key                     # Header name for API keys
 DEFAULT_API_KEY=your-default-api-key              # Default API key
-RATE_LIMIT_ENABLED=true                           # Enable rate limiting
-RATE_LIMIT_REQUESTS=100                           # Number of requests per window
-RATE_LIMIT_WINDOW=3600                            # Rate limit window in seconds
+
+# Application Settings
+DEBUG=False
+ENVIRONMENT=development
+LOG_LEVEL=INFO
+
+# API Rate Limiting
+RATE_LIMIT_REQUESTS=100
+RATE_LIMIT_PERIOD=3600
+
+# Feature Flags
+ENABLE_CACHING=True
+ENABLE_ASYNC_PROCESSING=True
+
+# Security Settings
+SECRET_KEY=your_secret_key_here
+ALLOWED_HOSTS=localhost,127.0.0.1
 
 # ===== ADVANCED CONFIGURATION =====
 
@@ -78,7 +96,9 @@ ENABLE_MODEL_CACHING=true                         # Cache model responses
 MODEL_CACHE_SIZE=1000                             # Number of responses to cache
 
 # Logging Configuration
-ENVIRONMENT=development
-LOG_LEVEL=INFO                                    # DEBUG, INFO, WARNING, ERROR, CRITICAL
 LOG_FORMAT=plain
-# LOG_FILE=/path/to/log/file.log                  # Uncomment to enable file logging
+# LOG_FILE=/path/to/log/file.log                  # Uncomment to enable file logging
+
+# Optional: External Service Integration
+# ENABLE_SENTRY=False
+# SENTRY_DSN=your_sentry_dsn_here

pytest.ini

@@ -5,4 +5,6 @@ python_classes = Test*
 python_functions = test_*
 
 # Display detailed test results
-addopts = --verbose 
+addopts = --verbose --tb=short
+filterwarnings =
+    ignore::PendingDeprecationWarning:starlette.formparsers 

src/tests/conftest.py

@@ -0,0 +1,37 @@
+import pytest
+import warnings
+from typing import Generator, Dict, Any
+from src.verifact_agents.verdict_writer import VerdictWriter
+
+# Suppress the starlette multipart deprecation warning
+warnings.filterwarnings("ignore", category=PendingDeprecationWarning, module="starlette.formparsers")
+
+@pytest.fixture(scope="session")
+def test_evidence() -> Dict[str, Any]:
+    """Provide sample evidence for testing."""
+    return {
+        "content": "This is a sample evidence content for testing purposes.",
+        "source": "Test Source",
+        "relevance": 0.9,
+        "stance": "supporting"
+    }
+
+@pytest.fixture(scope="session")
+def test_claim() -> str:
+    """Provide a sample claim for testing."""
+    return "This is a test claim that needs verification."
+
+@pytest.fixture(scope="session")
+def verdict_writer() -> Generator[VerdictWriter, None, None]:
+    """Create a VerdictWriter instance for testing."""
+    writer = VerdictWriter()
+    yield writer
+
+@pytest.fixture(autouse=True)
+def setup_test_environment():
+    """Set up test environment variables."""
+    import os
+    os.environ["ENVIRONMENT"] = "test"
+    os.environ["LOG_LEVEL"] = "DEBUG"
+    yield
+    # Cleanup after tests if needed

src/tests/verifact_agents/test_verdict_writer.py

@@ -0,0 +1,138 @@
+import pytest
+from src.verifact_agents.verdict_writer import VerdictWriter, Verdict
+
+@pytest.mark.asyncio
+async def test_verdict_writer_returns_valid_verdict():
+    writer = VerdictWriter()
+    claim = "Bananas are a good source of potassium."
+    evidence = [
+        {
+            "content": "Bananas contain around 422 mg of potassium per medium fruit, making them a good dietary source.",
+            "source": "Healthline",
+            "relevance": 0.9,
+            "stance": "supporting"
+        },
+        {
+            "content": "Potatoes and beans contain more potassium than bananas, but bananas are still a decent source.",
+            "source": "Harvard School of Public Health",
+            "relevance": 0.8,
+            "stance": "neutral"
+        }
+    ]
+
+    verdict = await writer.run(claim=claim, evidence=evidence, detail_level="standard")
+
+    assert isinstance(verdict, Verdict)
+    assert verdict.claim == claim
+    assert verdict.verdict in ["true", "false", "partially true", "unverifiable"]
+    assert 0.0 <= verdict.confidence <= 1.0
+    assert isinstance(verdict.explanation, str) and len(verdict.explanation.strip()) > 0
+    assert isinstance(verdict.sources, list) and all(isinstance(s, str) for s in verdict.sources)
+    assert len(verdict.sources) > 0
+
+@pytest.mark.asyncio
+async def test_verdict_writer_brief_detail():
+    writer = VerdictWriter()
+    claim = "Water freezes at 0 degrees Celsius."
+    evidence = [
+        {
+            "content": "Under standard atmospheric conditions, water freezes at 0°C.",
+            "source": "Britannica",
+            "relevance": 0.95,
+            "stance": "supporting"
+        }
+    ]
+
+    verdict = await writer.run(claim=claim, evidence=evidence, detail_level="brief")
+
+    assert isinstance(verdict.explanation, str)
+    assert len(verdict.explanation.split()) <= 30  # brief explanation is usually short
+
+@pytest.mark.asyncio
+async def test_verdict_writer_detailed_includes_sources():
+    writer = VerdictWriter()
+    claim = "The Earth revolves around the Sun."
+    evidence = [
+        {
+            "content": "Astronomical evidence and observations confirm the heliocentric model.",
+            "source": "NASA",
+            "relevance": 0.99,
+            "stance": "supporting"
+        }
+    ]
+
+    verdict = await writer.run(claim=claim, evidence=evidence, detail_level="detailed")
+
+    assert any(source.lower().startswith("http") == False for source in verdict.sources)
+    assert "nasa" in " ".join(verdict.sources).lower()
+    assert "sun" in verdict.explanation.lower() or "earth" in verdict.explanation.lower()
+
+@pytest.mark.asyncio
+async def test_confidence_score_considers_evidence_relevance():
+    writer = VerdictWriter()
+    claim = "Cats are better pets than dogs."
+    evidence = [
+        {"content": "Cats require less maintenance.", "source": "PetGuide", "relevance": 0.9, "stance": "supporting"},
+        {"content": "Dogs are more loyal and emotionally supportive.", "source": "DogWorld", "relevance": 0.85, "stance": "contradicting"},
+    ]
+    verdict = await writer.run(claim=claim, evidence=evidence, detail_level="standard")
+    assert 0.4 <= verdict.confidence <= 0.6  # Mixed evidence should yield moderate confidence
+
+@pytest.mark.asyncio
+async def test_explanation_maintains_political_neutrality():
+    writer = VerdictWriter()
+    claim = "Voter ID laws reduce election fraud."
+    evidence = [
+        {"content": "Some studies suggest voter ID laws deter fraud.", "source": "Heritage Foundation", "relevance": 0.8, "stance": "supporting"},
+        {"content": "Other studies show minimal fraud cases regardless of ID laws.", "source": "Brennan Center", "relevance": 0.9, "stance": "contradicting"},
+    ]
+    verdict = await writer.run(claim=claim, evidence=evidence, detail_level="detailed")
+    explanation = verdict.explanation.lower()
+    assert "republican" not in explanation
+    assert "democrat" not in explanation
+    assert "bias" not in explanation
+
+@pytest.mark.asyncio
+async def test_alternative_perspectives_are_included():
+    writer = VerdictWriter()
+    claim = "Electric cars are better for the environment."
+    evidence = [
+        {"content": "EVs emit less CO2 over their lifetime.", "source": "EPA", "relevance": 0.9, "stance": "supporting"},
+        {"content": "Battery production has environmental impacts.", "source": "Nature", "relevance": 0.8, "stance": "contradicting"},
+    ]
+    verdict = await writer.run(claim=claim, evidence=evidence, detail_level="detailed")
+    explanation = verdict.explanation.lower()
+    assert "battery" in explanation or "production" in explanation
+    assert any(term in explanation for term in ["emissions", "co2", "co₂", "co 2"])
+
+@pytest.mark.asyncio
+async def test_explanation_detail_levels_vary():
+    writer = VerdictWriter()
+    claim = "Electric cars are better for the environment than gas cars."
+    evidence = [
+        {
+            "content": "Electric vehicles produce fewer greenhouse gas emissions over their lifetime.",
+            "source": "EPA",
+            "relevance": 0.95,
+            "stance": "supporting"
+        },
+        {
+            "content": "Battery production for electric vehicles involves mining and emissions.",
+            "source": "Nature",
+            "relevance": 0.8,
+            "stance": "contradicting"
+        }
+    ]
+
+    brief = await writer.run(claim=claim, evidence=evidence, detail_level="brief")
+    standard = await writer.run(claim=claim, evidence=evidence, detail_level="standard")
+    detailed = await writer.run(claim=claim, evidence=evidence, detail_level="detailed")
+
+    # Ensure increasing richness in explanation
+    assert len(brief.explanation.split()) < len(standard.explanation.split()) < len(detailed.explanation.split())
+
+    # Optional: Check sources and content presence in detailed output
+    assert len(detailed.sources) >= 2
+    explanation = detailed.explanation.lower()
+    assert "battery" in explanation
+    assert "emissions" in explanation or "co2" in explanation