-
-
Notifications
You must be signed in to change notification settings - Fork 1.7k
fix: preserve file/line info in PR review extraction recovery #1857
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -18,17 +18,22 @@ | |
| # services/ package at both apps/backend/services/ and runners/github/services/. | ||
| # To avoid collision, add the github services dir directly and import bare module names. | ||
| _backend_dir = Path(__file__).parent.parent / "apps" / "backend" | ||
| _github_services_dir = _backend_dir / "runners" / "github" / "services" | ||
| _github_runner_dir = _backend_dir / "runners" / "github" | ||
| _github_services_dir = _github_runner_dir / "services" | ||
| if str(_backend_dir) not in sys.path: | ||
| sys.path.insert(0, str(_backend_dir)) | ||
| if str(_github_runner_dir) not in sys.path: | ||
| sys.path.insert(0, str(_github_runner_dir)) | ||
| if str(_github_services_dir) not in sys.path: | ||
| sys.path.insert(0, str(_github_services_dir)) | ||
|
Comment on lines
+21
to
28
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🧹 Nitpick | 🔵 Trivial Fragile This file now inserts three directories into 🤖 Prompt for AI Agents |
||
|
|
||
| from agents.tools_pkg.models import AGENT_CONFIGS | ||
| from pydantic_models import ( | ||
| ExtractedFindingSummary, | ||
| FollowupExtractionResponse, | ||
| ParallelFollowupResponse, | ||
| ) | ||
| from recovery_utils import create_finding_from_summary | ||
| from sdk_utils import RECOVERABLE_ERRORS | ||
|
|
||
|
|
||
|
|
@@ -53,20 +58,38 @@ def test_minimal_valid_response(self): | |
| assert resp.dismissed_finding_count == 0 | ||
|
|
||
| def test_full_valid_response(self): | ||
| """Accepts fully populated response.""" | ||
| """Accepts fully populated response with ExtractedFindingSummary objects.""" | ||
| resp = FollowupExtractionResponse( | ||
| verdict="READY_TO_MERGE", | ||
| verdict_reasoning="All findings resolved", | ||
| resolved_finding_ids=["NCR-001", "NCR-002"], | ||
| unresolved_finding_ids=[], | ||
| new_finding_summaries=["HIGH: potential cleanup issue in batch_commands.py"], | ||
| new_finding_summaries=[ | ||
| ExtractedFindingSummary( | ||
| severity="HIGH", | ||
| description="potential cleanup issue in batch_commands.py", | ||
| file="apps/backend/cli/batch_commands.py", | ||
| line=42, | ||
| ) | ||
| ], | ||
| confirmed_finding_count=1, | ||
| dismissed_finding_count=1, | ||
| ) | ||
| assert len(resp.resolved_finding_ids) == 2 | ||
| assert len(resp.new_finding_summaries) == 1 | ||
| assert resp.new_finding_summaries[0].file == "apps/backend/cli/batch_commands.py" | ||
| assert resp.new_finding_summaries[0].line == 42 | ||
| assert resp.confirmed_finding_count == 1 | ||
|
|
||
| def test_finding_summary_defaults(self): | ||
| """ExtractedFindingSummary defaults file='unknown' and line=0.""" | ||
| summary = ExtractedFindingSummary( | ||
| severity="MEDIUM", | ||
| description="Some issue without location", | ||
| ) | ||
| assert summary.file == "unknown" | ||
| assert summary.line == 0 | ||
|
|
||
| def test_schema_is_small(self): | ||
| """Schema should be significantly smaller than ParallelFollowupResponse.""" | ||
| extraction_schema = json.dumps( | ||
|
|
@@ -75,10 +98,11 @@ def test_schema_is_small(self): | |
| followup_schema = json.dumps( | ||
| ParallelFollowupResponse.model_json_schema() | ||
| ) | ||
| # Extraction schema should be less than half the size of the full schema | ||
| assert len(extraction_schema) < len(followup_schema) / 2, ( | ||
| # Actual ratio is ~50.7% after adding ExtractedFindingSummary nesting. | ||
| # Threshold at 55% gives headroom while still guarding against schema bloat. | ||
| assert len(extraction_schema) < len(followup_schema) * 0.55, ( | ||
| f"Extraction schema ({len(extraction_schema)} chars) should be " | ||
| f"less than half of full schema ({len(followup_schema)} chars)" | ||
| f"less than 55% of full schema ({len(followup_schema)} chars)" | ||
| ) | ||
|
|
||
| def test_all_verdict_values_accepted(self): | ||
|
|
@@ -143,3 +167,81 @@ def test_extraction_agent_low_thinking(self): | |
| """Extraction agent should use low thinking (lightweight call).""" | ||
| config = AGENT_CONFIGS["pr_followup_extraction"] | ||
| assert config["thinking_default"] == "low" | ||
|
|
||
|
|
||
| # ============================================================================ | ||
| # Test create_finding_from_summary with file/line params | ||
| # ============================================================================ | ||
|
|
||
|
|
||
| class TestCreateFindingFromSummary: | ||
| """Tests for create_finding_from_summary with file/line support.""" | ||
|
|
||
| def test_backward_compatible_defaults(self): | ||
| """Calling without file/line still produces file='unknown', line=0.""" | ||
| finding = create_finding_from_summary("HIGH: some issue", 0) | ||
| assert finding.file == "unknown" | ||
| assert finding.line == 0 | ||
| assert finding.severity.value == "high" | ||
|
|
||
| def test_file_and_line_passed_through(self): | ||
| """File and line params are used in the resulting finding.""" | ||
| finding = create_finding_from_summary( | ||
| summary="Missing null check", | ||
| index=0, | ||
| file="src/parser.py", | ||
| line=42, | ||
| ) | ||
| assert finding.file == "src/parser.py" | ||
| assert finding.line == 42 | ||
|
|
||
| def test_severity_override(self): | ||
| """severity_override takes precedence over parsed severity.""" | ||
| finding = create_finding_from_summary( | ||
| summary="HIGH: some issue", | ||
| index=0, | ||
| severity_override="CRITICAL", | ||
| ) | ||
| assert finding.severity.value == "critical" | ||
|
|
||
| def test_severity_override_case_insensitive(self): | ||
| """severity_override works regardless of case.""" | ||
| finding = create_finding_from_summary( | ||
| summary="some issue", | ||
| index=0, | ||
| severity_override="high", | ||
| ) | ||
| assert finding.severity.value == "high" | ||
|
|
||
| def test_severity_override_invalid_falls_back(self): | ||
| """Invalid severity_override falls back to parsed severity.""" | ||
| finding = create_finding_from_summary( | ||
| summary="LOW: minor issue", | ||
| index=0, | ||
| severity_override="UNKNOWN", | ||
| ) | ||
| # Falls back to parsed "LOW" from summary | ||
| assert finding.severity.value == "low" | ||
|
|
||
| def test_id_prefix(self): | ||
| """Custom id_prefix is used in the finding ID.""" | ||
| finding = create_finding_from_summary( | ||
| summary="some issue", index=0, id_prefix="FU" | ||
| ) | ||
| assert finding.id.startswith("FU-") | ||
|
|
||
| def test_all_params_together(self): | ||
| """All new params work together correctly.""" | ||
| finding = create_finding_from_summary( | ||
| summary="Regex issue in subtask title truncation", | ||
| index=3, | ||
| id_prefix="FU", | ||
| severity_override="MEDIUM", | ||
| file="apps/backend/agents/planner.py", | ||
| line=187, | ||
| ) | ||
| assert finding.id.startswith("FU-") | ||
| assert finding.severity.value == "medium" | ||
| assert finding.file == "apps/backend/agents/planner.py" | ||
| assert finding.line == 187 | ||
| assert "Regex issue" in finding.title | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The
severityfield inExtractedFindingSummaryis defined as astr. For better type safety and to enforce the allowed values at the schema level, it would be better to usetyping.Literal. This would be consistent with other models in this file that useLiteralfor fields with a fixed set of values. To handle potential case variations from the AI, afield_validatorcan be added to normalize the severity to uppercase before validation.