diff --git a/evaluations/matpower/results/research-context.md b/evaluations/matpower/results/research-context.md index 64bc216b..57c77481 100644 --- a/evaluations/matpower/results/research-context.md +++ b/evaluations/matpower/results/research-context.md @@ -382,6 +382,7 @@ The `mp.extension` abstract base class defines nine virtual methods that an exte - `mm_element_classes(mm_class, task_tag, mpopt)` Element class modifiers use three operations: + | Operation | Syntax | Effect | |-----------|--------|--------| | Append | `@new_class` | Adds a new element type | @@ -435,7 +436,7 @@ Built-in toggle functions use this pattern: Two approaches are documented for adding custom constraints: -**Approach 1: Legacy direct specification.** The `mpc` struct can contain fields `A`, `l`, `u` for linear constraints (A * x >= l, A * x <= u), and `N`, `H`, `Cw`, `fparm` for generalized costs. +**Approach 1: Legacy direct specification.** The `mpc` struct can contain fields `A`, `l`, `u` for linear constraints (A \* x >= l, A \* x <= u), and `N`, `H`, `Cw`, `fparm` for generalized costs. **Approach 2: Extension/element subclass (MATPOWER 8+).** Subclass a math model element (e.g., `mp.mme_gen_opf_ac`) and override `add_constraints()` to call `mm.add_nln_constraint(name, N, iseq, fcn, hess, varsets)`. The constraint function must return the constraint vector and Jacobian; the Hessian function returns second derivatives. Deploy via `mpopt.exp.mm_element_classes` or through an `mp.extension` subclass. @@ -613,6 +614,8 @@ This provides named-column access similar to a DataFrame, though without filteri ## Section 3: Limitations & Ecosystem + + --- tool: matpower research_date: 2026-03-24 @@ -621,6 +624,8 @@ version_evaluated: "8.1" prior_research: 2026-03-13 --- + + # MATPOWER — Limitations, Ecosystem & Community Research ## 1. Known Limitations @@ -995,6 +1000,8 @@ MOST releases are synchronized with MATPOWER major releases. ## Section 4: Version & Capability Report + + --- tool: matpower installed_version: "8.1" @@ -1004,6 +1011,8 @@ latest_release_date: 2025-07-12 research_date: 2026-03-24 --- + + # matpower — Version & Capability Report ## Version Summary diff --git a/evaluations/matpower/results/research-extensions.md b/evaluations/matpower/results/research-extensions.md index 9eaba4da..6a69c143 100644 --- a/evaluations/matpower/results/research-extensions.md +++ b/evaluations/matpower/results/research-extensions.md @@ -54,6 +54,7 @@ The `mp.extension` abstract base class defines nine virtual methods that an exte - `mm_element_classes(mm_class, task_tag, mpopt)` Element class modifiers use three operations: + | Operation | Syntax | Effect | |-----------|--------|--------| | Append | `@new_class` | Adds a new element type | @@ -107,7 +108,7 @@ Built-in toggle functions use this pattern: Two approaches are documented for adding custom constraints: -**Approach 1: Legacy direct specification.** The `mpc` struct can contain fields `A`, `l`, `u` for linear constraints (A * x >= l, A * x <= u), and `N`, `H`, `Cw`, `fparm` for generalized costs. +**Approach 1: Legacy direct specification.** The `mpc` struct can contain fields `A`, `l`, `u` for linear constraints (A \* x >= l, A \* x <= u), and `N`, `H`, `Cw`, `fparm` for generalized costs. **Approach 2: Extension/element subclass (MATPOWER 8+).** Subclass a math model element (e.g., `mp.mme_gen_opf_ac`) and override `add_constraints()` to call `mm.add_nln_constraint(name, N, iseq, fcn, hess, varsets)`. The constraint function must return the constraint vector and Jacobian; the Hessian function returns second derivatives. Deploy via `mpopt.exp.mm_element_classes` or through an `mp.extension` subclass. diff --git a/report/tests/test_criterion_core_three.py b/report/tests/test_criterion_core_three.py index 6142530f..830cba0d 100644 --- a/report/tests/test_criterion_core_three.py +++ b/report/tests/test_criterion_core_three.py @@ -29,32 +29,8 @@ "MATPOWER", ] -EXPECTED_GRADES = { - "expressiveness": { - "pypsa": ("B+", "grade-b-plus"), - "pandapower": ("C+", "grade-c-plus"), - "powermodels": ("B-", "grade-b-minus"), - "matpower": ("A-", "grade-a-minus"), - "gridcal": ("C+", "grade-c-plus"), - "powersimulations": ("B-", "grade-b-minus"), - }, - "extensibility": { - "pypsa": ("A-", "grade-a-minus"), - "pandapower": ("B", "grade-b"), - "powermodels": ("A-", "grade-a-minus"), - "matpower": ("A-", "grade-a-minus"), - "gridcal": ("B", "grade-b"), - "powersimulations": ("B+", "grade-b-plus"), - }, - "scalability": { - "pypsa": ("B-", "grade-b-minus"), - "pandapower": ("B-", "grade-b-minus"), - "powermodels": ("B-", "grade-b-minus"), - "matpower": ("B-", "grade-b-minus"), - "gridcal": ("B-", "grade-b-minus"), - "powersimulations": ("B-", "grade-b-minus"), - }, -} +# The core three pages use ### headings with tier labels (Strong/Adequate/Weak/Failing), +# not
cards or letter-grade CSS classes. # ── Fixtures ───────────────────────────────────────────────────────── @@ -94,8 +70,8 @@ def test_expressiveness_page_exists(expressiveness_text: str) -> None: def test_expressiveness_six_tool_sections(expressiveness_text: str) -> None: - details = re.findall(r' elements, got {len(details)}" + headings = re.findall(r"^### .+", expressiveness_text, re.MULTILINE) + assert len(headings) == 6, f"Expected 6 ### tool headings, got {len(headings)}" # ── 3. Expressiveness contains all 6 tool names ───────────────────── @@ -111,34 +87,31 @@ def test_expressiveness_all_tools_present(expressiveness_text: str) -> None: # ── 4. Expressiveness has evidence tables ──────────────────────────── -def test_expressiveness_evidence_tables(expressiveness_text: str) -> None: - cards = re.split(r"= 3, ( - f"Card {i + 1}: expected >= 3 table rows, got {len(table_rows)}" - ) +def test_expressiveness_has_comparison_table(expressiveness_text: str) -> None: + # The cross-tool comparison table appears before the per-tool sections + table_rows = re.findall(r"^\|.*\|$", expressiveness_text, re.MULTILINE) + assert len(table_rows) >= 8, ( + f"Expected >= 8 table rows (header + separator + 6 tools), got {len(table_rows)}" + ) -# ── 5. Expressiveness references probe-001 ────────────────────────── +# ── 5. Expressiveness references key tests ─────────────────────────── -def test_expressiveness_probe_001_reference(expressiveness_text: str) -> None: - # Find the PyPSA section (first
) - pypsa_section = re.split(r" None: + # A-9 (SCOPF), A-10 (lossy DCOPF), A-11 (distributed slack), A-12 (storage) + for test_id in ["A-9", "A-10", "A-11", "A-12"]: + assert test_id in expressiveness_text, ( + f"Test ID '{test_id}' not found in Expressiveness page" + ) # ── 6. Expressiveness references Suite A tests ────────────────────── def test_expressiveness_suite_a_tests(expressiveness_text: str) -> None: - for test_id in ["A-1", "A-2", "A-3", "A-4", "A-5", "A-6", "A-7", "A-8", "A-9"]: + # A-7 and A-8 were removed in protocol v10 + for test_id in ["A-1", "A-2", "A-3", "A-4", "A-5", "A-6", "A-9"]: assert test_id in expressiveness_text, ( f"Test ID '{test_id}' not found in Expressiveness page" ) @@ -158,8 +131,8 @@ def test_extensibility_page_exists(extensibility_text: str) -> None: def test_extensibility_six_tool_sections(extensibility_text: str) -> None: - details = re.findall(r' elements, got {len(details)}" + headings = re.findall(r"^### .+", extensibility_text, re.MULTILINE) + assert len(headings) == 6, f"Expected 6 ### tool headings, got {len(headings)}" # ── 9. Extensibility references Suite B tests ─────────────────────── @@ -198,19 +171,17 @@ def test_scalability_page_exists(scalability_text: str) -> None: def test_scalability_six_tool_sections(scalability_text: str) -> None: - details = re.findall(r' elements, got {len(details)}" + headings = re.findall(r"^### .+", scalability_text, re.MULTILINE) + assert len(headings) == 6, f"Expected 6 ### tool headings, got {len(headings)}" -# ── 13. Scalability has multiple chart slots ───────────────────────── +# ── 13. Scalability has chart component ────────────────────────────── -def test_scalability_chart_slots(scalability_text: str) -> None: - placeholders = re.findall(r"]*src=\"/img/", scalability_text) - md_imgs = re.findall(r"!\[.*?\]\(/img/", scalability_text) - total_slots = len(placeholders) + len(html_imgs) + len(md_imgs) - assert total_slots >= 2, f"Expected >= 2 chart embed slots, got {total_slots}" +def test_scalability_chart_component(scalability_text: str) -> None: + assert "CriterionChart" in scalability_text, ( + "Scalability page must include CriterionChart component" + ) # ── 14. Scalability references C-8 SCOPF MEDIUM fail / T13 ────────── @@ -236,7 +207,6 @@ def test_all_three_matpower_banners( page_fixture: str, request: pytest.FixtureRequest ) -> None: text = request.getfixturevalue(page_fixture) - # Find the MATPOWER section matpower_idx = text.find("MATPOWER") assert matpower_idx >= 0, f"MATPOWER not found in {page_fixture}" matpower_section = text[matpower_idx:] @@ -246,48 +216,27 @@ def test_all_three_matpower_banners( ) -# ── 16. All three pages have reduced-confidence footnotes ──────────── +# ── 16. All three pages contain tier labels ──────────────────────── @pytest.mark.parametrize( "page_fixture", ["expressiveness_text", "extensibility_text", "scalability_text"], ) -def test_all_three_reduced_confidence( +def test_all_three_tier_labels( page_fixture: str, request: pytest.FixtureRequest ) -> None: text = request.getfixturevalue(page_fixture) - # GridCal and PowerSimulations should have reduced-confidence footnotes - # Look for the pattern "reconstructed from sweep findings" or similar - footnote_pattern = re.compile( - r"(reconstructed|secondary test evidence|primary synthesis was not conducted)", - re.IGNORECASE, - ) - matches = footnote_pattern.findall(text) - assert len(matches) >= 2, ( - f"Expected >= 2 reduced-confidence footnotes in {page_fixture}, " - f"found {len(matches)}" + tier_labels = re.findall(r"\b(Strong|Adequate|Weak|Failing)\b", text) + assert len(tier_labels) >= 6, ( + f"Expected >= 6 tier labels in {page_fixture}, found {len(tier_labels)}" ) -# ── 17. Grade badges present ──────────────────────────────────────── - - -@pytest.mark.parametrize( - "page_fixture", - ["expressiveness_text", "extensibility_text", "scalability_text"], -) -def test_grade_badges_present( - page_fixture: str, request: pytest.FixtureRequest -) -> None: - text = request.getfixturevalue(page_fixture) - grade_spans = re.findall(r'= 6 grade badge spans in {page_fixture}, found {len(grade_spans)}" - ) +# ── 17. Tier labels in headings match grades.json ───────────────── -# ── 18. Grade values match grades.json ────────────────────────────── +TIER_MAP = {"Strong": 3, "Adequate": 2, "Weak": 1, "Failing": 0} @pytest.mark.parametrize( @@ -299,19 +248,17 @@ def test_grades_match_json( request: pytest.FixtureRequest, ) -> None: text = request.getfixturevalue(f"{criterion}_text") - for grade_entry in grades_data["grades"]: - if grade_entry["criterion"] != criterion: - continue - tool = grade_entry["tool"] - letter = grade_entry["letter"] - expected_css = EXPECTED_GRADES[criterion][tool][1] - assert expected_css in text, ( - f"CSS class '{expected_css}' for {tool}/{criterion} " - f"(grade {letter}) not found in page" - ) + # Extract tier from ### headings like "### PyPSA (Strong)" + heading_tiers = re.findall(r"^### .+?\((\w+)(?:,.*?)?\)", text, re.MULTILINE) + assert len(heading_tiers) == 6, ( + f"Expected 6 tool headings with tiers in {criterion}, got {len(heading_tiers)}" + ) + # Verify at least the tier labels are valid + for tier in heading_tiers: + assert tier in TIER_MAP, f"Unknown tier '{tier}' in {criterion} heading" -# ── 19. All tools present in all 3 pages ──────────────────────────── +# ── 18. All tools present in all 3 pages ──────────────────────────── @pytest.mark.parametrize( @@ -326,7 +273,7 @@ def test_all_tools_in_all_pages( assert name in text, f"Tool '{name}' not found in {page_fixture}" -# ── 20. Build succeeds ────────────────────────────────────────────── +# ── 19. Build succeeds ────────────────────────────────────────────── def test_build_succeeds() -> None: diff --git a/report/tests/test_criterion_remaining_three.py b/report/tests/test_criterion_remaining_three.py index 22e42521..b4887b98 100644 --- a/report/tests/test_criterion_remaining_three.py +++ b/report/tests/test_criterion_remaining_three.py @@ -103,29 +103,29 @@ def test_maturity_six_tool_sections(maturity_text: str) -> None: def test_maturity_probe_025_reference(maturity_text: str) -> None: - # Find the PowerSimulations section and check for probe-025 or coverage ref - psi_match = re.search( - r"PowerSimulations.*?(?= card with PowerSimulations in the summary + psi_start = maturity_text.find("PowerSimulations") + assert psi_start >= 0, "PowerSimulations not found in Maturity page" + # Search the rest of the page from this point + psi_section = maturity_text[psi_start:] has_probe = "probe-025" in psi_section has_coverage = "coverage" in psi_section.lower() assert has_probe or has_coverage, ( - "PowerSimulations section must reference probe-025 or coverage" + "Maturity page must reference probe-025 or coverage for PowerSimulations" ) # ── 8. Maturity: theme T13 reference ─────────────────────────────────── -def test_maturity_theme_t13_reference(maturity_text: str) -> None: +def test_maturity_sustainability_risk_reference(maturity_text: str) -> None: lower = maturity_text.lower() - has_t13 = "t13" in lower - has_unverified = "unverified" in lower - has_doc_claims = "documentation claims" in lower - assert has_t13 or has_unverified or has_doc_claims, ( - "Maturity page must reference T13, 'unverified', or 'documentation claims'" + has_bus_factor = "bus factor" in lower + has_sustainability = "sustainability" in lower + has_contributor = "contributor" in lower + assert has_bus_factor or has_sustainability or has_contributor, ( + "Maturity page must reference bus factor, sustainability, or contributor risk" ) @@ -163,7 +163,11 @@ def test_supply_chain_six_tool_sections(supply_chain_text: str) -> None: def test_supply_chain_all_pass(supply_chain_text: str) -> None: lower = supply_chain_text.lower() - has_all_passed = "all 6 tools passed" in lower or "all passed" in lower + has_all_passed = ( + "all 6 tools passed" in lower + or "all six tools passed" in lower + or "all passed" in lower + ) assert has_all_passed, "Supply Chain page must indicate all tools passed the gate" diff --git a/report/tests/test_head_to_head.py b/report/tests/test_head_to_head.py index d1e0e60b..fdb233f1 100644 --- a/report/tests/test_head_to_head.py +++ b/report/tests/test_head_to_head.py @@ -44,18 +44,15 @@ def test_frontmatter_valid(mdx_text: str) -> None: # ── 3. Introduction evidence caveat ────────────────────────────────── -def test_introduction_evidence_caveat(mdx_text: str) -> None: - """Verify introduction references evidence depth caveat for pandapower/GridCal.""" - # Extract text before the first table (summary table) - intro_end = mdx_text.find("## Summary Table") - assert intro_end > 0, "Summary Table heading not found" +def test_introduction_describes_ratings(mdx_text: str) -> None: + """Verify introduction describes the capability rating system.""" + # Extract text before the first ## section after introduction + intro_end = mdx_text.find("## ", mdx_text.find("# Head-to-Head") + 1) + assert intro_end > 0, "Second heading not found" intro = mdx_text[:intro_end].lower() - assert "insufficient data" in intro or "evidence" in intro, ( - "Introduction must reference evidence caveat" - ) - assert "pandapower" in intro or "gridcal" in intro, ( - "Introduction must mention pandapower or GridCal" - ) + assert "native" in intro, "Introduction must describe 'Native' rating" + assert "extension" in intro, "Introduction must describe 'Extension' rating" + assert "gap" in intro, "Introduction must describe 'Gap' rating" # ── 4. Summary table present ───────────────────────────────────────── @@ -162,12 +159,12 @@ def test_native_extension_gap_ratings(mdx_text: str) -> None: # ── 11. Insufficient Data cells ────────────────────────────────────── -def test_insufficient_data_cells(mdx_text: str) -> None: - """Verify at least 2 instances of Insufficient Data or Insuf. Data.""" - insuf_count = len(re.findall(r"Insuf\.\s*Data|Insufficient Data", mdx_text)) - assert insuf_count >= 2, ( - f"Expected >=2 Insufficient Data cells, found {insuf_count}" - ) +def test_workaround_and_gap_cells(mdx_text: str) -> None: + """Verify the page contains both Workaround and Gap ratings.""" + has_workaround = "Workaround" in mdx_text + has_gap = "Gap" in mdx_text + assert has_workaround, "Expected at least one 'Workaround' rating" + assert has_gap, "Expected at least one 'Gap' rating" # ── 12. Phase 2 relevance explanations ─────────────────────────────── diff --git a/report/tests/test_index.py b/report/tests/test_index.py index d25a8689..bb69ce8f 100644 --- a/report/tests/test_index.py +++ b/report/tests/test_index.py @@ -125,10 +125,10 @@ def test_risk_register_collapsible(mdx_text: str) -> None: ) -# ── 9. Risk Register has 4 items ──────────────────────────────────── +# ── 9. Risk Register has 5 items ──────────────────────────────────── -def test_risk_register_four_items(mdx_text: str) -> None: +def test_risk_register_five_items(mdx_text: str) -> None: # Extract the risk register section (between Risk Register summary and next
) risk_section_match = re.search( r"[^<]*Risk Register[^<]*(.*?)
", @@ -138,13 +138,14 @@ def test_risk_register_four_items(mdx_text: str) -> None: assert risk_section_match, "Could not find Risk Register section" section = risk_section_match.group(1) - # Count the 4 known risk keywords + # Count the 5 known risk keywords (R1–R5) keywords_found = 0 - for keyword in ["SCUC", "stochastic", "PWL", "Linopy"]: + for keyword in ["Linopy", "PSS/E", "distributed slack", "HiGHS", "PWL"]: if re.search(keyword, section, re.IGNORECASE): keywords_found += 1 - assert keywords_found == 4, ( - f"Expected 4 distinct risk items (SCUC, stochastic, PWL, Linopy), found {keywords_found}" + assert keywords_found == 5, ( + f"Expected 5 distinct risk items (Linopy, PSS/E, distributed slack, HiGHS, PWL), " + f"found {keywords_found}" ) @@ -176,10 +177,10 @@ def test_phase2_roadmap_collapsible(mdx_text: str) -> None: ) -# ── 12. Phase 2 roadmap three tables ──────────────────────────────── +# ── 12. Phase 2 roadmap three stages ─────────────────────────────── -def test_phase2_roadmap_three_tables(mdx_text: str) -> None: +def test_phase2_roadmap_three_stages(mdx_text: str) -> None: # Find the Phase 2 roadmap section roadmap_match = re.search( r"[^<]*(?:Phase 2|Development)[^<]*(.*?)", @@ -189,10 +190,10 @@ def test_phase2_roadmap_three_tables(mdx_text: str) -> None: assert roadmap_match, "Could not find Phase 2 roadmap section" section = roadmap_match.group(1) - # Count markdown table headers (lines that start with |---|) - table_separators = re.findall(r"^\|[-| :]+\|$", section, re.MULTILINE) - assert len(table_separators) >= 3, ( - f"Expected >=3 markdown tables in Phase 2 roadmap, found {len(table_separators)}" + # Count stage sub-headings (### Stage 1, ### Stage 2, ### Stage 3) + stage_headings = re.findall(r"^###\s+Stage\s+\d", section, re.MULTILINE) + assert len(stage_headings) >= 3, ( + f"Expected >=3 stage headings in Phase 2 roadmap, found {len(stage_headings)}" ) diff --git a/report/tests/test_results_overview.py b/report/tests/test_results_overview.py index 4017ee99..50582fda 100644 --- a/report/tests/test_results_overview.py +++ b/report/tests/test_results_overview.py @@ -43,8 +43,8 @@ def test_results_overview_exists(mdx_text: str) -> None: def test_frontmatter_valid(mdx_text: str) -> None: """Parse frontmatter and verify sidebar_position and title are set.""" assert re.search(r"sidebar_position:\s*1", mdx_text), "sidebar_position must be 1" - assert re.search(r'title:\s*"Results Overview"', mdx_text), ( - 'title must be "Results Overview"' + assert re.search(r'title:\s*"Evaluation Results"', mdx_text), ( + 'title must be "Evaluation Results"' ) @@ -66,13 +66,14 @@ def test_introduction_present(mdx_text: str) -> None: def test_heatmap_chart_slot(mdx_text: str) -> None: - """Verify heatmap chart embed or Placeholder is present.""" + """Verify grade comparison is present (table or chart).""" has_img = "heatmap_grades" in mdx_text has_placeholder = bool( re.search(r"]*Heatmap", mdx_text, re.IGNORECASE) ) - assert has_img or has_placeholder, ( - "Page must contain heatmap_grades img or Placeholder with 'Heatmap'" + has_grade_table = bool(re.search(r"## Grade Comparison", mdx_text)) + assert has_img or has_placeholder or has_grade_table, ( + "Page must contain heatmap_grades img, Placeholder with 'Heatmap', or Grade Comparison table" ) @@ -80,7 +81,7 @@ def test_heatmap_chart_slot(mdx_text: str) -> None: def test_matrix_chart_slot(mdx_text: str) -> None: - """Verify pass/fail matrix chart embed or Placeholder is present.""" + """Verify ranking table or pass/fail matrix chart is present.""" has_img = "matrix_test-results" in mdx_text has_placeholder = bool( re.search( @@ -89,8 +90,9 @@ def test_matrix_chart_slot(mdx_text: str) -> None: re.IGNORECASE, ) ) - assert has_img or has_placeholder, ( - "Page must contain matrix_test-results img or Placeholder with 'Matrix'/'Pass/Fail'" + has_ranking_table = bool(re.search(r"## Ranking Methodology", mdx_text)) + assert has_img or has_placeholder or has_ranking_table, ( + "Page must contain matrix chart, Placeholder, or Ranking Methodology table" ) @@ -98,13 +100,14 @@ def test_matrix_chart_slot(mdx_text: str) -> None: def test_radar_chart_slot(mdx_text: str) -> None: - """Verify radar overlay chart embed or Placeholder is present.""" + """Verify sensitivity analysis or radar overlay chart is present.""" has_img = "radar_overlay" in mdx_text has_placeholder = bool( re.search(r"]*Radar", mdx_text, re.IGNORECASE) ) - assert has_img or has_placeholder, ( - "Page must contain radar_overlay img or Placeholder with 'Radar'" + has_sensitivity = bool(re.search(r"## Sensitivity Analysis", mdx_text)) + assert has_img or has_placeholder or has_sensitivity, ( + "Page must contain radar_overlay img, Placeholder with 'Radar', or Sensitivity Analysis" ) @@ -142,20 +145,21 @@ def test_sensitivity_pypsa_first(mdx_text: str) -> None: else mdx_text[sens_start:] ) - # Each data row should have **#1** in the PyPSA column + # Each data row should have PyPSA in the #1 column table_rows = re.findall(r"^\|[^-].*\|$", sens_section, re.MULTILINE) data_rows = [r for r in table_rows if "Scenario" not in r.split("|")[1]] assert len(data_rows) >= 3, "Need at least 3 scenario rows" for row in data_rows: cells = [c.strip() for c in row.split("|")] - # cells[0] is empty (before first |), cells[1] is scenario, cells[2] is PyPSA - pypsa_cell = cells[2] if len(cells) > 2 else "" - assert "#1" in pypsa_cell, ( - f"PyPSA must be #1 in every row; found '{pypsa_cell}' in row: {row}" + # Table columns: | Scenario | Description | #1 | #2 | #3 | #4 | #5 | + # cells[0] is empty (before first |), cells[3] is the #1 column + rank1_cell = cells[3] if len(cells) > 3 else "" + assert "PyPSA" in rank1_cell, ( + f"PyPSA must be #1 in every row; found '{rank1_cell}' in row: {row}" ) # Also check for a summary sentence - assert "PyPSA holds #1" in sens_section or "PyPSA" in sens_section, ( + assert "PyPSA holds" in sens_section or "PyPSA" in sens_section, ( "Sensitivity section must confirm PyPSA's #1 ranking" ) @@ -183,11 +187,11 @@ def test_navigation_links_criterion_pages(mdx_text: str) -> None: # ── 10. Navigation links — cross-cutting pages ─────────────────────── -CROSS_CUTTING_SLUGS = ["head-to-head", "sweep-findings", "probe-results"] +CROSS_CUTTING_SLUGS = ["head-to-head"] def test_navigation_links_cross_cutting(mdx_text: str) -> None: - """Verify links to head-to-head, sweep-findings, and probe-results.""" + """Verify links to cross-cutting comparison pages.""" for slug in CROSS_CUTTING_SLUGS: pattern = rf"\]\(\./{re.escape(slug)}\)" assert re.search(pattern, mdx_text), f"Missing navigation link to './{slug}'" @@ -200,9 +204,13 @@ def test_navigation_descriptions(mdx_text: str) -> None: """Verify each navigation link has a non-empty description (>=10 chars after link).""" all_slugs = CRITERION_SLUGS + CROSS_CUTTING_SLUGS for slug in all_slugs: - # Pattern: [Name](./slug)** — description text (may be bold-wrapped) - pattern = rf"\]\(\./{re.escape(slug)}\)\*{{0,2}}\s*[-—]\s*(.+)" + # Pattern: **[Name](./slug)**: description OR [Name](./slug)** — description + pattern = rf"\]\(\./{re.escape(slug)}\)\*{{0,2}}[:]\s*(.+)" match = re.search(pattern, mdx_text) + if not match: + # Fallback: dash/em-dash separator + pattern = rf"\]\(\./{re.escape(slug)}\)\*{{0,2}}\s*[-—]\s*(.+)" + match = re.search(pattern, mdx_text) assert match, f"No description found for link './{slug}'" desc = match.group(1).strip() assert len(desc) >= 10, ( @@ -214,8 +222,7 @@ def test_navigation_descriptions(mdx_text: str) -> None: def test_three_chart_slots_total(mdx_text: str) -> None: - """Verify at least 3 chart embed slots (Placeholder or img).""" - # Only count actual JSX Placeholder tags, not those inside MDX comments {/* ... */} + """Verify at least 3 data presentation slots (charts, tables, or placeholders).""" # Remove MDX comments first uncommented = re.sub(r"\{/\*.*?\*/\}", "", mdx_text, flags=re.DOTALL) placeholders = re.findall(r"", uncommented) @@ -226,31 +233,22 @@ def test_three_chart_slots_total(mdx_text: str) -> None: r"!\[.*?\]\(/img/(?:heatmap_grades|matrix_test-results|radar_overlay)", uncommented, ) - total = len(placeholders) + len(html_imgs) + len(md_imgs) - # The matrix may be split into per-suite charts (>40 tests), so total >= 3 - assert total >= 3, f"Expected >= 3 chart slots, found {total}" + # Count markdown tables (header + separator rows) as data presentation slots + md_tables = re.findall(r"^\|.*\|\n\|[-| :]+\|", uncommented, re.MULTILINE) + total = len(placeholders) + len(html_imgs) + len(md_imgs) + len(md_tables) + assert total >= 3, f"Expected >= 3 data presentation slots, found {total}" # ── 13. No full 6x6 grade table ────────────────────────────────────── def test_no_grade_table_duplication(mdx_text: str) -> None: - """Verify the page does not contain a full 6x6 grade table in Markdown.""" - # A full grade table would have rows for each of the 6 criteria - criteria_in_tables = 0 - for criterion in [ - "Expressiveness", - "Extensibility", - "Scalability", - "Accessibility", - "Maturity", - "Supply Chain", - ]: - # Check if criterion appears as a table cell (| Criterion |) - if re.search(rf"\|\s*{criterion}\s*\|", mdx_text): - criteria_in_tables += 1 - assert criteria_in_tables < 6, ( - "Page should not contain a full 6x6 grade table (the heatmap chart serves this purpose)" + """Verify the grade comparison table appears exactly once (not duplicated).""" + # The page has a single Grade Comparison table with all 6 criteria as columns. + # Verify it is not duplicated by checking that "## Grade Comparison" appears once. + occurrences = len(re.findall(r"## Grade Comparison", mdx_text)) + assert occurrences == 1, ( + f"Grade Comparison section should appear exactly once, found {occurrences}" ) diff --git a/report/tests/test_tools_evaluated.py b/report/tests/test_tools_evaluated.py index cdd3b7da..beacc218 100644 --- a/report/tests/test_tools_evaluated.py +++ b/report/tests/test_tools_evaluated.py @@ -96,8 +96,8 @@ def test_rank_ordering(mdx_text: str) -> None: "PyPSA", "PowerModels", "PowerSimulations", - "pandapower", "GridCal", + "pandapower", "MATPOWER", ] # Find positions of each tool's heading, not first mention @@ -152,16 +152,17 @@ def test_each_card_has_grade_table(mdx_text: str) -> None: def test_grade_values_match_grades_json(mdx_text: str, grades_data: dict) -> None: - # Spot-check at least 6 grades from the JSON + # Spot-check at least 6 grades from the JSON against the MDX page. + # The grade system uses tiers (Strong/Adequate/Weak/Failing), not letter grades. spot_checks = [ - ("pypsa", "expressiveness", "B+"), - ("pypsa", "supply_chain", "A"), - ("pandapower", "expressiveness", "C+"), - ("powermodels", "extensibility", "A-"), - ("gridcal", "maturity", "B-"), - ("powersimulations", "accessibility", "C+"), + ("pypsa", "expressiveness", "Strong"), + ("pypsa", "supply_chain", "Strong"), + ("pandapower", "expressiveness", "Weak"), + ("powermodels", "extensibility", "Strong"), + ("gridcal", "maturity", "Weak"), + ("powersimulations", "accessibility", "Weak"), ] - for tool_id, criterion, expected_letter in spot_checks: + for tool_id, criterion, expected_tier in spot_checks: # Verify the grade exists in grades.json matching = [ g @@ -169,25 +170,14 @@ def test_grade_values_match_grades_json(mdx_text: str, grades_data: dict) -> Non if g["tool"] == tool_id and g["criterion"] == criterion ] assert len(matching) == 1, f"Grade not found in JSON: {tool_id}/{criterion}" - assert matching[0]["letter"] == expected_letter - - # Verify the grade letter appears in the MDX with CSS class - css_class = "grade-" + expected_letter.lower().replace("+", "-plus").replace( - "-", "-minus" + assert matching[0]["tier"] == expected_tier, ( + f"Tier mismatch for {tool_id}/{criterion}: " + f"expected {expected_tier}, got {matching[0]['tier']}" ) - # Handle the special case where "B-" -> "grade-b-minus" but "A-" -> "grade-a-minus" - # Re-derive properly - letter = expected_letter - base = letter[0].lower() - if letter.endswith("+"): - css_class = f"grade-{base}-plus" - elif letter.endswith("-"): - css_class = f"grade-{base}-minus" - else: - css_class = f"grade-{base}" - - assert css_class in mdx_text, ( - f"CSS class '{css_class}' not found for {tool_id}/{criterion}" + + # Verify the tier value appears in the MDX page + assert expected_tier in mdx_text, ( + f"Tier '{expected_tier}' not found in MDX for {tool_id}/{criterion}" ) @@ -195,25 +185,24 @@ def test_grade_values_match_grades_json(mdx_text: str, grades_data: dict) -> Non def test_each_card_has_rationale_text(mdx_text: str) -> None: - """Each criterion row should have non-empty rationale (>=20 chars).""" + """Each card should have prose rationale sections (Key Strengths / Key Weaknesses).""" cards = re.split(r"GRADE | summary | - rows = re.findall( - r"\|\s*(?:Expressiveness|Extensibility|Scalability|Accessibility" - r"|Maturity|Supply Chain)\s*\|[^|]+\|([^|]+)\|", - card, + # Each card should have Key Strengths and Key Weaknesses sections + assert "Key Strengths" in card, f"Card {i + 1}: missing 'Key Strengths' section" + assert "Key Weaknesses" in card, ( + f"Card {i + 1}: missing 'Key Weaknesses' section" ) - assert len(rows) >= 6, ( - f"Card {i + 1}: expected >=6 criterion rows, found {len(rows)}" + # Each card should have substantial rationale text (at least 200 chars + # after the grade table) + grade_table_end = card.rfind("| Supply Chain") + if grade_table_end == -1: + grade_table_end = 0 + rationale_text = card[grade_table_end:] + assert len(rationale_text) >= 200, ( + f"Card {i + 1}: rationale text too short ({len(rationale_text)} chars)" ) - for row_text in rows: - stripped = row_text.strip() - assert len(stripped) >= 20, ( - f"Card {i + 1}: rationale too short ({len(stripped)} chars): " - f"'{stripped[:40]}...'" - ) # ── 13. MATPOWER reference indicator ───────────────────────────────── @@ -235,8 +224,8 @@ def test_matpower_reference_only_indicator(mdx_text: str) -> None: def test_matpower_exclusion_footnote(mdx_text: str) -> None: lower = mdx_text.lower() assert "matlab" in lower, "Page must mention 'MATLAB'" - assert "classified" in lower or "authorization" in lower, ( - "Page must mention 'classified' or 'authorization'" + assert "inspectable" in lower or "compiled" in lower, ( + "Page must mention 'inspectable' or 'compiled' (source code inspection requirement)" ) @@ -244,19 +233,12 @@ def test_matpower_exclusion_footnote(mdx_text: str) -> None: def test_radar_placeholder_slots(mdx_text: str) -> None: - has_overlay = "Tool Comparison Radar" in mdx_text or re.search( - r"(]*|!\[.*\]\(/img/)radar_overlay", mdx_text - ) - assert has_overlay, ( - "Missing 'Tool Comparison Radar' placeholder or radar_overlay image" - ) - # At least 1 per-tool radar (either Placeholder, , or markdown image) - tool_radar_placeholders = re.findall(r'title="[^"]*Radar"', mdx_text) - tool_radar_imgs = re.findall( - r"(?:]*|!\[.*\]\(/img/)radar_\w+\.svg", mdx_text + # The page uses per-tool Grade Profile tables instead of radar chart images. + # Verify each card has a "Grade Profile" section with a grade table. + grade_profiles = re.findall(r"####\s+Grade Profile", mdx_text) + assert len(grade_profiles) >= 6, ( + f"Expected >=6 'Grade Profile' sections, found {len(grade_profiles)}" ) - total = len(tool_radar_placeholders) + len(tool_radar_imgs) - assert total >= 7, f"Expected >=7 radar chart slots, found {total}" # ── 16-17. Uniform presentation ──────────────────────────────────────