From eb0e922adb70db66378735171c69ee3c94b43a13 Mon Sep 17 00:00:00 2001 From: Joe Date: Wed, 8 Apr 2026 14:46:47 -0600 Subject: [PATCH] chore: remove Grid Operations Primer section from report site Remove the primer page, SVG diagrams, style guide, validation doc, test suite, sidebar entry, and smoke test expectation. --- CLAUDE.md | 89 + data/fnm/.gitignore | 28 + data/fnm/README.md | 51 + data/fnm/__init__.py | 1 + data/fnm/docs/README.md | 11 + data/fnm/docs/field-criticality-matrix.md | 633 ++++++ data/fnm/docs/intermediate-schema.md | 1253 ++++++++++++ data/fnm/docs/mapping-guide.md | 458 +++++ data/fnm/docs/parser-comparison-report.md | 202 ++ data/fnm/docs/per-unit-conventions.md | 828 ++++++++ data/fnm/docs/rubric-v4-justification.md | 97 + .../docs/supplemental-csv-representability.md | 104 + data/fnm/docs/supplemental-csvs.md | 477 +++++ data/fnm/docs/three-winding-transformers.md | 661 +++++++ data/fnm/intermediate/README.md | 16 + .../fnm/intermediate/schemas/area.schema.json | 67 + .../intermediate/schemas/branch.schema.json | 280 +++ data/fnm/intermediate/schemas/bus.schema.json | 184 ++ .../intermediate/schemas/facts.schema.json | 156 ++ .../schemas/fixed_shunt.schema.json | 70 + .../schemas/generator.schema.json | 320 +++ .../schemas/impedance_correction.schema.json | 243 +++ .../schemas/interarea_transfer.schema.json | 56 + .../fnm/intermediate/schemas/load.schema.json | 155 ++ .../intermediate/schemas/manifest.schema.json | 104 + .../schemas/multi_section_line.schema.json | 157 ++ .../schemas/multi_terminal_dc.schema.json | 96 + .../intermediate/schemas/owner.schema.json | 34 + .../schemas/switched_shunt.schema.json | 304 +++ .../schemas/transformer.schema.json | 883 +++++++++ .../schemas/two_terminal_dc.schema.json | 484 +++++ .../intermediate/schemas/vsc_dc.schema.json | 427 ++++ .../fnm/intermediate/schemas/zone.schema.json | 34 + data/fnm/manifest.json | 63 + data/fnm/reference/README.md | 16 + data/fnm/reference/pass_conditions.json | 196 ++ data/fnm/scripts/README.md | 15 + data/fnm/scripts/__init__.py | 605 ++++++ data/fnm/scripts/acpf_reference.py | 1366 +++++++++++++ data/fnm/scripts/bus_exclusion_registry.py | 1188 +++++++++++ data/fnm/scripts/csv_join_keys.py | 1102 +++++++++++ .../fnm/scripts/dcpf_acpf_characterization.py | 1753 ++++++++++++++++ data/fnm/scripts/dcpf_reference.py | 1624 +++++++++++++++ data/fnm/scripts/diagnose_acpf.m | 149 ++ data/fnm/scripts/export_intermediate_csvs.py | 1364 +++++++++++++ data/fnm/scripts/fnm_gating.py | 240 +++ data/fnm/scripts/fnm_gating_cli.py | 78 + data/fnm/scripts/fnm_gating_fixtures.py | 71 + data/fnm/scripts/generate_schema_reference.py | 1699 ++++++++++++++++ data/fnm/scripts/gridcal_parser.py | 696 +++++++ data/fnm/scripts/intermediate_schema.py | 1591 +++++++++++++++ data/fnm/scripts/manifest_io.py | 336 ++++ data/fnm/scripts/matpower_parser.py | 571 ++++++ data/fnm/scripts/parser_comparison.py | 1375 +++++++++++++ data/fnm/scripts/pass_conditions.py | 1759 +++++++++++++++++ data/fnm/scripts/raw_record_counter.py | 290 +++ data/fnm/scripts/run_psse2mpc.m | 111 ++ data/fnm/scripts/solve_main_island.m | 280 +++ data/fnm/scripts/solve_references.m | 554 ++++++ data/fnm/scripts/solved_snapshot.py | 966 +++++++++ .../test_dcpf_reference_separate_tables.py | 547 +++++ .../scripts/test_export_intermediate_csvs.py | 569 ++++++ .../test_validate_dcpf_reproducibility.py | 548 +++++ .../scripts/test_verify_materialization.py | 270 +++ data/fnm/scripts/tests/__init__.py | 1 + .../test_csv_representability_summary.py | 396 ++++ data/fnm/scripts/tests/test_fnm_gating.py | 166 ++ data/fnm/scripts/tests/test_fnm_gating_cli.py | 111 ++ .../scripts/tests/test_fnm_gating_fixtures.py | 84 + .../test_intermediate_schema_reference.py | 539 +++++ data/fnm/scripts/tests/test_manifest.py | 271 +++ .../tests/test_supplemental_csv_reference.py | 608 ++++++ .../scripts/validate_dcpf_reproducibility.py | 843 ++++++++ data/fnm/scripts/validation_report.py | 1621 +++++++++++++++ data/fnm/scripts/verify_materialization.py | 557 ++++++ data/fnm/tests/__init__.py | 1 + data/fnm/tests/conftest.py | 43 + data/fnm/tests/test_acpf_reference.py | 890 +++++++++ data/fnm/tests/test_bus_exclusion_registry.py | 507 +++++ data/fnm/tests/test_csv_join_keys.py | 472 +++++ .../tests/test_dcpf_acpf_characterization.py | 795 ++++++++ data/fnm/tests/test_dcpf_reference.py | 1001 ++++++++++ .../tests/test_field_criticality_matrix.py | 555 ++++++ data/fnm/tests/test_gridcal_parser.py | 262 +++ data/fnm/tests/test_intermediate_schema.py | 688 +++++++ data/fnm/tests/test_mapping_guide.py | 437 ++++ data/fnm/tests/test_matpower_parser.py | 415 ++++ data/fnm/tests/test_parser_comparison.py | 575 ++++++ data/fnm/tests/test_pass_conditions.py | 608 ++++++ .../tests/test_per_unit_conventions_doc.py | 371 ++++ data/fnm/tests/test_raw_record_counter.py | 284 +++ .../fnm/tests/test_rubric_v4_justification.py | 360 ++++ data/fnm/tests/test_solved_snapshot.py | 436 ++++ .../test_three_winding_transformers_doc.py | 353 ++++ data/fnm/tests/test_validation_report.py | 610 ++++++ .../assets/grid-primer-diagram-validation.md | 148 -- report/docs/assets/grid-primer-style-guide.md | 148 -- report/docs/grid-primer.mdx | 287 --- report/scripts/smoke_test.py | 5 - report/sidebars.js | 1 - .../grid-primer/stage-1_single-bus.excalidraw | 41 - .../img/grid-primer/stage-1_single-bus.svg | 24 - .../grid-primer/stage-2_two-bus.excalidraw | 62 - .../img/grid-primer/stage-2_two-bus.svg | 41 - .../stage-3_meshed-network.excalidraw | 81 - .../grid-primer/stage-3_meshed-network.svg | 75 - .../stage-4_opf-dispatch.excalidraw | 87 - .../img/grid-primer/stage-4_opf-dispatch.svg | 91 - .../grid-primer/stage-5_congestion.excalidraw | 67 - .../img/grid-primer/stage-5_congestion.svg | 88 - .../img/grid-primer/stage-6_scopf.excalidraw | 78 - .../static/img/grid-primer/stage-6_scopf.svg | 105 - report/tests/test_grid_primer_prose.py | 427 ---- 113 files changed, 45815 insertions(+), 1856 deletions(-) create mode 100644 CLAUDE.md create mode 100644 data/fnm/.gitignore create mode 100644 data/fnm/README.md create mode 100644 data/fnm/__init__.py create mode 100644 data/fnm/docs/README.md create mode 100644 data/fnm/docs/field-criticality-matrix.md create mode 100644 data/fnm/docs/intermediate-schema.md create mode 100644 data/fnm/docs/mapping-guide.md create mode 100644 data/fnm/docs/parser-comparison-report.md create mode 100644 data/fnm/docs/per-unit-conventions.md create mode 100644 data/fnm/docs/rubric-v4-justification.md create mode 100644 data/fnm/docs/supplemental-csv-representability.md create mode 100644 data/fnm/docs/supplemental-csvs.md create mode 100644 data/fnm/docs/three-winding-transformers.md create mode 100644 data/fnm/intermediate/README.md create mode 100644 data/fnm/intermediate/schemas/area.schema.json create mode 100644 data/fnm/intermediate/schemas/branch.schema.json create mode 100644 data/fnm/intermediate/schemas/bus.schema.json create mode 100644 data/fnm/intermediate/schemas/facts.schema.json create mode 100644 data/fnm/intermediate/schemas/fixed_shunt.schema.json create mode 100644 data/fnm/intermediate/schemas/generator.schema.json create mode 100644 data/fnm/intermediate/schemas/impedance_correction.schema.json create mode 100644 data/fnm/intermediate/schemas/interarea_transfer.schema.json create mode 100644 data/fnm/intermediate/schemas/load.schema.json create mode 100644 data/fnm/intermediate/schemas/manifest.schema.json create mode 100644 data/fnm/intermediate/schemas/multi_section_line.schema.json create mode 100644 data/fnm/intermediate/schemas/multi_terminal_dc.schema.json create mode 100644 data/fnm/intermediate/schemas/owner.schema.json create mode 100644 data/fnm/intermediate/schemas/switched_shunt.schema.json create mode 100644 data/fnm/intermediate/schemas/transformer.schema.json create mode 100644 data/fnm/intermediate/schemas/two_terminal_dc.schema.json create mode 100644 data/fnm/intermediate/schemas/vsc_dc.schema.json create mode 100644 data/fnm/intermediate/schemas/zone.schema.json create mode 100644 data/fnm/manifest.json create mode 100644 data/fnm/reference/README.md create mode 100644 data/fnm/reference/pass_conditions.json create mode 100644 data/fnm/scripts/README.md create mode 100644 data/fnm/scripts/__init__.py create mode 100644 data/fnm/scripts/acpf_reference.py create mode 100644 data/fnm/scripts/bus_exclusion_registry.py create mode 100644 data/fnm/scripts/csv_join_keys.py create mode 100644 data/fnm/scripts/dcpf_acpf_characterization.py create mode 100644 data/fnm/scripts/dcpf_reference.py create mode 100644 data/fnm/scripts/diagnose_acpf.m create mode 100644 data/fnm/scripts/export_intermediate_csvs.py create mode 100644 data/fnm/scripts/fnm_gating.py create mode 100644 data/fnm/scripts/fnm_gating_cli.py create mode 100644 data/fnm/scripts/fnm_gating_fixtures.py create mode 100644 data/fnm/scripts/generate_schema_reference.py create mode 100644 data/fnm/scripts/gridcal_parser.py create mode 100644 data/fnm/scripts/intermediate_schema.py create mode 100644 data/fnm/scripts/manifest_io.py create mode 100644 data/fnm/scripts/matpower_parser.py create mode 100644 data/fnm/scripts/parser_comparison.py create mode 100644 data/fnm/scripts/pass_conditions.py create mode 100644 data/fnm/scripts/raw_record_counter.py create mode 100644 data/fnm/scripts/run_psse2mpc.m create mode 100644 data/fnm/scripts/solve_main_island.m create mode 100644 data/fnm/scripts/solve_references.m create mode 100644 data/fnm/scripts/solved_snapshot.py create mode 100644 data/fnm/scripts/test_dcpf_reference_separate_tables.py create mode 100644 data/fnm/scripts/test_export_intermediate_csvs.py create mode 100644 data/fnm/scripts/test_validate_dcpf_reproducibility.py create mode 100644 data/fnm/scripts/test_verify_materialization.py create mode 100644 data/fnm/scripts/tests/__init__.py create mode 100644 data/fnm/scripts/tests/test_csv_representability_summary.py create mode 100644 data/fnm/scripts/tests/test_fnm_gating.py create mode 100644 data/fnm/scripts/tests/test_fnm_gating_cli.py create mode 100644 data/fnm/scripts/tests/test_fnm_gating_fixtures.py create mode 100644 data/fnm/scripts/tests/test_intermediate_schema_reference.py create mode 100644 data/fnm/scripts/tests/test_manifest.py create mode 100644 data/fnm/scripts/tests/test_supplemental_csv_reference.py create mode 100644 data/fnm/scripts/validate_dcpf_reproducibility.py create mode 100644 data/fnm/scripts/validation_report.py create mode 100644 data/fnm/scripts/verify_materialization.py create mode 100644 data/fnm/tests/__init__.py create mode 100644 data/fnm/tests/conftest.py create mode 100644 data/fnm/tests/test_acpf_reference.py create mode 100644 data/fnm/tests/test_bus_exclusion_registry.py create mode 100644 data/fnm/tests/test_csv_join_keys.py create mode 100644 data/fnm/tests/test_dcpf_acpf_characterization.py create mode 100644 data/fnm/tests/test_dcpf_reference.py create mode 100644 data/fnm/tests/test_field_criticality_matrix.py create mode 100644 data/fnm/tests/test_gridcal_parser.py create mode 100644 data/fnm/tests/test_intermediate_schema.py create mode 100644 data/fnm/tests/test_mapping_guide.py create mode 100644 data/fnm/tests/test_matpower_parser.py create mode 100644 data/fnm/tests/test_parser_comparison.py create mode 100644 data/fnm/tests/test_pass_conditions.py create mode 100644 data/fnm/tests/test_per_unit_conventions_doc.py create mode 100644 data/fnm/tests/test_raw_record_counter.py create mode 100644 data/fnm/tests/test_rubric_v4_justification.py create mode 100644 data/fnm/tests/test_solved_snapshot.py create mode 100644 data/fnm/tests/test_three_winding_transformers_doc.py create mode 100644 data/fnm/tests/test_validation_report.py delete mode 100644 report/docs/assets/grid-primer-diagram-validation.md delete mode 100644 report/docs/assets/grid-primer-style-guide.md delete mode 100644 report/docs/grid-primer.mdx delete mode 100644 report/static/img/grid-primer/stage-1_single-bus.excalidraw delete mode 100644 report/static/img/grid-primer/stage-1_single-bus.svg delete mode 100644 report/static/img/grid-primer/stage-2_two-bus.excalidraw delete mode 100644 report/static/img/grid-primer/stage-2_two-bus.svg delete mode 100644 report/static/img/grid-primer/stage-3_meshed-network.excalidraw delete mode 100644 report/static/img/grid-primer/stage-3_meshed-network.svg delete mode 100644 report/static/img/grid-primer/stage-4_opf-dispatch.excalidraw delete mode 100644 report/static/img/grid-primer/stage-4_opf-dispatch.svg delete mode 100644 report/static/img/grid-primer/stage-5_congestion.excalidraw delete mode 100644 report/static/img/grid-primer/stage-5_congestion.svg delete mode 100644 report/static/img/grid-primer/stage-6_scopf.excalidraw delete mode 100644 report/static/img/grid-primer/stage-6_scopf.svg delete mode 100644 report/tests/test_grid_primer_prose.py diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..36f26742 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,89 @@ +# CLAUDE.md — grc-tech-evaluation + +## Repo Overview + +Standalone evaluation workspace for comparing six power-system modeling tools. +This is NOT an installable Python package — each `evaluations//` directory +is an independent project with its own virtualenv and dependency set. + +No ZGE internal dependencies. No connection to the trading platform repos. + +## Directory Layout + +- `evaluation_guides/` — Rubric and test protocol (read-only reference) +- `data/networks/` — Shared MATPOWER .m case files used by all tools +- `evaluations//` — One directory per tool under evaluation +- `evaluations//results//` — Test outputs organized by rubric dimension + +## Execution Environment + +**Never run code, tests, linters, or pre-commit locally.** Always use the devcontainer. +All commands (pytest, julia, octave, pre-commit, etc.) must run inside the container. + +Use the `dc-exec` helper (`.devcontainer/dc-exec`) which works from both the main +checkout and any git worktree: + +```bash +# Open a shell inside the devcontainer +.devcontainer/dc-exec bash + +# Run a one-off command +.devcontainer/dc-exec + +# Run in a specific container directory +.devcontainer/dc-exec -C /workspace/evaluations/pypsa uv run python -c "import pypsa" +``` + +`dc-exec` finds the container via `git worktree list` + Docker labels, so it works +from `.claude/worktrees//` where `devcontainer exec --workspace-folder .` cannot. + +## Per-Tool Setup + +### Python tools (pypsa, pandapower, gridcal) + +```bash +cd evaluations/ +uv sync +uv run python verify_install.py +``` + +Do NOT use `pip install` — these are uv-managed projects. + +### Julia tools (powermodels, powersimulations) + +```bash +cd evaluations/ +julia --project=. -e 'using Pkg; Pkg.instantiate()' +julia --project=. verify_install.jl +``` + +**Julia startup is slow by design.** `Pkg.instantiate()` compiles packages to native code on first run +(can take many minutes). Subsequent runs reuse the precompiled cache but still pay a 5–15s load +tax per invocation to deserialize it. + +For repeated evaluation runs, stay in the REPL and `include()` your script instead of re-launching: + +```julia +# start once +julia --project=. + +# then inside the REPL, re-run without restart overhead: +julia> include("my_eval_script.jl") +``` + +### MATPOWER (Octave) + +```bash +cd evaluations/matpower +bash setup.sh # downloads MATPOWER 8.1 +octave verify_install.m +``` + +## Conventions + +- Python 3.12, Julia 1.10 +- Ruff for Python linting (line-length = 100) +- Conventional commits enforced by pre-commit +- Each tool has isolated dependencies — no shared virtualenv +- Results go in `evaluations//results//` +- The seven rubric dimensions: gate, expressiveness, extensibility, scalability, accessibility, maturity, supply_chain diff --git a/data/fnm/.gitignore b/data/fnm/.gitignore new file mode 100644 index 00000000..9fbc0fe0 --- /dev/null +++ b/data/fnm/.gitignore @@ -0,0 +1,28 @@ +# FNM Data Files — NDA-restricted, never commit +*.raw +*.RAW +*.csv +*.CSV +*.parquet +*.m + +# Intermediate and reference output directories +intermediate/** +!intermediate/ +!intermediate/README.md +!intermediate/schemas/ +!intermediate/schemas/*.json +reference/** +!reference/ +!reference/*/ +!reference/pass_conditions.json + +# Allow tracked infrastructure files +!manifest.json +!README.md +!**/README.md +!scripts/**/*.py +!scripts/**/*.m +!docs/**/*.md +!docs/**/*.json +!.gitignore diff --git a/data/fnm/README.md b/data/fnm/README.md new file mode 100644 index 00000000..affda0eb --- /dev/null +++ b/data/fnm/README.md @@ -0,0 +1,51 @@ +# FNM Data Directory + +This directory contains artifacts related to the Full Network Model (FNM) ingestion +pipeline. The FNM data itself is NDA-restricted and **must never be committed to version +control**. + +## Directory Layout + +``` +data/fnm/ +├── manifest.json # Machine-readable list of expected FNM source files +├── .gitignore # Blocks all FNM data files from version control +├── README.md # This file +├── intermediate/ # Parser output (Parquet, intermediate formats) +├── reference/ # Phase 3 verification / reference solution datasets +├── docs/ # Reference documentation for PSS/E record types, supplemental CSVs +└── scripts/ # Python modules for parsing, validation, and manifest I/O +``` + +## NDA Restrictions + +FNM source files (PSS/E RAW files and supplemental CSVs) are provided under NDA. +They must not be committed, shared publicly, or stored in any unencrypted location outside +approved infrastructure. + +The `.gitignore` in this directory blocks all data file extensions (`*.raw`, `*.csv`, +`*.parquet`, `*.m`) and the `intermediate/` and `reference/` directories from being tracked. + +## FNM_PATH Environment Variable + +All FNM-dependent code is gated by the `FNM_PATH` environment variable. Set it to the +directory containing the FNM source files: + +```bash +export FNM_PATH=/path/to/fnm/source/files +``` + +When `FNM_PATH` is not set, FNM-dependent tests are skipped and parsers will raise errors +if invoked directly. + +## Obtaining FNM Source Files + +FNM source files must be obtained through the ISO's authorized distribution channels. +Contact the data engineering team for access. The `manifest.json` file lists all expected +source files with descriptions. + +## Manifest + +The `manifest.json` file enumerates every expected FNM source file. Use the +`scripts/manifest_io.py` module to load, validate, and update the manifest +programmatically rather than editing the JSON directly. diff --git a/data/fnm/__init__.py b/data/fnm/__init__.py new file mode 100644 index 00000000..9d48db4f --- /dev/null +++ b/data/fnm/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/data/fnm/docs/README.md b/data/fnm/docs/README.md new file mode 100644 index 00000000..3b22b2d2 --- /dev/null +++ b/data/fnm/docs/README.md @@ -0,0 +1,11 @@ +# FNM Documentation Directory + +This directory contains reference documentation for FNM data formats, PSS/E record types, +and supplemental CSV schemas. Documentation here supports developers working with the +FNM ingestion pipeline. + +## Contents (populated by later phases) + +- PSS/E v31 RAW format record type documentation +- Supplemental CSV field descriptions and schemas +- Data dictionary mapping FNM fields to internal representations diff --git a/data/fnm/docs/field-criticality-matrix.md b/data/fnm/docs/field-criticality-matrix.md new file mode 100644 index 00000000..7ed2c5d4 --- /dev/null +++ b/data/fnm/docs/field-criticality-matrix.md @@ -0,0 +1,633 @@ +# Field Criticality Matrix + +**Version:** 1.0 +**Audience:** evaluate-tool agents, human reviewers +**Primary use:** Fidelity scoring weight assignment for FNM ingestion evaluation +**Dependencies:** Intermediate Schema Reference (`intermediate-schema.md`), + Record-Type Mapping Guide (`mapping-guide.md`), + Per-Unit Convention Reference (`per-unit-conventions.md`) + +## Tier Definitions + +| Tier | Label | Scope | Description | +|------|-------|-------|-------------| +| 1 | DCPF-critical | DC Power Flow | Field is required for correct DC power flow results. Loss or corruption of this field causes incorrect real power flows, generation dispatch, or topology. Includes: bus type codes, branch topology (from/to bus), branch reactance, generator active power output, load active power, transformer tap magnitude, phase shifter angle. | +| 2 | ACPF-critical | AC Power Flow | Field is additionally required for correct AC power flow convergence and accuracy. Not needed for DCPF but essential for ACPF. Includes: branch resistance and charging susceptance, bus voltage magnitude and angle (solved state), generator reactive power limits and voltage setpoint, transformer tap limits and control mode, switched shunt discrete steps, area interchange targets, shunt admittance values. | +| 3 | Informational | Context / Metadata | Field provides organizational, descriptive, or operational context but does not enter the power flow Jacobian or affect the power flow solution. Loss of this field reduces data completeness but does not degrade power flow accuracy. Includes: bus names, area/zone/owner numbers and names, generator MVA base, line thermal ratings, equipment identifiers. | +| 4 | Discardable | Always-Default / Padding | Field is confirmed by Phase 1 D7's `x-psse-present-but-inactive` annotation to be uniformly at the PSS/E default value across all FNM records. It carries no information for this specific network model. Tools that omit or zero this field lose nothing. Evaluate-tool must not penalize omission. | + +The tiers are strictly ordered: DCPF-critical is the highest priority, Discardable is the lowest. +A field is assigned to the highest applicable tier (e.g., a field needed for both DCPF and ACPF is DCPF-critical, not ACPF-critical). +The Discardable tier is not a judgment about the field's general importance -- it only reflects that the field is at its default value in *this specific FNM file*. + +## Summary + +| Record Type | Total | DCPF-Critical | ACPF-Critical | Informational | Discardable | +|-------------|-------|---------------|---------------|---------------|-------------| +| Bus | 13 | 3 | 2 | 8 | 0 | +| Load | 13 | 4 | 5 | 4 | 0 | +| Fixed Shunt | 5 | 0 | 5 | 0 | 0 | +| Generator | 28 | 4 | 5 | 19 | 0 | +| Branch | 24 | 5 | 6 | 13 | 0 | +| Transformer | 83 | 10 | 44 | 29 | 0 | +| Area | 5 | 0 | 3 | 2 | 0 | +| Two-Terminal DC | 46 | 0 | 46 | 0 | 0 | +| VSC DC | 41 | 0 | 41 | 0 | 0 | +| Impedance Correction | 23 | 0 | 23 | 0 | 0 | +| Multi-Terminal DC | 8 | 0 | 8 | 0 | 0 | +| Multi-Section Line | 13 | 0 | 12 | 1 | 0 | +| Zone | 2 | 0 | 0 | 2 | 0 | +| Interarea Transfer | 4 | 0 | 0 | 4 | 0 | +| Owner | 2 | 0 | 0 | 2 | 0 | +| FACTS | 14 | 0 | 14 | 0 | 0 | +| Switched Shunt | 26 | 0 | 23 | 3 | 0 | +| **Total** | **350** | **26** | **237** | **87** | **0** | + +## Bus + +**Intermediate format table:** `bus` +**Record-type tier (from mapping guide):** Tier 1 -- Essential for any power flow +**Total fields:** 13 +**Tier breakdown:** 3 DCPF-critical, 2 ACPF-critical, 8 Informational, 0 Discardable + +| Field | Type | Tier | Rationale | +|-------|------|------|-----------| +| `I` | integer | DCPF-critical | Bus number is the primary key for all topology references; every branch, generator, and load references a bus by this number, making it essential for constructing the B-matrix and adjacency structure | +| `NAME` | string | Informational | Human-readable bus name used for identification in reports and diagrams; does not enter the power flow equations or affect any computed result | +| `BASKV` | number | ACPF-critical | Base voltage in kV defines the per-unit voltage base for this bus; does not enter the DCPF B-matrix directly but is essential for converting per-unit impedances to physical units and for correct impedance base calculation in ACPF | +| `IDE` | integer | DCPF-critical | Bus type code (1=PQ, 2=PV, 3=swing, 4=isolated) determines which variables are fixed versus solved in the power flow formulation, controlling topology and solution structure in both DCPF and ACPF | +| `AREA` | integer | Informational | Area number for organizational grouping; the bus-level area assignment is a grouping key used for interchange accounting, not a direct variable in the power flow equations | +| `ZONE` | integer | Informational | Zone number for geographic or administrative grouping used in reporting and load allocation; no direct impact on power flow equations | +| `OWNER` | integer | Informational | Ownership tracking number; administrative metadata for cost allocation with no electrical effect on the power flow solution | +| `VM` | number | ACPF-critical | Bus voltage magnitude in per-unit serves as the initial condition for ACPF iteration and the solved-state verification target; not used in DCPF which solves only for voltage angles | +| `VA` | number | DCPF-critical | Bus voltage angle in degrees is the primary DCPF solution variable; the swing bus angle serves as the reference and all branch power flows are computed from angle differences | +| `NVHI` | number | Informational | Normal operating voltage high limit in per-unit; used for post-solution voltage violation monitoring and OPF constraints, does not enter the power flow Jacobian | +| `NVLO` | number | Informational | Normal operating voltage low limit in per-unit; post-solution monitoring parameter for voltage violation flagging, not a power flow variable | +| `EVHI` | number | Informational | Emergency voltage high limit in per-unit; applied during contingency analysis for wider voltage tolerance, not a variable in the power flow equations | +| `EVLO` | number | Informational | Emergency voltage low limit in per-unit; contingency analysis parameter with no effect on the power flow solution | + +Voltage limit fields (NVHI, NVLO, EVHI, EVLO) are classified as Informational rather than ACPF-critical because they are post-solution constraint checks used in OPF and contingency analysis, not variables in the power flow equations. + +## Load + +**Intermediate format table:** `load` +**Record-type tier (from mapping guide):** Tier 1 -- Essential for any power flow +**Total fields:** 13 +**Tier breakdown:** 3 DCPF-critical, 5 ACPF-critical, 5 Informational, 0 Discardable + +| Field | Type | Tier | Rationale | +|-------|------|------|-----------| +| `I` | integer | DCPF-critical | Bus number where load withdraws power; determines which bus in the network topology receives the power withdrawal, essential for the DCPF power injection vector | +| `ID` | string | Informational | Two-character load identifier distinguishing multiple loads at the same bus. Traceability label only — tools that aggregate multiple loads to bus-level active power (e.g., via PPC import) produce identical bus injections and identical DCPF results. Absence does not affect bus angles or branch flows. | +| `STATUS` | integer | DCPF-critical | Load status (1=in-service, 0=out-of-service) determines whether this load's power withdrawal is included in the power flow solution, directly affecting the power injection vector | +| `AREA` | integer | Informational | Area assignment for this load used for area interchange calculations; organizational grouping that does not directly enter the power flow equations | +| `ZONE` | integer | Informational | Zone assignment for this load; administrative grouping for reporting with no power flow impact | +| `PL` | number | DCPF-critical | Constant-power active load demand in MW; direct real power withdrawal at the bus entering the DCPF power injection vector and the ACPF power balance equations | +| `QL` | number | ACPF-critical | Constant-power reactive load demand in MVAR; enters the ACPF reactive power balance equations but is ignored in DCPF which does not model reactive power | +| `IP` | number | ACPF-critical | Constant-current active power component in MW at 1.0 pu voltage; voltage-dependent load model that scales linearly with bus voltage magnitude in ACPF | +| `IQ` | number | ACPF-critical | Constant-current reactive power component in MVAR at 1.0 pu voltage; voltage-dependent reactive load entering the ACPF reactive balance equations | +| `YP` | number | ACPF-critical | Constant-admittance active power component in MW at 1.0 pu voltage; scales with the square of bus voltage magnitude in the ACPF load model | +| `YQ` | number | ACPF-critical | Constant-admittance reactive power component in MVAR at 1.0 pu voltage; voltage-squared-dependent reactive load in the ACPF formulation | +| `OWNER` | integer | Informational | Owner number for this load; administrative metadata for cost allocation with no effect on power flow computation | +| `SCALE` | integer | Informational | Load scaling flag (1=participates in scaling, 0=fixed); operational parameter controlling whether the load is adjusted during area interchange scaling, not a direct power flow variable | + +## Fixed Shunt + +**Intermediate format table:** `fixed_shunt` +**Record-type tier (from mapping guide):** Tier 2 -- Required for ACPF accuracy +**Total fields:** 5 +**Tier breakdown:** 0 DCPF-critical, 5 ACPF-critical, 0 Informational, 0 Discardable + +| Field | Type | Tier | Rationale | +|-------|------|------|-----------| +| `I` | integer | ACPF-critical | Bus number where the fixed shunt is connected; determines placement of the shunt admittance in the Y-bus matrix for ACPF, not needed for DCPF which ignores reactive compensation | +| `ID` | string | ACPF-critical | Shunt identifier forming part of the composite primary key; required to distinguish multiple shunts at the same bus and correctly aggregate their admittance contributions in ACPF | +| `STATUS` | integer | ACPF-critical | Shunt status (1=in-service, 0=out-of-service) determines whether this shunt's admittance is included in the Y-bus matrix for ACPF computation | +| `GL` | number | ACPF-critical | Active component of shunt admittance to ground in MW at 1.0 pu voltage; contributes real power loss to the Y-bus diagonal element in ACPF | +| `BL` | number | ACPF-critical | Reactive component of shunt admittance to ground in MVAR at 1.0 pu voltage; provides reactive power compensation in the Y-bus diagonal for ACPF voltage regulation | + +All fixed shunt fields are ACPF-critical because fixed shunts provide reactive compensation that enters the Y-bus admittance matrix in ACPF but is ignored in DCPF's lossless real-power-only formulation. + +## Generator + +**Intermediate format table:** `generator` +**Record-type tier (from mapping guide):** Tier 1 -- Essential for any power flow +**Total fields:** 28 +**Tier breakdown:** 3 DCPF-critical, 5 ACPF-critical, 20 Informational, 0 Discardable + +| Field | Type | Tier | Rationale | +|-------|------|------|-----------| +| `I` | integer | DCPF-critical | Bus number where generator injects power; determines the bus in the network topology that receives the generation injection, essential for the DCPF power injection vector | +| `ID` | string | Informational | Two-character machine identifier distinguishing multiple generators at the same bus. Traceability label only — tools that enumerate generators by bus-row index rather than ID produce identical bus injection sums and identical DCPF results. Absence does not affect bus angles or branch flows. | +| `PG` | number | DCPF-critical | Active power output in MW; direct real power injection at the bus entering the DCPF power injection vector and ACPF active power balance equations | +| `QG` | number | ACPF-critical | Reactive power output in MVAR; enters the ACPF reactive power balance equations as a solved variable at PV buses, not used in DCPF | +| `QT` | number | ACPF-critical | Maximum reactive power limit in MVAR; determines the upper bound for PV-to-PQ bus conversion when the generator reaches its reactive capability limit in ACPF | +| `QB` | number | ACPF-critical | Minimum reactive power limit in MVAR; determines the lower bound for PV-to-PQ bus conversion in ACPF reactive capability enforcement | +| `VS` | number | ACPF-critical | Voltage setpoint in per-unit for PV bus regulation; the target voltage that the generator maintains by adjusting reactive output in ACPF iteration | +| `IREG` | integer | ACPF-critical | Remote regulated bus number (0=local regulation); determines which bus voltage is controlled by this generator in ACPF, critical for correct voltage regulation topology | +| `MBASE` | number | Informational | Machine MVA base for per-unit impedance conversion; relevant for generator internal impedance in dynamic and short-circuit studies, not for steady-state power flow | +| `ZR` | number | Informational | Machine resistance in per-unit on MBASE; part of the generator internal impedance model for short-circuit studies, not used in steady-state power flow computation | +| `ZX` | number | Informational | Machine reactance in per-unit on MBASE; sub-transient or transient reactance for dynamic studies, not a steady-state power flow parameter | +| `RT` | number | Informational | Step-up transformer resistance in per-unit on MBASE; generator-side transformer impedance for short-circuit analysis, not used in steady-state power flow | +| `XT` | number | Informational | Step-up transformer reactance in per-unit on MBASE; generator-side transformer impedance for dynamic studies, not a power flow variable | +| `GTAP` | number | Informational | Step-up transformer off-nominal turns ratio in per-unit; relates to the generator-side transformer model for dynamic studies, not used in steady-state power flow | +| `STAT` | integer | DCPF-critical | Generator status (1=in-service, 0=out-of-service); determines whether this generator's power injection is included in the power flow solution, directly affecting topology and power balance | +| `RMPCT` | number | Informational | Percent of total MVAR range allocated to remote voltage regulation; operational parameter for distributed slack reactive control, not a direct power flow equation variable | +| `PT` | number | Informational | Maximum active power output in MW (turbine limit); operational context for generator remote regulation calculations, not a constraint in basic DCPF or ACPF | +| `PB` | number | Informational | Minimum active power output in MW (turbine limit); operational context for generator remote regulation calculations, not a constraint in basic power flow | +| `O1` | integer | Informational | Owner number 1; administrative ownership tracking with no electrical effect on the power flow solution | +| `F1` | number | Informational | Fraction of generator owned by owner 1; ownership cost allocation metadata with no power flow impact | +| `O2` | integer | Informational | Owner number 2; secondary ownership tracking with no electrical effect on the power flow solution | +| `F2` | number | Informational | Fraction owned by owner 2; ownership metadata with no power flow impact | +| `O3` | integer | Informational | Owner number 3; tertiary ownership tracking with no electrical effect on the power flow solution | +| `F3` | number | Informational | Fraction owned by owner 3; ownership metadata with no power flow impact | +| `O4` | integer | Informational | Owner number 4; quaternary ownership tracking with no electrical effect on the power flow solution | +| `F4` | number | Informational | Fraction owned by owner 4; ownership metadata with no power flow impact | +| `WMOD` | integer | Informational | Wind machine reactive power control mode (0=standard); operational mode flag for wind-specific generator models, does not affect the steady-state power flow equations | +| `WPF` | number | Informational | Wind machine power factor for WMOD=1 mode; wind-specific operational parameter, not a direct power flow variable in steady-state analysis | + +## Branch + +**Intermediate format table:** `branch` +**Record-type tier (from mapping guide):** Tier 1 -- Essential for any power flow +**Total fields:** 24 +**Tier breakdown:** 4 DCPF-critical, 6 ACPF-critical, 14 Informational, 0 Discardable + +| Field | Type | Tier | Rationale | +|-------|------|------|-----------| +| `I` | integer | DCPF-critical | From-bus number defining one endpoint of the branch in the network topology; essential for constructing the B-matrix adjacency structure in DCPF | +| `J` | integer | DCPF-critical | To-bus number defining the other endpoint of the branch; together with I, establishes the branch connectivity required for DCPF and ACPF admittance matrices | +| `CKT` | string | Informational | Circuit identifier distinguishing parallel branches between the same bus pair. Traceability label only — tools that enumerate parallel branches by row index rather than CKT enumerate all branch impedances correctly and produce identical B-matrix construction and DCPF results. Absence does not affect bus angles or branch flows. | +| `R` | number | ACPF-critical | Series resistance in per-unit on system MVA base; enters the Y-bus for ACPF real power loss computation but is ignored in DCPF which assumes lossless branches | +| `X` | number | DCPF-critical | Series reactance in per-unit on system MVA base; enters the B-matrix for DCPF (as 1/X) and the Y-bus for ACPF, the dominant impedance component for power transfer | +| `B` | number | ACPF-critical | Total line charging susceptance in per-unit; provides reactive power injection at both branch endpoints in ACPF, ignored in DCPF's lossless model | +| `RATEA` | number | Informational | Normal thermal rating in MVA (Rate A); post-solution constraint check for continuous loading monitoring, does not enter the power flow equations | +| `RATEB` | number | Informational | Emergency thermal rating in MVA (Rate B); post-solution overload limit, not a power flow variable | +| `RATEC` | number | Informational | Short-term thermal rating in MVA (Rate C); post-solution constraint for emergency loading, not a power flow variable | +| `GI` | number | ACPF-critical | From-bus end shunt conductance in per-unit; contributes to the Y-bus diagonal element at the from-bus in ACPF, representing distributed line losses | +| `BI` | number | ACPF-critical | From-bus end shunt susceptance in per-unit; contributes to the Y-bus diagonal at the from-bus in ACPF, representing asymmetric line charging | +| `GJ` | number | ACPF-critical | To-bus end shunt conductance in per-unit; contributes to the Y-bus diagonal at the to-bus in ACPF, representing distributed line losses | +| `BJ` | number | ACPF-critical | To-bus end shunt susceptance in per-unit; contributes to the Y-bus diagonal at the to-bus in ACPF, representing asymmetric line charging | +| `ST` | integer | DCPF-critical | Branch status (1=in-service, 0=out-of-service); determines whether this branch exists in the network topology for both DCPF and ACPF admittance matrix construction | +| `MET` | integer | Informational | Metered end flag (1=from-bus, 2=to-bus); determines which end is used for loss allocation in accounting, does not affect the power flow solution | +| `LEN` | number | Informational | Line length in user-selected units; informational field for documentation and distance-based calculations, not used in power flow computation | +| `O1` | integer | Informational | Owner number 1; administrative ownership tracking with no electrical effect on the power flow solution | +| `F1` | number | Informational | Fraction owned by owner 1; ownership cost allocation metadata with no power flow impact | +| `O2` | integer | Informational | Owner number 2; secondary ownership tracking with no electrical effect on the power flow solution | +| `F2` | number | Informational | Fraction owned by owner 2; ownership metadata with no power flow impact | +| `O3` | integer | Informational | Owner number 3; tertiary ownership tracking with no electrical effect on the power flow solution | +| `F3` | number | Informational | Fraction owned by owner 3; ownership metadata with no power flow impact | +| `O4` | integer | Informational | Owner number 4; quaternary ownership tracking with no electrical effect on the power flow solution | +| `F4` | number | Informational | Fraction owned by owner 4; ownership metadata with no power flow impact | + +Line ratings (RATEA, RATEB, RATEC) are classified as Informational rather than ACPF-critical because thermal limits do not affect the power flow solution -- they are post-solution constraint checks used in contingency analysis and OPF. + +## Transformer + +**Intermediate format table:** `transformer` +**Record-type tier (from mapping guide):** Tier 1 -- Essential for any power flow +**Total fields:** 83 +**Tier breakdown:** 6 DCPF-critical, 44 ACPF-critical, 33 Informational, 0 Discardable + +| Field | Type | Tier | Rationale | +|-------|------|------|-----------| +| `I` | integer | DCPF-critical | Winding 1 (primary) bus number; defines one endpoint of the transformer in the network topology, essential for the B-matrix adjacency structure | +| `J` | integer | DCPF-critical | Winding 2 (secondary) bus number; defines the other endpoint, establishing the transformer branch connectivity | +| `K` | integer | Informational | Winding 3 (tertiary) bus number; K=0 for 2-winding, K!=0 for 3-winding. Tools loading from MATPOWER PPC format receive pre-expanded star-equivalent 2-winding pairs — the original K value is not present but the star-equivalent topology preserves identical B-matrix structure and DCPF results. Tools using native PSS/E 3-winding support carry this field natively. | +| `CKT` | string | Informational | Circuit identifier for parallel transformers between the same bus pair. Traceability label only — same rationale as Branch.CKT; enumeration by row index produces identical admittance matrix construction. | +| `CW` | integer | ACPF-critical | Winding data I/O code determining how WINDV tap ratios are interpreted (1=pu on bus base kV, 2=kV, 3=pu on nominal kV); controls impedance parameter interpretation for ACPF | +| `CZ` | integer | ACPF-critical | Impedance data I/O code determining the per-unit base for R and X values (1=pu on system base, 2=pu on winding base, 3=losses in W); essential for correct impedance conversion in ACPF | +| `CM` | integer | ACPF-critical | Magnetizing admittance I/O code determining how MAG1/MAG2 are interpreted (1=pu on system base, 2=exciting current/losses); controls magnetizing branch parameter interpretation | +| `MAG1` | number | ACPF-critical | Magnetizing conductance or no-load losses (depending on CM); contributes to the Y-bus shunt admittance at the transformer primary in ACPF | +| `MAG2` | number | ACPF-critical | Magnetizing susceptance or exciting current (depending on CM); contributes to the Y-bus shunt admittance representing core losses and magnetizing current in ACPF | +| `NMETR` | integer | Informational | Non-metered end code (1, 2, or 3); determines which winding is used for loss allocation accounting, does not affect the power flow solution | +| `NAME` | string | Informational | Transformer name up to 12 characters; human-readable identification with no effect on power flow computation | +| `STAT` | integer | DCPF-critical | Transformer status (0=all out, 1=in-service, 2-4=partial winding status); determines whether and how the transformer exists in the network topology for admittance matrix construction | +| `O1` | integer | Informational | Owner number 1; administrative ownership tracking with no electrical effect on the power flow solution | +| `F1` | number | Informational | Fraction owned by owner 1; ownership cost allocation metadata with no power flow impact | +| `O2` | integer | Informational | Owner number 2; secondary ownership tracking with no electrical effect on the power flow solution | +| `F2` | number | Informational | Fraction owned by owner 2; ownership metadata with no power flow impact | +| `O3` | integer | Informational | Owner number 3; tertiary ownership tracking with no electrical effect on the power flow solution | +| `F3` | number | Informational | Fraction owned by owner 3; ownership metadata with no power flow impact | +| `O4` | integer | Informational | Owner number 4; quaternary ownership tracking with no electrical effect on the power flow solution | +| `F4` | number | Informational | Fraction owned by owner 4; ownership metadata with no power flow impact | +| `VECGRP` | string | Informational | Vector group designation (e.g., YNyn0); describes the winding connection configuration for reference, does not directly enter the power flow equations | +| `R1_2` | number | ACPF-critical | Resistance of winding 1-2 pair; enters the Y-bus for ACPF real power loss computation through the transformer, ignored in DCPF's lossless assumption | +| `X1_2` | number | DCPF-critical | Reactance of winding 1-2 pair; enters the B-matrix for DCPF (as 1/X) and the Y-bus for ACPF, the dominant impedance parameter for transformer power transfer | +| `SBASE1_2` | number | ACPF-critical | MVA base for winding 1-2 impedance; determines the per-unit base for R1_2 and X1_2 when CZ=1, essential for correct impedance conversion in ACPF | +| `R2_3` | number | ACPF-critical | Resistance of winding 2-3 pair (3-winding only); enters the Y-bus for the star-bus equivalent 2-3 leg in ACPF | +| `X2_3` | number | Informational | Reactance of winding 2-3 pair (3-winding only). Tools loading from MATPOWER PPC receive pre-computed star-equivalent branch impedances that embed X2_3's contribution — the raw PSS/E field is absent but DCPF accuracy is preserved via the star-equivalent. Tools with native 3-winding support carry X2_3 directly. | +| `SBASE2_3` | number | ACPF-critical | MVA base for winding 2-3 impedance; determines per-unit base for R2_3/X2_3 when CZ=1, required for correct impedance conversion | +| `R3_1` | number | ACPF-critical | Resistance of winding 3-1 pair (3-winding only); enters the Y-bus for the star-bus equivalent 3-1 leg in ACPF | +| `X3_1` | number | Informational | Reactance of winding 3-1 pair (3-winding only). Same rationale as X2_3: star-equivalent conversion preserves DCPF accuracy; raw field absent only in MATPOWER PPC pathway. | +| `SBASE3_1` | number | ACPF-critical | MVA base for winding 3-1 impedance; determines per-unit base for R3_1/X3_1 when CZ=1, required for correct impedance conversion | +| `VMSTAR` | number | ACPF-critical | Star-bus voltage magnitude initial value for 3-winding transformers; provides the initial voltage guess for the synthetic star bus in ACPF iteration | +| `ANSTAR` | number | ACPF-critical | Star-bus voltage angle initial value for 3-winding transformers; provides the initial angle guess for the synthetic star bus in ACPF iteration | +| `WINDV1` | number | DCPF-critical | Winding 1 off-nominal turns ratio or voltage; scales effective reactance in DCPF and affects voltage transformation ratio in ACPF | +| `NOMV1` | number | ACPF-critical | Winding 1 nominal voltage in kV; needed with CW modes for tap ratio interpretation, affects how WINDV1 is converted to per-unit in ACPF | +| `ANG1` | number | DCPF-critical | Winding 1 phase shift angle in degrees; directly enters the DCPF formulation for phase-shifting transformers, controlling real power flow direction | +| `RATA1` | number | ACPF-critical | Winding 1 normal rating in MVA; preservation-critical field used for transformer capacity assessment and OPF thermal constraints, required for complete transformer model specification in ACPF | +| `RATB1` | number | Informational | Winding 1 emergency rating in MVA; post-solution thermal constraint, not a power flow variable | +| `RATC1` | number | Informational | Winding 1 short-term rating in MVA; post-solution thermal constraint, not a power flow variable | +| `COD1` | integer | ACPF-critical | Winding 1 tap changer control mode code; determines whether and how the tap ratio is adjusted during ACPF solution for voltage or flow regulation | +| `CONT1` | integer | ACPF-critical | Winding 1 controlled bus number; identifies the target bus for voltage regulation by the tap changer in ACPF | +| `RMA1` | number | ACPF-critical | Maximum tap ratio or angle limit for winding 1; upper bound on tap changer adjustment range in ACPF | +| `RMI1` | number | ACPF-critical | Minimum tap ratio or angle limit for winding 1; lower bound on tap changer adjustment range in ACPF | +| `VMA1` | number | ACPF-critical | Maximum voltage or flow target for winding 1 control; upper control target for tap changer regulation in ACPF | +| `VMI1` | number | ACPF-critical | Minimum voltage or flow target for winding 1 control; lower control target for tap changer regulation in ACPF | +| `NTP1` | integer | ACPF-critical | Number of tap positions for winding 1; defines the discrete tap step resolution for the tap changer in ACPF | +| `TAB1` | integer | Informational | Impedance correction table number for winding 1; references a piecewise-linear correction curve, secondary adjustment that is often unused | +| `CR1` | number | Informational | Load drop compensation resistance for winding 1; secondary voltage regulation refinement typically at default values, not a primary power flow parameter | +| `CX1` | number | Informational | Load drop compensation reactance for winding 1; secondary voltage regulation refinement typically at default values, not a primary power flow parameter | +| `CNXA1` | integer | Informational | Connection angle for winding 1 wye-delta transformers; advanced winding connection parameter, does not enter the standard power flow equations | +| `WINDV2` | number | ACPF-critical | Winding 2 off-nominal turns ratio or voltage; affects voltage transformation in ACPF (usually 1.0 for standard 2-winding transformers) | +| `NOMV2` | number | ACPF-critical | Winding 2 nominal voltage in kV; needed with CW modes for winding 2 tap ratio interpretation in ACPF | +| `ANG2` | number | ACPF-critical | Winding 2 phase shift angle in degrees; affects phase shifting on the secondary winding in ACPF | +| `RATA2` | number | ACPF-critical | Winding 2 normal rating in MVA; preservation-critical field used for transformer capacity assessment and OPF thermal constraints, required for complete transformer model specification in ACPF | +| `RATB2` | number | Informational | Winding 2 emergency rating in MVA; post-solution thermal constraint, not a power flow variable | +| `RATC2` | number | Informational | Winding 2 short-term rating in MVA; post-solution thermal constraint, not a power flow variable | +| `COD2` | integer | ACPF-critical | Winding 2 tap changer control mode code; determines how the winding 2 tap ratio is adjusted in ACPF | +| `CONT2` | integer | ACPF-critical | Winding 2 controlled bus number; identifies the target bus for winding 2 voltage regulation in ACPF | +| `RMA2` | number | ACPF-critical | Maximum tap ratio or angle limit for winding 2; upper bound on winding 2 tap changer range in ACPF | +| `RMI2` | number | ACPF-critical | Minimum tap ratio or angle limit for winding 2; lower bound on winding 2 tap changer range in ACPF | +| `VMA2` | number | ACPF-critical | Maximum voltage or flow target for winding 2 control; upper control target for winding 2 regulation in ACPF | +| `VMI2` | number | ACPF-critical | Minimum voltage or flow target for winding 2 control; lower control target for winding 2 regulation in ACPF | +| `NTP2` | integer | ACPF-critical | Number of tap positions for winding 2; defines the discrete tap step resolution for winding 2 in ACPF | +| `TAB2` | integer | Informational | Impedance correction table number for winding 2; references a piecewise-linear correction curve, secondary parameter | +| `CR2` | number | Informational | Load drop compensation resistance for winding 2; secondary voltage regulation refinement, not a primary power flow parameter | +| `CX2` | number | Informational | Load drop compensation reactance for winding 2; secondary voltage regulation refinement, not a primary power flow parameter | +| `CNXA2` | integer | Informational | Connection angle for winding 2; advanced winding connection parameter, does not enter the standard power flow equations | +| `WINDV3` | number | ACPF-critical | Winding 3 off-nominal turns ratio or voltage; affects voltage transformation for the tertiary winding in 3-winding transformers in ACPF | +| `NOMV3` | number | ACPF-critical | Winding 3 nominal voltage in kV; needed with CW modes for winding 3 tap ratio interpretation in ACPF | +| `ANG3` | number | ACPF-critical | Winding 3 phase shift angle in degrees; affects phase shifting on the tertiary winding in ACPF | +| `RATA3` | number | ACPF-critical | Winding 3 normal rating in MVA; preservation-critical field used for transformer capacity assessment and OPF thermal constraints, required for complete transformer model specification in ACPF | +| `RATB3` | number | Informational | Winding 3 emergency rating in MVA; post-solution thermal constraint, not a power flow variable | +| `RATC3` | number | Informational | Winding 3 short-term rating in MVA; post-solution thermal constraint, not a power flow variable | +| `COD3` | integer | ACPF-critical | Winding 3 tap changer control mode code; determines how the winding 3 tap ratio is adjusted in ACPF | +| `CONT3` | integer | ACPF-critical | Winding 3 controlled bus number; identifies the target bus for winding 3 voltage regulation in ACPF | +| `RMA3` | number | ACPF-critical | Maximum tap ratio or angle limit for winding 3; upper bound on winding 3 tap changer range in ACPF | +| `RMI3` | number | ACPF-critical | Minimum tap ratio or angle limit for winding 3; lower bound on winding 3 tap changer range in ACPF | +| `VMA3` | number | ACPF-critical | Maximum voltage or flow target for winding 3 control; upper control target for winding 3 regulation in ACPF | +| `VMI3` | number | ACPF-critical | Minimum voltage or flow target for winding 3 control; lower control target for winding 3 regulation in ACPF | +| `NTP3` | integer | ACPF-critical | Number of tap positions for winding 3; defines the discrete tap step resolution for winding 3 in ACPF | +| `TAB3` | integer | Informational | Impedance correction table number for winding 3; references a piecewise-linear correction curve, secondary parameter | +| `CR3` | number | Informational | Load drop compensation resistance for winding 3; secondary voltage regulation refinement, not a primary power flow parameter | +| `CX3` | number | Informational | Load drop compensation reactance for winding 3; secondary voltage regulation refinement, not a primary power flow parameter | +| `CNXA3` | integer | Informational | Connection angle for winding 3; advanced winding connection parameter, does not enter the standard power flow equations | + +CW, CZ, and CM codes are classified as ACPF-critical because they determine how transformer impedance and tap ratio values are interpreted, even though the codes themselves are not numerical parameters in the power flow equations. Normal winding ratings (RATA1, RATA2, RATA3) are ACPF-critical because they are preservation-critical fields required for complete transformer model specification; emergency and short-term ratings (RATB1-3, RATC1-3) are Informational as they are secondary thermal limits not flagged as preservation-critical. + +## Area + +**Intermediate format table:** `area` +**Record-type tier (from mapping guide):** Tier 2 -- Required for ACPF accuracy +**Total fields:** 5 +**Tier breakdown:** 0 DCPF-critical, 3 ACPF-critical, 2 Informational, 0 Discardable + +| Field | Type | Tier | Rationale | +|-------|------|------|-----------| +| `I` | integer | Informational | Area number serves as the primary key for organizational grouping; the area ID itself is a lookup key for interchange data, not a direct variable in the power flow equations | +| `ISW` | integer | ACPF-critical | Area slack bus number determining which generator absorbs area interchange mismatch; controls generation redispatch in area interchange control during ACPF solution | +| `PDES` | number | ACPF-critical | Desired net area interchange in MW; the target power export/import for area interchange control in ACPF, directly affecting generation redispatch | +| `PTOL` | number | ACPF-critical | Area interchange tolerance in MW; convergence criterion for area interchange control determining when the ACPF solution has satisfied the interchange target | +| `ARNAME` | string | Informational | Area name for human-readable identification; descriptive label with no effect on power flow computation | + +## Two-Terminal DC + +**Intermediate format table:** `two_terminal_dc` +**Record-type tier (from mapping guide):** Tier 2 -- Required for ACPF accuracy +**Total fields:** 46 +**Tier breakdown:** 0 DCPF-critical, 46 ACPF-critical, 0 Informational, 0 Discardable + +| Field | Type | Tier | Rationale | +|-------|------|------|-----------| +| `NAME` | string | ACPF-critical | HVDC line name serving as the primary key; required to identify and instantiate this DC link in the power flow model | +| `MDC` | integer | ACPF-critical | Control mode (0=blocked, 1=current, 2=power) determining how the DC line controls are modeled in ACPF; affects converter firing angle and power flow interaction | +| `RDC` | number | ACPF-critical | DC line resistance in ohms; determines DC power losses between rectifier and inverter, affecting the AC-DC power balance in ACPF | +| `SETVL` | number | ACPF-critical | Current or power demand setpoint; the operating target for the DC line control system, directly affecting AC-side power injections in ACPF | +| `VSCHD` | number | ACPF-critical | Scheduled DC voltage in kV; determines the operating voltage of the DC line, affecting converter reactive power consumption in ACPF | +| `VCMOD` | number | ACPF-critical | Mode switch DC voltage; threshold for control mode switching in the DC line control logic during ACPF solution | +| `RCOMP` | number | ACPF-critical | Compounding resistance; adjusts the voltage regulation reference point for the DC line controller in ACPF | +| `DELTI` | number | ACPF-critical | Inverter firing angle margin in degrees; safety margin for commutation failure prevention, affecting inverter reactive power in ACPF | +| `METER` | string | ACPF-critical | Metered end indicator (R=rectifier, I=inverter); determines which converter's power is controlled to the setpoint in the ACPF DC line model | +| `DCVMIN` | number | ACPF-critical | Minimum DC voltage in per-unit; lower limit on DC voltage for converter control logic in ACPF | +| `CCCITMX` | integer | ACPF-critical | Maximum converter control iterations; convergence parameter for the DC converter control loop within ACPF | +| `CCCACC` | number | ACPF-critical | Converter control acceleration factor; convergence tuning parameter for the DC converter control loop in ACPF | +| `IPR` | integer | ACPF-critical | Rectifier AC bus number; identifies the AC bus where the rectifier injects/absorbs power, essential for ACPF network connectivity | +| `NBR` | integer | ACPF-critical | Number of rectifier bridges; determines the rectifier transformer configuration and commutation voltage in the ACPF converter model | +| `ANMXR` | number | ACPF-critical | Maximum rectifier firing angle in degrees; upper limit on rectifier control range affecting reactive power consumption in ACPF | +| `ANMNR` | number | ACPF-critical | Minimum rectifier firing angle in degrees; lower limit on rectifier control range for the ACPF converter model | +| `RCR` | number | ACPF-critical | Rectifier commutating resistance; part of the rectifier transformer impedance model affecting commutation overlap in ACPF | +| `XCR` | number | ACPF-critical | Rectifier commutating reactance; dominant commutation impedance determining overlap angle and reactive power in the ACPF converter model | +| `EBASR` | number | ACPF-critical | Rectifier primary-side base voltage in kV; voltage base for the rectifier converter transformer, needed for correct per-unit conversion in ACPF | +| `TRR` | number | ACPF-critical | Rectifier transformer ratio; off-nominal turns ratio for the rectifier converter transformer in ACPF | +| `TAPR` | number | ACPF-critical | Rectifier tap setting; current tap position of the rectifier converter transformer, affecting DC voltage in ACPF | +| `TMXR` | number | ACPF-critical | Maximum rectifier tap; upper limit on rectifier transformer tap adjustment range in ACPF | +| `TMNR` | number | ACPF-critical | Minimum rectifier tap; lower limit on rectifier transformer tap adjustment range in ACPF | +| `STPR` | number | ACPF-critical | Rectifier tap step size; discrete step increment for rectifier transformer tap changes in ACPF | +| `ICR` | integer | ACPF-critical | Rectifier firing angle control bus; AC bus used for converter control feedback in the ACPF DC line model | +| `IFR` | integer | ACPF-critical | Rectifier commutating bus (from-side); defines the from-bus of the rectifier commutating branch in ACPF | +| `ITR` | integer | ACPF-critical | Rectifier commutating bus (to-side); defines the to-bus of the rectifier commutating branch in ACPF | +| `IDR` | string | ACPF-critical | Rectifier circuit identifier; distinguishes parallel converter transformer circuits in the ACPF model | +| `XCAPR` | number | ACPF-critical | Rectifier capacitor reactance; capacitive compensation reactance at the rectifier terminal in ACPF | +| `IPI` | integer | ACPF-critical | Inverter AC bus number; identifies the AC bus where the inverter injects/absorbs power, essential for ACPF network connectivity | +| `NBI` | integer | ACPF-critical | Number of inverter bridges; determines the inverter transformer configuration and commutation voltage in ACPF | +| `ANMXI` | number | ACPF-critical | Maximum inverter firing angle in degrees; upper limit on inverter control range in the ACPF converter model | +| `ANMNI` | number | ACPF-critical | Minimum inverter firing angle in degrees; lower limit on inverter extinction angle for commutation safety in ACPF | +| `RCI` | number | ACPF-critical | Inverter commutating resistance; part of the inverter transformer impedance model in ACPF | +| `XCI` | number | ACPF-critical | Inverter commutating reactance; dominant commutation impedance for the inverter side in ACPF | +| `EBASI` | number | ACPF-critical | Inverter primary-side base voltage in kV; voltage base for the inverter converter transformer in ACPF | +| `TRI` | number | ACPF-critical | Inverter transformer ratio; off-nominal turns ratio for the inverter converter transformer in ACPF | +| `TAPI` | number | ACPF-critical | Inverter tap setting; current tap position of the inverter converter transformer in ACPF | +| `TMXI` | number | ACPF-critical | Maximum inverter tap; upper limit on inverter transformer tap range in ACPF | +| `TMNI` | number | ACPF-critical | Minimum inverter tap; lower limit on inverter transformer tap range in ACPF | +| `STPI` | number | ACPF-critical | Inverter tap step size; discrete step increment for inverter transformer tap changes in ACPF | +| `ICI` | integer | ACPF-critical | Inverter firing angle control bus; AC bus used for inverter control feedback in ACPF | +| `IFI` | integer | ACPF-critical | Inverter commutating bus (from-side); defines the from-bus of the inverter commutating branch in ACPF | +| `ITI` | integer | ACPF-critical | Inverter commutating bus (to-side); defines the to-bus of the inverter commutating branch in ACPF | +| `IDI` | string | ACPF-critical | Inverter circuit identifier; distinguishes parallel inverter transformer circuits in the ACPF model | +| `XCAPI` | number | ACPF-critical | Inverter capacitor reactance; capacitive compensation reactance at the inverter terminal in ACPF | + +All two-terminal DC fields are classified as ACPF-critical because HVDC converters interact with the AC system through reactive power consumption and active power injection, affecting ACPF convergence but not the DCPF formulation which does not model DC links. + +## VSC DC + +**Intermediate format table:** `vsc_dc` +**Record-type tier (from mapping guide):** Tier 2 -- Required for ACPF accuracy +**Total fields:** 41 +**Tier breakdown:** 0 DCPF-critical, 41 ACPF-critical, 0 Informational, 0 Discardable + +| Field | Type | Tier | Rationale | +|-------|------|------|-----------| +| `NAME` | string | ACPF-critical | VSC DC line name serving as the primary key; required to identify and instantiate this VSC link in the ACPF power flow model | +| `MDC` | integer | ACPF-critical | Control mode determining how the VSC link operates in ACPF; affects active and reactive power control at both converter terminals | +| `RDC` | number | ACPF-critical | DC line resistance in ohms; determines DC power losses between converters, affecting the AC-side power balance in ACPF | +| `O1` | integer | ACPF-critical | Owner number 1 for the VSC link; in context of a Tier 2 ACPF device record, ownership is part of the complete device specification needed for model instantiation | +| `F1` | number | ACPF-critical | Ownership fraction for owner 1; part of the complete VSC device record needed for ACPF model instantiation | +| `O2` | integer | ACPF-critical | Owner number 2; part of the complete VSC device record specification | +| `F2` | number | ACPF-critical | Ownership fraction for owner 2; part of the complete VSC device record specification | +| `O3` | integer | ACPF-critical | Owner number 3; part of the complete VSC device record specification | +| `F3` | number | ACPF-critical | Ownership fraction for owner 3; part of the complete VSC device record specification | +| `O4` | integer | ACPF-critical | Owner number 4; part of the complete VSC device record specification | +| `F4` | number | ACPF-critical | Ownership fraction for owner 4; part of the complete VSC device record specification | +| `IBUS1` | integer | ACPF-critical | Converter 1 AC bus number; identifies the AC bus where converter 1 connects, essential for ACPF network topology | +| `TYPE1` | integer | ACPF-critical | Converter 1 type code; determines the converter operating characteristics in the ACPF model | +| `MODE1` | integer | ACPF-critical | Converter 1 control mode; specifies whether converter 1 controls active power, DC voltage, or other quantities in ACPF | +| `DCSET1` | number | ACPF-critical | Converter 1 DC setpoint; operating target value for the controlled quantity at converter 1 in ACPF | +| `ACSET1` | number | ACPF-critical | Converter 1 AC setpoint; AC-side voltage or reactive power target for converter 1 in ACPF | +| `ALOSS1` | number | ACPF-critical | Converter 1 loss coefficient A; constant term in the converter loss model affecting AC-side power balance in ACPF | +| `BLOSS1` | number | ACPF-critical | Converter 1 loss coefficient B; current-proportional loss term in the converter loss model for ACPF | +| `MINLOSS1` | number | ACPF-critical | Converter 1 minimum loss; floor on converter losses in the ACPF loss model | +| `SMAX1` | number | ACPF-critical | Converter 1 MVA rating; maximum apparent power capacity limiting converter operation in ACPF | +| `IMAX1` | number | ACPF-critical | Converter 1 current rating in amperes; maximum current capacity limiting converter operation in ACPF | +| `PWF1` | number | ACPF-critical | Converter 1 power weighting factor; determines power sharing between converters in the ACPF DC link model | +| `MAXQ1` | number | ACPF-critical | Converter 1 maximum reactive power in MVAR; upper Q limit for converter 1 affecting ACPF reactive power balance | +| `MINQ1` | number | ACPF-critical | Converter 1 minimum reactive power in MVAR; lower Q limit for converter 1 in ACPF | +| `REMOT1` | integer | ACPF-critical | Converter 1 remote bus for voltage control; identifies the bus whose voltage converter 1 regulates in ACPF | +| `RMPCT1` | number | ACPF-critical | Converter 1 MVAR percent for remote regulation; fraction of reactive range used for remote voltage control in ACPF | +| `IBUS2` | integer | ACPF-critical | Converter 2 AC bus number; identifies the AC bus where converter 2 connects, essential for ACPF network topology | +| `TYPE2` | integer | ACPF-critical | Converter 2 type code; determines converter 2 operating characteristics in the ACPF model | +| `MODE2` | integer | ACPF-critical | Converter 2 control mode; specifies whether converter 2 controls active power, DC voltage, or other quantities in ACPF | +| `DCSET2` | number | ACPF-critical | Converter 2 DC setpoint; operating target for the controlled quantity at converter 2 in ACPF | +| `ACSET2` | number | ACPF-critical | Converter 2 AC setpoint; AC-side voltage or reactive power target for converter 2 in ACPF | +| `ALOSS2` | number | ACPF-critical | Converter 2 loss coefficient A; constant term in converter 2 loss model for ACPF | +| `BLOSS2` | number | ACPF-critical | Converter 2 loss coefficient B; current-proportional loss term in converter 2 loss model for ACPF | +| `MINLOSS2` | number | ACPF-critical | Converter 2 minimum loss; floor on converter 2 losses in ACPF | +| `SMAX2` | number | ACPF-critical | Converter 2 MVA rating; maximum apparent power capacity for converter 2 in ACPF | +| `IMAX2` | number | ACPF-critical | Converter 2 current rating in amperes; maximum current capacity for converter 2 in ACPF | +| `PWF2` | number | ACPF-critical | Converter 2 power weighting factor; determines power sharing in the ACPF DC link model | +| `MAXQ2` | number | ACPF-critical | Converter 2 maximum reactive power in MVAR; upper Q limit for converter 2 in ACPF | +| `MINQ2` | number | ACPF-critical | Converter 2 minimum reactive power in MVAR; lower Q limit for converter 2 in ACPF | +| `REMOT2` | integer | ACPF-critical | Converter 2 remote bus for voltage control; identifies the bus whose voltage converter 2 regulates in ACPF | +| `RMPCT2` | number | ACPF-critical | Converter 2 MVAR percent for remote regulation; fraction of reactive range used for remote voltage control in ACPF | + +## Impedance Correction + +**Intermediate format table:** `impedance_correction` +**Record-type tier (from mapping guide):** Tier 2 -- Required for ACPF accuracy +**Total fields:** 23 +**Tier breakdown:** 0 DCPF-critical, 23 ACPF-critical, 0 Informational, 0 Discardable + +| Field | Type | Tier | Rationale | +|-------|------|------|-----------| +| `T` | integer | ACPF-critical | Correction table number serving as the primary key; referenced by transformer TAB fields to apply piecewise-linear impedance corrections in ACPF | +| `T1` | number | ACPF-critical | Tap ratio or angle breakpoint 1 in the piecewise-linear impedance correction curve; defines where the correction factor changes in ACPF | +| `F1` | number | ACPF-critical | Impedance correction factor at breakpoint 1; multiplier applied to transformer impedance at this tap position in ACPF | +| `T2` | number | ACPF-critical | Tap ratio or angle breakpoint 2; second point in the piecewise-linear impedance correction curve for ACPF | +| `F2` | number | ACPF-critical | Impedance correction factor at breakpoint 2; multiplier for transformer impedance in ACPF | +| `T3` | number | ACPF-critical | Tap ratio or angle breakpoint 3; third point in the piecewise-linear correction curve for ACPF | +| `F3` | number | ACPF-critical | Impedance correction factor at breakpoint 3; multiplier for transformer impedance in ACPF | +| `T4` | number | ACPF-critical | Tap ratio or angle breakpoint 4; fourth point in the correction curve for ACPF | +| `F4` | number | ACPF-critical | Impedance correction factor at breakpoint 4; multiplier for transformer impedance in ACPF | +| `T5` | number | ACPF-critical | Tap ratio or angle breakpoint 5; fifth point in the correction curve for ACPF | +| `F5` | number | ACPF-critical | Impedance correction factor at breakpoint 5; multiplier for transformer impedance in ACPF | +| `T6` | number | ACPF-critical | Tap ratio or angle breakpoint 6; sixth point in the correction curve for ACPF | +| `F6` | number | ACPF-critical | Impedance correction factor at breakpoint 6; multiplier for transformer impedance in ACPF | +| `T7` | number | ACPF-critical | Tap ratio or angle breakpoint 7; seventh point in the correction curve for ACPF | +| `F7` | number | ACPF-critical | Impedance correction factor at breakpoint 7; multiplier for transformer impedance in ACPF | +| `T8` | number | ACPF-critical | Tap ratio or angle breakpoint 8; eighth point in the correction curve for ACPF | +| `F8` | number | ACPF-critical | Impedance correction factor at breakpoint 8; multiplier for transformer impedance in ACPF | +| `T9` | number | ACPF-critical | Tap ratio or angle breakpoint 9; ninth point in the correction curve for ACPF | +| `F9` | number | ACPF-critical | Impedance correction factor at breakpoint 9; multiplier for transformer impedance in ACPF | +| `T10` | number | ACPF-critical | Tap ratio or angle breakpoint 10; tenth point in the correction curve for ACPF | +| `F10` | number | ACPF-critical | Impedance correction factor at breakpoint 10; multiplier for transformer impedance in ACPF | +| `T11` | number | ACPF-critical | Tap ratio or angle breakpoint 11; eleventh point in the correction curve for ACPF | +| `F11` | number | ACPF-critical | Impedance correction factor at breakpoint 11; multiplier for transformer impedance in ACPF | + +## Multi-Terminal DC + +**Intermediate format table:** `multi_terminal_dc` +**Record-type tier (from mapping guide):** Tier 2 -- Required for ACPF accuracy +**Total fields:** 8 +**Tier breakdown:** 0 DCPF-critical, 8 ACPF-critical, 0 Informational, 0 Discardable + +| Field | Type | Tier | Rationale | +|-------|------|------|-----------| +| `NAME` | string | ACPF-critical | Multi-terminal DC system name serving as the primary key; required to identify and instantiate this MTDC system in the ACPF model | +| `NCONV` | integer | ACPF-critical | Number of AC converters in the MTDC system; defines the system topology and determines how many converter models are created in ACPF | +| `NDCBS` | integer | ACPF-critical | Number of DC buses in the MTDC system; defines the DC-side network topology for the ACPF MTDC model | +| `NDCLN` | integer | ACPF-critical | Number of DC links in the MTDC system; defines the DC-side branch connectivity for ACPF | +| `MDC` | integer | ACPF-critical | Control mode for the MTDC system; determines the overall operating strategy affecting AC-side power injections in ACPF | +| `VCONV` | integer | ACPF-critical | DC voltage controlling converter number; identifies which converter maintains the DC voltage reference in the ACPF MTDC model | +| `VCMOD` | number | ACPF-critical | Mode switch DC voltage threshold; voltage level at which the MTDC control mode transitions during ACPF solution | +| `VCONVN` | integer | ACPF-critical | New voltage controlling converter after mode switch; backup converter for DC voltage control in the ACPF model | + +## Multi-Section Line + +**Intermediate format table:** `multi_section_line` +**Record-type tier (from mapping guide):** Tier 2 -- Required for ACPF accuracy +**Total fields:** 13 +**Tier breakdown:** 0 DCPF-critical, 12 ACPF-critical, 1 Informational, 0 Discardable + +| Field | Type | Tier | Rationale | +|-------|------|------|-----------| +| `I` | integer | ACPF-critical | From-bus number of the multi-section line; identifies one endpoint of the grouped transmission corridor, essential for matching branches to multi-section groupings in ACPF | +| `J` | integer | ACPF-critical | To-bus number of the multi-section line; identifies the other endpoint of the grouped corridor for branch aggregation in ACPF | +| `ID` | string | ACPF-critical | Line identifier distinguishing parallel multi-section groupings; required to correctly associate branch sections with the correct multi-section line in ACPF | +| `MET` | integer | Informational | Metered end flag (1=from-bus, 2=to-bus); determines which end is used for loss allocation accounting, does not affect the power flow solution | +| `DUM1` | integer | ACPF-critical | Intermediate bus 1 along the multi-section line; defines the internal topology of the multi-section corridor by specifying section boundaries in ACPF | +| `DUM2` | integer | ACPF-critical | Intermediate bus 2; second section boundary bus in the multi-section line topology for ACPF | +| `DUM3` | integer | ACPF-critical | Intermediate bus 3; third section boundary bus in the multi-section line for ACPF | +| `DUM4` | integer | ACPF-critical | Intermediate bus 4; fourth section boundary in the multi-section line topology for ACPF | +| `DUM5` | integer | ACPF-critical | Intermediate bus 5; fifth section boundary in the multi-section line for ACPF | +| `DUM6` | integer | ACPF-critical | Intermediate bus 6; sixth section boundary in the multi-section line for ACPF | +| `DUM7` | integer | ACPF-critical | Intermediate bus 7; seventh section boundary in the multi-section line for ACPF | +| `DUM8` | integer | ACPF-critical | Intermediate bus 8; eighth section boundary in the multi-section line for ACPF | +| `DUM9` | integer | ACPF-critical | Intermediate bus 9; ninth section boundary in the multi-section line for ACPF | + +## Zone + +**Intermediate format table:** `zone` +**Record-type tier (from mapping guide):** Tier 3 -- Organizational / metadata +**Total fields:** 2 +**Tier breakdown:** 0 DCPF-critical, 0 ACPF-critical, 2 Informational, 0 Discardable + +| Field | Type | Tier | Rationale | +|-------|------|------|-----------| +| `I` | integer | Informational | Zone number serving as the primary key for geographic or administrative grouping; organizational identifier with no electrical effect on power flow computation | +| `ZONAME` | string | Informational | Zone name for human-readable identification; descriptive label for reporting purposes with no impact on power flow equations | + +All Zone fields are Informational because Zone is a Tier 3 record type containing organizational metadata with no direct electrical effect on power flow. + +## Interarea Transfer + +**Intermediate format table:** `interarea_transfer` +**Record-type tier (from mapping guide):** Tier 3 -- Organizational / metadata +**Total fields:** 4 +**Tier breakdown:** 0 DCPF-critical, 0 ACPF-critical, 4 Informational, 0 Discardable + +| Field | Type | Tier | Rationale | +|-------|------|------|-----------| +| `ARFROM` | integer | Informational | From-area number for the scheduled transfer; administrative reference for interchange tracking with no direct effect on the power flow equations | +| `ARTO` | integer | Informational | To-area number for the scheduled transfer; administrative reference for interchange accounting, not a power flow variable | +| `TRID` | string | Informational | Transfer identifier distinguishing multiple transfers between the same area pair; administrative key with no electrical significance | +| `PTRAN` | number | Informational | Scheduled transfer amount in MW; reporting value for interchange accounting that does not directly enter the power flow solution as a variable or constraint | + +All Interarea Transfer fields are Informational because Interarea Transfer is a Tier 3 record type. Interchange scheduling data is used for monitoring and reporting, not as a direct input to the power flow equations. + +## Owner + +**Intermediate format table:** `owner` +**Record-type tier (from mapping guide):** Tier 3 -- Organizational / metadata +**Total fields:** 2 +**Tier breakdown:** 0 DCPF-critical, 0 ACPF-critical, 2 Informational, 0 Discardable + +| Field | Type | Tier | Rationale | +|-------|------|------|-----------| +| `I` | integer | Informational | Owner number serving as the primary key for ownership entities; administrative identifier for cost allocation with no electrical effect on power flow | +| `OWNAME` | string | Informational | Owner name for human-readable identification; descriptive label for ownership tracking with no impact on power flow computation | + +All Owner fields are Informational because Owner is a Tier 3 record type containing administrative metadata with no direct electrical effect on power flow. + +## FACTS + +**Intermediate format table:** `facts` +**Record-type tier (from mapping guide):** Tier 2 -- Required for ACPF accuracy +**Total fields:** 14 +**Tier breakdown:** 0 DCPF-critical, 14 ACPF-critical, 0 Informational, 0 Discardable + +| Field | Type | Tier | Rationale | +|-------|------|------|-----------| +| `NAME` | string | ACPF-critical | FACTS device name serving as the primary key; required to identify and instantiate this device in the ACPF power flow model | +| `I` | integer | ACPF-critical | Sending end bus number; identifies the AC bus where the FACTS device connects, essential for ACPF network topology | +| `J` | integer | ACPF-critical | Terminal bus number (0=shunt device); determines whether the FACTS device operates as a shunt or series element in the ACPF Y-bus | +| `MODE` | integer | ACPF-critical | FACTS control mode determining the device operating behavior (e.g., voltage regulation, power flow control) in ACPF | +| `SET1` | number | ACPF-critical | Control setpoint 1; primary operating target for the FACTS device control logic in ACPF, interpretation depends on MODE | +| `SET2` | number | ACPF-critical | Control setpoint 2; secondary operating target for the FACTS device in ACPF, interpretation depends on MODE | +| `VSREF` | number | ACPF-critical | Series voltage reference in per-unit; reference voltage for the series element of the FACTS device in ACPF | +| `REMOT` | integer | ACPF-critical | Remote bus for voltage control; identifies the bus whose voltage the FACTS device regulates in ACPF | +| `MESSION` | number | ACPF-critical | Sending end impedance; impedance parameter of the FACTS device at the sending end, entering the ACPF Y-bus model | +| `LINX` | number | ACPF-critical | Series reactance in per-unit; the series reactive impedance of the FACTS device entering the ACPF Y-bus | +| `RMPCT` | number | ACPF-critical | MVAR percent for remote regulation; fraction of reactive capability allocated to remote bus voltage control in ACPF | +| `OWNER` | integer | ACPF-critical | Owner number for the FACTS device; in context of a Tier 2 ACPF device record, part of the complete device specification | +| `SET3` | number | ACPF-critical | Control setpoint 3; tertiary operating parameter for advanced FACTS control modes in ACPF | +| `SET4` | number | ACPF-critical | Control setpoint 4; quaternary operating parameter for advanced FACTS control modes in ACPF | + +All FACTS fields are ACPF-critical because FACTS devices (SVCs, STATCOMs, TCSCs, UPFCs) provide dynamic reactive compensation and power flow control that directly affects ACPF convergence and voltage profiles but is not modeled in DCPF. + +## Switched Shunt + +**Intermediate format table:** `switched_shunt` +**Record-type tier (from mapping guide):** Tier 2 -- Required for ACPF accuracy +**Total fields:** 26 +**Tier breakdown:** 0 DCPF-critical, 23 ACPF-critical, 3 Informational, 0 Discardable + +| Field | Type | Tier | Rationale | +|-------|------|------|-----------| +| `I` | integer | ACPF-critical | Bus number where the switched shunt is connected; determines placement of the switched admittance in the Y-bus for ACPF voltage regulation | +| `MODSW` | integer | ACPF-critical | Control mode (0=fixed, 1=discrete, 2=continuous) determining how the switched shunt adjusts its susceptance during ACPF solution for voltage control | +| `ADJM` | integer | Informational | Adjustment method flag (0=steps, 1=direct); secondary control parameter indicating how shunt blocks are selected, typically at default | +| `STAT` | integer | ACPF-critical | Switched shunt status (1=in-service, 0=out-of-service); determines whether this device's susceptance is included in the ACPF Y-bus | +| `VSWHI` | number | ACPF-critical | Upper voltage control limit in per-unit; the voltage threshold above which the switched shunt reduces capacitive output in ACPF | +| `VSWLO` | number | ACPF-critical | Lower voltage control limit in per-unit; the voltage threshold below which the switched shunt increases capacitive output in ACPF | +| `SWREM` | integer | ACPF-critical | Remote bus for voltage control (0=local); identifies the bus whose voltage triggers switching actions in ACPF | +| `RMPCT` | number | Informational | MVAR percent for remote voltage regulation; fraction of reactive range allocated to remote voltage control, secondary tuning parameter | +| `RMIDNT` | string | Informational | Shunt identifier name; human-readable label for the switched shunt device with no effect on power flow computation | +| `BINIT` | number | ACPF-critical | Initial susceptance in MVAR; the starting reactive compensation value for the switched shunt at the beginning of ACPF iteration | +| `N1` | integer | ACPF-critical | Number of steps in shunt block 1; defines the discrete switching capacity of the first compensation block in ACPF | +| `B1` | number | ACPF-critical | Susceptance per step in block 1 in MVAR; the reactive power increment for each switching step of block 1 in ACPF | +| `N2` | integer | ACPF-critical | Number of steps in block 2; second discrete compensation block capacity for ACPF | +| `B2` | number | ACPF-critical | Susceptance per step in block 2 in MVAR; reactive increment for block 2 in ACPF | +| `N3` | integer | ACPF-critical | Number of steps in block 3; third discrete compensation block for ACPF | +| `B3` | number | ACPF-critical | Susceptance per step in block 3 in MVAR; reactive increment for block 3 in ACPF | +| `N4` | integer | ACPF-critical | Number of steps in block 4; fourth discrete compensation block for ACPF | +| `B4` | number | ACPF-critical | Susceptance per step in block 4 in MVAR; reactive increment for block 4 in ACPF | +| `N5` | integer | ACPF-critical | Number of steps in block 5; fifth discrete compensation block for ACPF | +| `B5` | number | ACPF-critical | Susceptance per step in block 5 in MVAR; reactive increment for block 5 in ACPF | +| `N6` | integer | ACPF-critical | Number of steps in block 6; sixth discrete compensation block for ACPF | +| `B6` | number | ACPF-critical | Susceptance per step in block 6 in MVAR; reactive increment for block 6 in ACPF | +| `N7` | integer | ACPF-critical | Number of steps in block 7; seventh discrete compensation block for ACPF | +| `B7` | number | ACPF-critical | Susceptance per step in block 7 in MVAR; reactive increment for block 7 in ACPF | +| `N8` | integer | ACPF-critical | Number of steps in block 8; eighth discrete compensation block for ACPF | +| `B8` | number | ACPF-critical | Susceptance per step in block 8 in MVAR; reactive increment for block 8 in ACPF | + +ADJM, RMPCT, and RMIDNT are classified as Informational because ADJM is a secondary control method flag that does not affect the power flow equations, RMPCT is a tuning parameter for reactive allocation, and RMIDNT is a descriptive label. + +## Tier Assignment Rules + +**DCPF-critical assignment criteria (all must apply):** +- Field directly enters the DC power flow B-matrix, power injection vector, or topology adjacency structure. +- OR field determines whether a component is in-service (status fields that control topology). +- OR field is a transformer tap magnitude or phase-shifting angle (these scale effective reactance in DCPF). +- AND the record type is Tier 1 or Tier 2 (Tier 3 record types cannot contribute DCPF-critical fields). + +**ACPF-critical assignment criteria (all must apply):** +- Field enters the AC power flow Y-bus matrix, voltage/reactive power constraints, or control mode logic, BUT is not needed for DCPF. +- OR field provides reactive power limits, voltage setpoints, tap ratio limits, or switching control parameters needed for ACPF convergence. +- OR field determines the per-unit base or convention mode (CW, CZ, CM) that affects interpretation of ACPF parameters. +- AND the record type is Tier 1 or Tier 2. + +**Informational assignment criteria:** +- Field provides context, identification, or operational metadata that does not enter any power flow equation or control logic. +- OR field is in a Tier 3 record type (all fields in Tier 3 record types are Informational by definition, unless Discardable). +- AND the field is NOT flagged as `x-psse-present-but-inactive` in Phase 1 D7. + +**Discardable assignment criteria:** +- Field IS flagged as `x-psse-present-but-inactive: true` in Phase 1 D7. +- This is the sole criterion -- no field may be Discardable without this flag. + +## v10 Reclassification Note + +Seven fields were reclassified from DCPF-critical to Informational in protocol v10 (2026-03-13): + +| Table | Field | Previous Tier | New Tier | Rationale | +|-------|-------|--------------|----------|-----------| +| load | ID | DCPF-critical | Informational | Identifier-only; bus injection sum is unaffected by its absence | +| generator | ID | DCPF-critical | Informational | Identifier-only; bus injection sum is unaffected by its absence | +| branch | CKT | DCPF-critical | Informational | Identifier-only; branch enumeration by index preserves B-matrix | +| transformer | CKT | DCPF-critical | Informational | Identifier-only; same rationale as branch.CKT | +| transformer | K | DCPF-critical | Informational | Star-equivalent conversion preserves DCPF topology | +| transformer | X2_3 | DCPF-critical | Informational | Star-equivalent impedances embed X2_3; DCPF accuracy preserved | +| transformer | X3_1 | DCPF-critical | Informational | Same rationale as X2_3 | + +**Impact:** DCPF-critical field count reduced from 26 to 19. G-FNM-2 pass condition +(100% DCPF-critical coverage) evaluated against 19 fields from v10 onward. Existing +G-FNM-2 results produced under v9 remain valid for their protocol version. + +## Cross-References + +- [Intermediate Schema Reference](intermediate-schema.md) -- field definitions and semantics (PRD 01) +- [Record-Type Mapping Guide](mapping-guide.md) -- record-type tier classification (PRD 02) +- [Per-Unit Convention Reference](per-unit-conventions.md) -- per-unit base assignments (PRD 03) +- [3-Winding Transformer Reference](three-winding-transformers.md) -- transformer field semantics (PRD 04) +- Phase 1 D7 JSON Schema files (`../intermediate/schemas/`) -- normative field inventories and `x-psse-*` annotations diff --git a/data/fnm/docs/intermediate-schema.md b/data/fnm/docs/intermediate-schema.md new file mode 100644 index 00000000..9ebbba7b --- /dev/null +++ b/data/fnm/docs/intermediate-schema.md @@ -0,0 +1,1253 @@ +# Intermediate Format Schema Reference + +**Version:** 1.0 +**Phase 1 Schema:** `../intermediate/schemas/` (JSON Schema Draft 2020-12) +**Audience:** evaluate-tool agents, human reviewers +**Normative definitions:** Phase 1 D7 JSON Schema files define data types, + required/optional status, and valid ranges. This document adds semantic + descriptions, worked examples, and ingestion verification guidance. + +## Table Summary + +| Table | PSS/E Record Type | Records | Columns | Primary Key | Purpose | +| ----- | ----------------- | ------- | ------- | ----------- | ------- | +| `bus` | Bus | ~30,000 | 13 | `[I]` | PSS/E v31 Bus record type | +| `load` | Load | ~15,000 | 13 | `[I, ID]` | PSS/E v31 Load record type | +| `fixed_shunt` | Fixed Shunt | ~500 | 5 | `[I, ID]` | PSS/E v31 Fixed Shunt record type | +| `generator` | Generator | ~5,000 | 28 | `[I, ID]` | PSS/E v31 Generator record type | +| `branch` | Branch | ~35,000 | 24 | `[I, J, CKT]` | PSS/E v31 Branch record type | +| `transformer` | Transformer | ~8,000 | 83 | `[I, J, K, CKT]` | PSS/E v31 Transformer record type | +| `area` | Area | ~30 | 5 | `[I]` | PSS/E v31 Area record type | +| `two_terminal_dc` | Two-Terminal DC | ~5 | 46 | `[NAME]` | PSS/E v31 Two-Terminal DC line record type | +| `vsc_dc` | VSC DC | ~2 | 41 | `[NAME]` | PSS/E v31 VSC DC line record type | +| `impedance_correction` | Impedance Correction | ~200 | 23 | `[T]` | PSS/E v31 Impedance Correction table record type | +| `multi_terminal_dc` | Multi-Terminal DC | ~1 | 8 | `[NAME]` | PSS/E v31 Multi-Terminal DC line header record type | +| `multi_section_line` | Multi-Section Line | ~800 | 13 | `[I, J, ID]` | PSS/E v31 Multi-Section Line grouping record type | +| `zone` | Zone | ~40 | 2 | `[I]` | PSS/E v31 Zone record type | +| `interarea_transfer` | Interarea Transfer | ~50 | 4 | `[ARFROM, ARTO, TRID]` | PSS/E v31 Interarea Transfer record type | +| `owner` | Owner | ~100 | 2 | `[I]` | PSS/E v31 Owner record type | +| `facts` | FACTS | ~50 | 14 | `[NAME]` | PSS/E v31 FACTS device record type | +| `switched_shunt` | Switched Shunt | ~3,000 | 26 | `[I]` | PSS/E v31 Switched Shunt record type | + +## Bus + +**Table name:** `bus` +**Schema file:** [`../intermediate/schemas/bus.schema.json`](../intermediate/schemas/bus.schema.json) +**Primary key:** `[I]` +**Purpose:** Defines every node (bus) in the transmission network. Each bus has a unique number, base voltage, type code (PQ/PV/swing/isolated), and solved-state voltage. All other record types reference buses by number. The bus table is the topological foundation of the network model. + +### Fields + +| Field | Type | Unit | Semantic Description | Expected Range | Nullable | Default | Evaluate-Tool Guidance | +| ----- | ---- | ---- | -------------------- | -------------- | -------- | ------- | ---------------------- | +| `I` | integer | — | Unique bus number identifying this node in the network topology. All branch, generator, load, and shunt records reference buses by this number. | 10000–99999 for large networks | no | none | Verify I is a positive integer preserved exactly; loss of bus number destroys topology | +| `NAME` | string | — | Alphanumeric bus name, up to 12 characters, padded with trailing spaces. Used for human-readable identification in reports and diagrams. | — | no | `" "` | Verify NAME is preserved including trailing whitespace; compare after stripping only if the tool normalizes whitespace | +| `BASKV` | number | kV | Bus base voltage in kV. Defines the voltage class for this bus and is the reference for all per-unit voltage calculations at this bus. A value of 0.0 is the PSS/E default but is physically meaningless for real network buses. | 69–500 for transmission | no | 0.0 | Verify BASKV > 0 for all buses with IDE != 4 (non-isolated); verify preserved to at least 1 decimal place | +| `IDE` | integer | — | Bus type code: 1=PQ (load), 2=PV (generator), 3=swing (reference), 4=isolated (disconnected). Determines the power flow solution method at this bus. | 1–4 | no | 1 | Verify IDE is one of {1, 2, 3, 4} and matches the source exactly; confirm bus type code maps to the tool's equivalent enum without silent coercion | +| `AREA` | integer | — | Area number to which this bus is assigned. Areas define interchange control regions in the power flow solution. | 1–50 for large ISOs | no | 1 | Verify AREA references a valid area number in the area table | +| `ZONE` | integer | — | Zone number for geographic or administrative grouping. Zones provide finer-grained grouping than areas, used in reporting and load allocation. | 1–50 | no | 1 | Verify ZONE references a valid zone number in the zone table | +| `OWNER` | integer | — | Owner number identifying the entity that owns this bus. Used for ownership tracking and cost allocation. | 1–200 | no | 1 | Verify OWNER references a valid owner number in the owner table | +| `VM` | number | pu | Bus voltage magnitude in per-unit on the bus base voltage (BASKV). In a solved case, this represents the steady-state voltage. In an unsolved case, this is the initial voltage guess. | 0.95–1.05 for solved case | no | 1.0 | Verify VM preserved to at least 4 decimal places; values outside 0.9–1.1 in a solved case indicate convergence issues | +| `VA` | number | deg | Bus voltage angle in degrees. The swing bus angle is the reference (typically 0.0). All other angles are relative to the swing bus. | -180–180 | no | 0.0 | Verify VA preserved to at least 2 decimal places; swing bus (IDE=3) should have VA near 0.0 | +| `NVHI` | number | pu | Normal operating voltage high limit in per-unit. Used by OPF and monitoring functions to flag voltage violations. | 1.05–1.10 | yes | 1.1 | If field is at PSS/E default (1.1), tool may omit — do not penalize | +| `NVLO` | number | pu | Normal operating voltage low limit in per-unit. Buses with voltage below this limit are flagged as voltage violations. | 0.90–0.95 | yes | 0.9 | If field is at PSS/E default (0.9), tool may omit — do not penalize | +| `EVHI` | number | pu | Emergency voltage high limit in per-unit. Applied during contingency analysis to allow wider voltage tolerance under emergency conditions. | 1.05–1.10 | yes | 1.1 | If field is at PSS/E default (1.1), tool may omit — do not penalize | +| `EVLO` | number | pu | Emergency voltage low limit in per-unit. More relaxed than normal low limit, applied during contingency analysis. | 0.85–0.95 | yes | 0.9 | If field is at PSS/E default (0.9), tool may omit — do not penalize | + +### Worked Example + +``` +I: 30100 +NAME: "MESA 230 " +BASKV: 230.0 +IDE: 1 +AREA: 5 +ZONE: 12 +OWNER: 3 +VM: 1.0142 +VA: -8.35 +NVHI: 1.1 +NVLO: 0.9 +EVHI: 1.1 +EVLO: 0.9 +``` + +### Nullable and Default Behavior + +All bus fields are required in PSS/E v31 and have well-defined defaults. BASKV=0.0 is the PSS/E default but indicates an uninitialized bus; real network buses always have BASKV > 0. VM defaults to 1.0 (flat start), and VA defaults to 0.0 degrees. The voltage limit fields (NVHI, NVLO, EVHI, EVLO) default to standard PSS/E values and may be omitted by tools without penalty. The canonical parser writes all fields including those at default values. + +### Cross-References + +- See [Per-Unit Convention Reference](per-unit-conventions.md#bus-voltage) for VM/VA per-unit basis. +- See [Field Criticality Matrix](field-criticality-matrix.md) for DCPF/ACPF criticality tiers. +- See [Record-Type Mapping Guide](mapping-guide.md#bus) for tool-specific bus representations. + +## Load + +**Table name:** `load` +**Schema file:** [`../intermediate/schemas/load.schema.json`](../intermediate/schemas/load.schema.json) +**Primary key:** `[I, ID]` +**Purpose:** Represents electrical demand at each bus. Loads are modeled with constant-power (PL, QL), constant-current (IP, IQ), and constant-admittance (YP, YQ) components. Multiple loads can exist at one bus, distinguished by their two-character ID. + +### Fields + +| Field | Type | Unit | Semantic Description | Expected Range | Nullable | Default | Evaluate-Tool Guidance | +| ----- | ---- | ---- | -------------------- | -------------- | -------- | ------- | ---------------------- | +| `I` | integer | — | Bus number at which this load is connected. Multiple loads can exist at the same bus, distinguished by ID. | 10000–99999 | no | none | Verify I references a valid bus number in the bus table | +| `ID` | string | — | Two-character load identifier. Together with I, forms the composite primary key. Default '1 ' (one followed by space). | — | no | `"1 "` | Verify ID is preserved as a 2-character string including trailing space | +| `STATUS` | integer | — | Load status: 1=in-service (included in power flow), 0=out-of-service (excluded from power flow solution). | 0–1 | no | 1 | Verify STATUS is one of {0, 1} and matches the source exactly | +| `AREA` | integer | — | Area number for this load, defaults to the bus's area. Used for area interchange calculations. | 1–50 | no | 1 | Verify AREA is a positive integer; if at default (1), tool may inherit from bus | +| `ZONE` | integer | — | Zone number for this load, defaults to the bus's zone. | 1–50 | no | 1 | Verify ZONE is a positive integer; if at default (1), tool may inherit from bus | +| `PL` | number | MW | Constant-power active load in MW. The primary real power demand at this bus. Positive values consume power. | 0–5000 | no | 0.0 | Verify PL preserved to at least 2 decimal places; sign convention: positive = consumption | +| `QL` | number | MVAR | Constant-power reactive load in MVAR. Positive values consume reactive power (lagging power factor). | -500–1000 | no | 0.0 | Verify QL preserved to at least 2 decimal places; sign convention: positive = lagging (inductive) | +| `IP` | number | MW | Constant-current active load component in MW at 1.0 pu voltage. Scales linearly with voltage magnitude. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `IQ` | number | MVAR | Constant-current reactive load component in MVAR at 1.0 pu voltage. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `YP` | number | MW | Constant-admittance active load component in MW at 1.0 pu voltage. Scales with voltage squared. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `YQ` | number | MVAR | Constant-admittance reactive load component in MVAR at 1.0 pu voltage. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `OWNER` | integer | — | Owner number for this load. | 1–200 | yes | 1 | If field is at PSS/E default (1), tool may omit — do not penalize | +| `SCALE` | integer | — | Load scaling flag: 1=load participates in scaling, 0=fixed load. Controls whether the load is adjusted during area interchange scaling. | 0–1 | yes | 1 | If field is at PSS/E default (1), tool may omit — do not penalize | + +### Worked Example + +``` +I: 30100 +ID: "1 " +STATUS: 1 +AREA: 5 +ZONE: 12 +PL: 245.80 +QL: 85.30 +IP: 0.0 +IQ: 0.0 +YP: 0.0 +YQ: 0.0 +OWNER: 3 +SCALE: 1 +``` + +### Nullable and Default Behavior + +The constant-current (IP, IQ) and constant-admittance (YP, YQ) load components default to 0.0, meaning the load is modeled as constant-power only. A zero value is semantically meaningful (not missing) -- it means that load component is absent. OWNER defaults to 1, and SCALE defaults to 1 (load participates in scaling). + +### Cross-References + +- See [Per-Unit Convention Reference](per-unit-conventions.md) for load component scaling. +- See [Record-Type Mapping Guide](mapping-guide.md#load) for tool-specific load representations. + +## Fixed Shunt + +**Table name:** `fixed_shunt` +**Schema file:** [`../intermediate/schemas/fixed_shunt.schema.json`](../intermediate/schemas/fixed_shunt.schema.json) +**Primary key:** `[I, ID]` +**Purpose:** Represents fixed (non-switchable) shunt compensation devices. Fixed shunts provide reactive power support (capacitive) or absorption (inductive) at a constant value regardless of voltage. Distinguished from switched shunts which have discrete steps. + +### Fields + +| Field | Type | Unit | Semantic Description | Expected Range | Nullable | Default | Evaluate-Tool Guidance | +| ----- | ---- | ---- | -------------------- | -------------- | -------- | ------- | ---------------------- | +| `I` | integer | — | Bus number at which this fixed shunt is connected. | 10000–99999 | no | none | Verify I references a valid bus number in the bus table | +| `ID` | string | — | Two-character shunt identifier. Together with I, forms the composite primary key. | — | no | `"1 "` | Verify ID is preserved as a 2-character string including trailing space | +| `STATUS` | integer | — | Shunt status: 1=in-service, 0=out-of-service. | 0–1 | no | 1 | Verify STATUS is one of {0, 1} and matches the source exactly | +| `GL` | number | MW | Active component of shunt admittance to ground in MW at 1.0 pu voltage. Positive GL represents real power consumption (resistive losses). | — | no | 0.0 | Verify GL preserved to at least 4 decimal places; most fixed shunts have GL=0 (purely reactive) | +| `BL` | number | MVAR | Reactive component of shunt admittance to ground in MVAR at 1.0 pu voltage. Positive BL is capacitive (generates reactive power), negative BL is inductive (absorbs reactive power). | -500–500 | no | 0.0 | Verify BL is positive for capacitive shunts, negative for inductive; verify preserved to at least 2 decimal places | + +### Worked Example + +``` +I: 42500 +ID: "1 " +STATUS: 1 +GL: 0.0 +BL: 150.0 +``` + +### Nullable and Default Behavior + +GL defaults to 0.0, meaning no active power loss in the shunt (purely reactive). BL defaults to 0.0 but is typically non-zero for any meaningful shunt device. STATUS defaults to 1 (in-service). + +### Cross-References + +- See [Per-Unit Convention Reference](per-unit-conventions.md#shunt-admittance) for BL sign convention. +- See [Record-Type Mapping Guide](mapping-guide.md#fixed-shunt) for tool-specific shunt representations. + +## Generator + +**Table name:** `generator` +**Schema file:** [`../intermediate/schemas/generator.schema.json`](../intermediate/schemas/generator.schema.json) +**Primary key:** `[I, ID]` +**Purpose:** Represents all generating units including conventional thermal, hydro, wind, and solar plants. Each generator has active/reactive output, capability limits, voltage setpoint, and machine impedance data. Multiple generators at one bus use different IDs. + +### Fields + +| Field | Type | Unit | Semantic Description | Expected Range | Nullable | Default | Evaluate-Tool Guidance | +| ----- | ---- | ---- | -------------------- | -------------- | -------- | ------- | ---------------------- | +| `I` | integer | — | Bus number at which this generator is connected. | 10000–99999 | no | none | Verify I references a valid bus number in the bus table | +| `ID` | string | — | Two-character machine identifier. Together with I, forms the composite primary key. Allows multiple generators at the same bus. | — | no | `"1 "` | Verify ID is preserved as a 2-character string including trailing space | +| `PG` | number | MW | Active power output of the generator in MW. Positive values indicate generation. Negative values indicate a synchronous condenser consuming real power. | 50–1000 for large units | no | 0.0 | Verify PG preserved to at least 2 decimal places | +| `QG` | number | MVAR | Reactive power output of the generator in MVAR. Determined by the power flow solution within the QB–QT limits. | -500–500 | no | 0.0 | Verify QG preserved to at least 2 decimal places | +| `QT` | number | MVAR | Maximum reactive power output in MVAR. Upper limit for the generator's reactive capability curve. | 0–1000 | no | 9999.0 | Verify QT preserved to at least 1 decimal place; default 9999.0 indicates unconstrained | +| `QB` | number | MVAR | Minimum reactive power output in MVAR. Lower limit for the generator's reactive capability. | -1000–0 | no | -9999.0 | Verify QB preserved to at least 1 decimal place; default -9999.0 indicates unconstrained | +| `VS` | number | pu | Voltage setpoint for voltage-regulating generators in per-unit. The generator adjusts reactive output to maintain this voltage at the regulated bus (local or remote via IREG). | 0.95–1.10 | no | 1.0 | Verify VS preserved to at least 4 decimal places | +| `IREG` | integer | — | **[preservation-critical]** Remote regulated bus number. 0=local voltage regulation (at bus I). Non-zero=remote bus whose voltage is controlled by this generator. Critical for correct voltage regulation topology in power flow. | 0 or valid bus number | no | 0 | MUST be preserved exactly; verify IREG=0 means local regulation, not missing; loss of remote regulation topology is a fidelity finding | +| `MBASE` | number | MVA | Machine MVA base for per-unit impedance conversion. Generator impedances ZR, ZX are on this base. | 50–1500 | no | 100.0 | Verify MBASE preserved to at least 1 decimal place | +| `ZR` | number | pu | Machine resistance in per-unit on MBASE. Part of the generator's internal impedance model for short-circuit studies. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `ZX` | number | pu | Machine reactance in per-unit on MBASE. Sub-transient or transient reactance used in short-circuit calculations. | 0.1–0.4 | yes | 1.0 | If field is at PSS/E default (1.0), tool may omit — do not penalize | +| `RT` | number | pu | Step-up transformer resistance in per-unit on MBASE. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `XT` | number | pu | Step-up transformer reactance in per-unit on MBASE. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `GTAP` | number | pu | Step-up transformer off-nominal turns ratio in per-unit on bus base kV. | 0.9–1.1 | yes | 1.0 | If field is at PSS/E default (1.0), tool may omit — do not penalize | +| `STAT` | integer | — | Generator status: 1=in-service, 0=out-of-service. | 0–1 | no | 1 | Verify STAT is one of {0, 1} and matches the source exactly | +| `RMPCT` | number | % | Percent of total MVAR range allocated to remote voltage regulation. | 0–100 | yes | 100.0 | If field is at PSS/E default (100.0), tool may omit — do not penalize | +| `PT` | number | MW | Maximum active power output in MW. | 50–2000 | yes | 9999.0 | Verify PT preserved to at least 1 decimal place; default 9999.0 indicates unconstrained | +| `PB` | number | MW | Minimum active power output in MW. | -100–0 | yes | -9999.0 | Verify PB preserved to at least 1 decimal place; default -9999.0 indicates unconstrained | +| `O1` | integer | — | Owner number 1. | 1–200 | yes | 1 | If field is at PSS/E default (1), tool may omit — do not penalize | +| `F1` | number | — | Fraction of generator owned by owner 1. | 0.0–1.0 | yes | 1.0 | If field is at PSS/E default (1.0), tool may omit — do not penalize | +| `O2` | integer | — | Owner number 2. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `F2` | number | — | Fraction owned by owner 2. | 0.0–1.0 | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `O3` | integer | — | Owner number 3. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `F3` | number | — | Fraction owned by owner 3. | 0.0–1.0 | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `O4` | integer | — | Owner number 4. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `F4` | number | — | Fraction owned by owner 4. | 0.0–1.0 | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `WMOD` | integer | — | Wind machine reactive power control mode. 0=standard, 1=constant power factor, 2=constant Q, 3=constant voltage. | 0–3 | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `WPF` | number | — | Wind machine power factor for WMOD=1 mode. | 0.8–1.0 | yes | 1.0 | If field is at PSS/E default (1.0), tool may omit — do not penalize | + +### Worked Example + +``` +I: 50200 +ID: "1 " +PG: 350.00 +QG: 45.20 +QT: 200.0 +QB: -100.0 +VS: 1.0250 +IREG: 0 +MBASE: 400.0 +ZR: 0.0 +ZX: 1.0 +RT: 0.0 +XT: 0.0 +GTAP: 1.0 +STAT: 1 +RMPCT: 100.0 +PT: 400.0 +PB: 100.0 +O1: 5 +F1: 1.0 +O2: 0 +F2: 0.0 +O3: 0 +F3: 0.0 +O4: 0 +F4: 0.0 +WMOD: 0 +WPF: 1.0 +``` + +### Nullable and Default Behavior + +QT=9999.0 and QB=-9999.0 indicate unconstrained reactive capability (PSS/E defaults). IREG=0 means local voltage regulation (at bus I), not 'no regulation'. This distinction is critical: zero is a meaningful value, not a null. ZR, ZX, RT, XT, GTAP are machine impedance parameters that default to their PSS/E values; tools commonly omit these for steady-state power flow. Owner fields O2-O4 default to 0, meaning single ownership. + +### Cross-References + +- See [Per-Unit Convention Reference](per-unit-conventions.md#generator-impedance) for MBASE-based per-unit. +- See [Record-Type Mapping Guide](mapping-guide.md#generator) for tool-specific generator representations. + +## Branch + +**Table name:** `branch` +**Schema file:** [`../intermediate/schemas/branch.schema.json`](../intermediate/schemas/branch.schema.json) +**Primary key:** `[I, J, CKT]` +**Purpose:** Represents transmission lines, cables, and series elements connecting two buses. Each branch has impedance (R, X, B), thermal ratings, and status. Parallel branches between the same bus pair are distinguished by circuit identifier CKT. + +### Fields + +| Field | Type | Unit | Semantic Description | Expected Range | Nullable | Default | Evaluate-Tool Guidance | +| ----- | ---- | ---- | -------------------- | -------------- | -------- | ------- | ---------------------- | +| `I` | integer | — | From-bus number. Together with J and CKT, forms the branch's composite primary key. | 10000–99999 | no | none | Verify I references a valid bus number in the bus table | +| `J` | integer | — | To-bus number. Branch connects bus I to bus J. The sign of J does not matter for topology (absolute value is used). | 10000–99999 | no | none | Verify J references a valid bus number in the bus table | +| `CKT` | string | — | Two-character circuit identifier allowing parallel branches between the same bus pair. | — | no | `"1 "` | Verify CKT is preserved as a 2-character string including trailing space | +| `R` | number | pu | Branch resistance in per-unit on system MVA base (SBASE) and bus base voltage. For transmission lines, R is typically much smaller than X (R/X ratio < 0.5). | 0.0001–0.1 | no | none | Verify R preserved to at least 5 decimal places; verify R < X for transmission lines (R/X < 1.0) | +| `X` | number | pu | Branch reactance in per-unit on system MVA base. The dominant impedance component for transmission lines. X must be non-zero for in-service branches. | 0.001–0.5 | no | none | Verify X is non-zero for all in-service branches (ST=1); verify X preserved to at least 5 decimal places | +| `B` | number | pu | Total branch charging susceptance in per-unit on system MVA base. For overhead lines, B is proportional to line length and voltage. For short lines, B may be 0. | 0.0–5.0 | no | 0.0 | Verify B preserved to at least 5 decimal places; B=0 is valid for short lines and cables | +| `RATEA` | number | MVA | Normal thermal rating in MVA (Rating A). Used for continuous loading monitoring. | 0–3000 | no | 0.0 | Verify RATEA preserved to at least 1 decimal place; 0.0 means no limit (not monitored) | +| `RATEB` | number | MVA | Emergency thermal rating in MVA (Rating B). Short-term overload limit. | 0–4000 | no | 0.0 | Verify RATEB preserved to at least 1 decimal place; 0.0 means no limit | +| `RATEC` | number | MVA | Long-term emergency rating in MVA (Rating C). | 0–5000 | no | 0.0 | Verify RATEC preserved to at least 1 decimal place; 0.0 means no limit | +| `GI` | number | pu | Line shunt conductance at from-bus end in per-unit. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `BI` | number | pu | Line shunt susceptance at from-bus end in per-unit. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `GJ` | number | pu | Line shunt conductance at to-bus end in per-unit. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `BJ` | number | pu | Line shunt susceptance at to-bus end in per-unit. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `ST` | integer | — | Branch status: 1=in-service, 0=out-of-service. Out-of-service branches are excluded from the admittance matrix. | 0–1 | no | 1 | Verify ST is one of {0, 1} and matches the source exactly | +| `MET` | integer | — | Metered end flag: 1=from-bus (I), 2=to-bus (J). Determines which end is used for loss allocation. | 1–2 | yes | 1 | If field is at PSS/E default (1), tool may omit — do not penalize | +| `LEN` | number | — | Line length in user-selected units. Informational field, not used in power flow calculations. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `O1` | integer | — | Owner number 1. | 1–200 | yes | 1 | If field is at PSS/E default (1), tool may omit — do not penalize | +| `F1` | number | — | Fraction owned by owner 1. | 0.0–1.0 | yes | 1.0 | If field is at PSS/E default (1.0), tool may omit — do not penalize | +| `O2` | integer | — | Owner number 2. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `F2` | number | — | Fraction owned by owner 2. | 0.0–1.0 | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `O3` | integer | — | Owner number 3. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `F3` | number | — | Fraction owned by owner 3. | 0.0–1.0 | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `O4` | integer | — | Owner number 4. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `F4` | number | — | Fraction owned by owner 4. | 0.0–1.0 | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | + +### Worked Example + +``` +I: 30100 +J: 30200 +CKT: "1 " +R: 0.00320 +X: 0.03150 +B: 0.52800 +RATEA: 600.0 +RATEB: 720.0 +RATEC: 800.0 +GI: 0.0 +BI: 0.0 +GJ: 0.0 +BJ: 0.0 +ST: 1 +MET: 1 +LEN: 0.0 +O1: 3 +F1: 1.0 +O2: 0 +F2: 0.0 +O3: 0 +F3: 0.0 +O4: 0 +F4: 0.0 +``` + +### Nullable and Default Behavior + +R and X have no default -- they must be provided for every branch. B defaults to 0.0 (no line charging), which is valid for short lines. Rating fields (RATEA, RATEB, RATEC) default to 0.0, meaning no thermal limit is enforced. GI, BI, GJ, BJ are line shunt elements that default to 0.0 (no shunt admittance at line ends). Owner fields O2-O4 default to 0. + +### Cross-References + +- See [Per-Unit Convention Reference](per-unit-conventions.md#branch-impedance) for conversion formulas. +- See [Field Criticality Matrix](field-criticality-matrix.md) for DCPF-critical branch fields. +- See [Record-Type Mapping Guide](mapping-guide.md#branch) for tool-specific branch representations. + +## Transformer + +**Table name:** `transformer` +**Schema file:** [`../intermediate/schemas/transformer.schema.json`](../intermediate/schemas/transformer.schema.json) +**Primary key:** `[I, J, K, CKT]` +**Purpose:** Represents 2-winding and 3-winding power transformers. The PSS/E RAW format uses a multi-line record (up to 5 lines) that is flattened into a single row in the intermediate format. The CW/CZ/CM codes control how impedance and turns-ratio data are interpreted -- these must be preserved exactly. + +### Fields + +| Field | Type | Unit | Semantic Description | Expected Range | Nullable | Default | Evaluate-Tool Guidance | +| ----- | ---- | ---- | -------------------- | -------------- | -------- | ------- | ---------------------- | +| `I` | integer | — | Winding 1 (primary) bus number. | 10000–99999 | no | none | Verify I references a valid bus number in the bus table | +| `J` | integer | — | Winding 2 (secondary) bus number. | 10000–99999 | no | none | Verify J references a valid bus number in the bus table | +| `K` | integer | — | **[preservation-critical]** Winding 3 bus number. K=0 indicates a 2-winding transformer; K!=0 indicates a 3-winding transformer. This field determines the topology interpretation for all subsequent winding data. | 0 or valid bus number | no | 0 | MUST be preserved exactly; K=0 vs K!=0 changes transformer topology interpretation entirely; loss is a critical fidelity finding | +| `CKT` | string | — | Circuit identifier for parallel transformers. | — | no | `"1 "` | Verify CKT is preserved as a 2-character string including trailing space | +| `CW` | integer | — | **[preservation-critical]** Winding data I/O code controlling how WINDV1/2/3 are interpreted: 1=turns ratio in pu on bus base kV, 2=voltage in kV, 3=turns ratio in pu on nominal kV. | 1–3 | no | 1 | MUST be preserved exactly; CW determines the interpretation of all winding voltage/turns-ratio fields; loss corrupts impedance calculations | +| `CZ` | integer | — | **[preservation-critical]** Impedance data I/O code: 1=pu on system base, 2=pu on winding MVA/kV base, 3=ohms/kV load loss. | 1–3 | no | 1 | MUST be preserved exactly; CZ determines per-unit base for R and X fields | +| `CM` | integer | — | **[preservation-critical]** Magnetizing admittance I/O code: 1=pu on system base, 2=no-load loss/exciting current. | 1–2 | no | 1 | MUST be preserved exactly; CM determines interpretation of MAG1/MAG2 | +| `MAG1` | number | — | Magnetizing conductance or no-load loss, depending on CM. | — | yes | 0.0 | Verify MAG1 preserved to at least 5 decimal places | +| `MAG2` | number | — | Magnetizing susceptance or exciting current, depending on CM. | — | yes | 0.0 | Verify MAG2 preserved to at least 5 decimal places | +| `NMETR` | integer | — | Non-metered end code. | — | yes | 2 | If field is at PSS/E default (2), tool may omit — do not penalize | +| `NAME` | string | — | Transformer name, up to 12 characters. | — | yes | `" "` | Verify NAME is preserved including trailing whitespace | +| `STAT` | integer | — | Transformer status: 0=out-of-service, 1=in-service, 2=winding 2 out, 3=winding 3 out, 4=winding 2 and 3 out. | 0–4 | no | 1 | Verify STAT is one of {0, 1, 2, 3, 4} and matches the source exactly | +| `O1` | integer | — | Owner number 1. | 1–200 | yes | 1 | If field is at PSS/E default (1), tool may omit — do not penalize | +| `F1` | number | — | Fraction by owner 1. | 0.0–1.0 | yes | 1.0 | If field is at PSS/E default (1.0), tool may omit — do not penalize | +| `O2` | integer | — | Owner 2. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `F2` | number | — | Fraction by owner 2. | 0.0–1.0 | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `O3` | integer | — | Owner 3. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `F3` | number | — | Fraction by owner 3. | 0.0–1.0 | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `O4` | integer | — | Owner 4. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `F4` | number | — | Fraction by owner 4. | 0.0–1.0 | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `VECGRP` | string | — | Vector group designation (12 chars). | — | yes | `" "` | If field is at PSS/E default (blank), tool may omit — do not penalize | +| `R1_2` | number | pu | Resistance of winding 1–2 pair, interpretation depends on CZ. | 0.0–0.1 | no | 0.0 | Verify R1_2 preserved to at least 5 decimal places | +| `X1_2` | number | pu | Reactance of winding 1–2 pair, interpretation depends on CZ. | 0.01–0.5 | no | none | Verify X1_2 is non-zero for in-service transformers; verify preserved to at least 5 decimal places | +| `SBASE1_2` | number | MVA | MVA base for winding 1–2 impedance. | 50–2000 | no | 100.0 | Verify SBASE1_2 preserved to at least 1 decimal place | +| `R2_3` | number | pu | Resistance of winding 2–3 pair (3W only). | — | yes | 0.0 | Verify non-null when K != 0; if K=0, tool may omit | +| `X2_3` | number | pu | Reactance of winding 2–3 pair (3W only). | — | yes | 0.0 | Verify non-null when K != 0; if K=0, tool may omit | +| `SBASE2_3` | number | MVA | MVA base for winding 2–3 (3W only). | — | yes | 100.0 | Verify non-null when K != 0; if K=0, tool may omit | +| `R3_1` | number | pu | Resistance of winding 3–1 pair (3W only). | — | yes | 0.0 | Verify non-null when K != 0; if K=0, tool may omit | +| `X3_1` | number | pu | Reactance of winding 3–1 pair (3W only). | — | yes | 0.0 | Verify non-null when K != 0; if K=0, tool may omit | +| `SBASE3_1` | number | MVA | MVA base for winding 3–1 (3W only). | — | yes | 100.0 | Verify non-null when K != 0; if K=0, tool may omit | +| `VMSTAR` | number | pu | Star-point bus voltage magnitude for 3W transformers. | — | yes | 1.0 | If field is at PSS/E default (1.0), tool may omit — do not penalize | +| `ANSTAR` | number | deg | Star-point bus voltage angle for 3W transformers. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `WINDV1` | number | — | **[preservation-critical]** Winding 1 off-nominal turns ratio or voltage. Interpretation depends on CW code. | 0.9–1.1 (pu) or kV | no | 1.0 | MUST be preserved exactly to at least 5 decimal places; loss corrupts transformer model | +| `NOMV1` | number | kV | **[preservation-critical]** Winding 1 nominal voltage in kV. Used with CW=3 for turns ratio calculation. Must correspond to a standard transmission voltage class. | 69–500 | no | 0.0 | MUST be preserved exactly; verify drawn from standard kV classes | +| `ANG1` | number | deg | **[preservation-critical]** Winding 1 phase shift angle in degrees. Non-zero for phase-shifting transformers. | -180–180 | no | 0.0 | MUST be preserved exactly to at least 2 decimal places; non-zero ANG1 indicates phase-shifting transformer | +| `RATA1` | number | MVA | **[preservation-critical]** Winding 1 normal rating in MVA. | 50–2000 | no | 0.0 | MUST be preserved exactly to at least 1 decimal place | +| `RATB1` | number | MVA | Winding 1 emergency rating in MVA. | 50–3000 | yes | 0.0 | Verify RATB1 preserved to at least 1 decimal place | +| `RATC1` | number | MVA | Winding 1 long-term emergency rating in MVA. | 50–4000 | yes | 0.0 | Verify RATC1 preserved to at least 1 decimal place | +| `COD1` | integer | — | Winding 1 tap control mode code. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `CONT1` | integer | — | Winding 1 controlled bus number. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `RMA1` | number | — | Winding 1 upper tap or voltage limit. | 0.9–1.1 | yes | 1.1 | Verify RMA1 preserved to at least 4 decimal places | +| `RMI1` | number | — | Winding 1 lower tap or voltage limit. | 0.9–1.1 | yes | 0.9 | Verify RMI1 preserved to at least 4 decimal places | +| `VMA1` | number | — | Winding 1 upper voltage limit for control. | 1.0–1.1 | yes | 1.1 | Verify VMA1 preserved to at least 4 decimal places | +| `VMI1` | number | — | Winding 1 lower voltage limit for control. | 0.9–1.0 | yes | 0.9 | Verify VMI1 preserved to at least 4 decimal places | +| `NTP1` | integer | — | Number of tap positions for winding 1. | 11–99 | yes | 33 | If field is at PSS/E default (33), tool may omit — do not penalize | +| `TAB1` | integer | — | Impedance correction table number for winding 1. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `CR1` | number | pu | Load drop compensation resistance for winding 1. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `CX1` | number | pu | Load drop compensation reactance for winding 1. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `CNXA1` | integer | — | Connection angle for winding 1. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `WINDV2` | number | — | **[preservation-critical]** Winding 2 off-nominal turns ratio or voltage. | 0.9–1.1 (pu) or kV | no | 1.0 | MUST be preserved exactly to at least 5 decimal places | +| `NOMV2` | number | kV | **[preservation-critical]** Winding 2 nominal voltage in kV. | 69–500 | no | 0.0 | MUST be preserved exactly; verify drawn from standard kV classes | +| `ANG2` | number | deg | Winding 2 phase shift angle in degrees. | -180–180 | yes | 0.0 | Verify ANG2 preserved to at least 2 decimal places | +| `RATA2` | number | MVA | **[preservation-critical]** Winding 2 normal rating in MVA. | 50–2000 | yes | 0.0 | MUST be preserved exactly to at least 1 decimal place | +| `RATB2` | number | MVA | Winding 2 emergency rating in MVA. | — | yes | 0.0 | Verify RATB2 preserved to at least 1 decimal place | +| `RATC2` | number | MVA | Winding 2 long-term emergency rating. | — | yes | 0.0 | Verify RATC2 preserved to at least 1 decimal place | +| `COD2` | integer | — | Winding 2 tap control mode code. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `CONT2` | integer | — | Winding 2 controlled bus number. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `RMA2` | number | — | Winding 2 upper tap or voltage limit. | 0.9–1.1 | yes | 1.1 | Verify RMA2 preserved to at least 4 decimal places | +| `RMI2` | number | — | Winding 2 lower tap or voltage limit. | 0.9–1.1 | yes | 0.9 | Verify RMI2 preserved to at least 4 decimal places | +| `VMA2` | number | — | Winding 2 upper voltage limit for control. | 1.0–1.1 | yes | 1.1 | Verify VMA2 preserved to at least 4 decimal places | +| `VMI2` | number | — | Winding 2 lower voltage limit for control. | 0.9–1.0 | yes | 0.9 | Verify VMI2 preserved to at least 4 decimal places | +| `NTP2` | integer | — | Number of tap positions for winding 2. | 11–99 | yes | 33 | If field is at PSS/E default (33), tool may omit — do not penalize | +| `TAB2` | integer | — | Impedance correction table for winding 2. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `CR2` | number | pu | Load drop compensation resistance for winding 2. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `CX2` | number | pu | Load drop compensation reactance for winding 2. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `CNXA2` | integer | — | Connection angle for winding 2. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `WINDV3` | number | — | **[preservation-critical]** Winding 3 off-nominal turns ratio or voltage. Null/default for 2-winding transformers (K=0). | 0.9–1.1 (pu) or kV | yes | 1.0 | MUST be preserved exactly to at least 5 decimal places when K != 0; verify winding 3 fields are non-null when K != 0 | +| `NOMV3` | number | kV | **[preservation-critical]** Winding 3 nominal voltage in kV. Null/default for 2W transformers. | 69–500 | yes | 0.0 | MUST be preserved exactly when K != 0; verify drawn from standard kV classes | +| `ANG3` | number | deg | Winding 3 phase shift angle in degrees. | -180–180 | yes | 0.0 | Verify ANG3 preserved to at least 2 decimal places when K != 0 | +| `RATA3` | number | MVA | **[preservation-critical]** Winding 3 normal rating in MVA. | 50–2000 | yes | 0.0 | MUST be preserved exactly to at least 1 decimal place when K != 0 | +| `RATB3` | number | MVA | Winding 3 emergency rating. | — | yes | 0.0 | If K=0, tool may omit — do not penalize | +| `RATC3` | number | MVA | Winding 3 long-term emergency rating. | — | yes | 0.0 | If K=0, tool may omit — do not penalize | +| `COD3` | integer | — | Winding 3 tap control mode. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `CONT3` | integer | — | Winding 3 controlled bus. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `RMA3` | number | — | Winding 3 upper tap/voltage limit. | — | yes | 1.1 | Verify RMA3 preserved to at least 4 decimal places when K != 0 | +| `RMI3` | number | — | Winding 3 lower tap/voltage limit. | — | yes | 0.9 | Verify RMI3 preserved to at least 4 decimal places when K != 0 | +| `VMA3` | number | — | Winding 3 upper voltage limit for control. | — | yes | 1.1 | Verify VMA3 preserved to at least 4 decimal places when K != 0 | +| `VMI3` | number | — | Winding 3 lower voltage limit for control. | — | yes | 0.9 | Verify VMI3 preserved to at least 4 decimal places when K != 0 | +| `NTP3` | integer | — | Number of tap positions for winding 3. | — | yes | 33 | If field is at PSS/E default (33), tool may omit — do not penalize | +| `TAB3` | integer | — | Impedance correction table for winding 3. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `CR3` | number | pu | Load drop compensation resistance for winding 3. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `CX3` | number | pu | Load drop compensation reactance for winding 3. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `CNXA3` | integer | — | Connection angle for winding 3. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | + +### Worked Example + +``` +I: 30100 +J: 30500 +K: 0 +CKT: "1 " +CW: 1 +CZ: 1 +CM: 1 +MAG1: 0.0 +MAG2: 0.0 +NMETR: 2 +NAME: "XF-230/69 " +STAT: 1 +O1: 3 +F1: 1.0 +O2: 0 +F2: 0.0 +O3: 0 +F3: 0.0 +O4: 0 +F4: 0.0 +VECGRP: " " +R1_2: 0.00250 +X1_2: 0.12500 +SBASE1_2: 200.0 +R2_3: 0.0 +X2_3: 0.0 +SBASE2_3: 100.0 +R3_1: 0.0 +X3_1: 0.0 +SBASE3_1: 100.0 +VMSTAR: 1.0 +ANSTAR: 0.0 +WINDV1: 1.0125 +NOMV1: 230.0 +ANG1: 0.0 +RATA1: 200.0 +RATB1: 240.0 +RATC1: 280.0 +COD1: 0 +CONT1: 0 +RMA1: 1.1 +RMI1: 0.9 +VMA1: 1.1 +VMI1: 0.9 +NTP1: 33 +TAB1: 0 +CR1: 0.0 +CX1: 0.0 +CNXA1: 0 +WINDV2: 1.0 +NOMV2: 69.0 +ANG2: 0.0 +RATA2: 200.0 +RATB2: 0.0 +RATC2: 0.0 +COD2: 0 +CONT2: 0 +RMA2: 1.1 +RMI2: 0.9 +VMA2: 1.1 +VMI2: 0.9 +NTP2: 33 +TAB2: 0 +CR2: 0.0 +CX2: 0.0 +CNXA2: 0 +WINDV3: 1.0 +NOMV3: 0.0 +ANG3: 0.0 +RATA3: 0.0 +RATB3: 0.0 +RATC3: 0.0 +COD3: 0 +CONT3: 0 +RMA3: 1.1 +RMI3: 0.9 +VMA3: 1.1 +VMI3: 0.9 +NTP3: 33 +TAB3: 0 +CR3: 0.0 +CX3: 0.0 +CNXA3: 0 +``` + +### Nullable and Default Behavior + +K=0 indicates a 2-winding transformer; all winding-3 fields revert to defaults. The CW, CZ, CM codes default to 1 but are preservation-critical because they control how all impedance and turns-ratio fields are interpreted. WINDV1/WINDV2 default to 1.0 (unity turns ratio). NOMV1/NOMV2 default to 0.0, meaning the bus base kV is used. VMSTAR and ANSTAR are meaningful only for 3W transformers. + +### Cross-References + +- See [Per-Unit Convention Reference](per-unit-conventions.md#transformer-impedance) for CW/CZ/CM conversions. +- See [3-Winding Transformer Reference](three-winding-transformers.md) for topology details. +- See [Field Criticality Matrix](field-criticality-matrix.md) for preservation-critical transformer fields. +- See [Record-Type Mapping Guide](mapping-guide.md#transformer) for tool-specific representations. + +## Area + +**Table name:** `area` +**Schema file:** [`../intermediate/schemas/area.schema.json`](../intermediate/schemas/area.schema.json) +**Primary key:** `[I]` +**Purpose:** Defines interchange control areas for the power flow solution. Each area has a slack bus, desired net interchange (export/import), and tolerance. Areas are the primary aggregation unit for balancing supply and demand. + +### Fields + +| Field | Type | Unit | Semantic Description | Expected Range | Nullable | Default | Evaluate-Tool Guidance | +| ----- | ---- | ---- | -------------------- | -------------- | -------- | ------- | ---------------------- | +| `I` | integer | — | Unique area number identifying this interchange control area. | 1–50 | no | none | Verify I is a positive integer preserved exactly | +| `ISW` | integer | — | **[preservation-critical]** Area slack bus number. The swing bus that absorbs area interchange mismatch. 0=no area slack bus specified. | 0 or valid bus number | no | 0 | MUST be preserved exactly; loss of area slack assignment corrupts area interchange control | +| `PDES` | number | MW | **[preservation-critical]** Desired net area interchange in MW. Positive=export, negative=import. | -5000–5000 | no | 0.0 | MUST be preserved exactly to at least 2 decimal places | +| `PTOL` | number | MW | **[preservation-critical]** Area interchange tolerance in MW. Convergence criterion for area interchange control. | 1.0–50.0 | no | 10.0 | MUST be preserved exactly to at least 1 decimal place | +| `ARNAME` | string | — | Area name, up to 12 characters. | — | no | `" "` | Verify ARNAME is preserved including trailing whitespace | + +### Worked Example + +``` +I: 5 +ISW: 50200 +PDES: 150.0 +PTOL: 10.0 +ARNAME: "SOUTH ZONE " +``` + +### Nullable and Default Behavior + +ISW=0 means no area slack bus is designated. PDES=0.0 means no net interchange target. PTOL=10.0 is the default interchange tolerance. All fields are required. + +### Cross-References + +- See [Record-Type Mapping Guide](mapping-guide.md#area) for tool-specific area representations. + +## Two-Terminal DC + +**Table name:** `two_terminal_dc` +**Schema file:** [`../intermediate/schemas/two_terminal_dc.schema.json`](../intermediate/schemas/two_terminal_dc.schema.json) +**Primary key:** `[NAME]` +**Purpose:** Represents conventional line-commutated converter (LCC) HVDC links with a rectifier and inverter terminal. Each record contains DC line parameters plus full converter transformer and control data for both ends. + +### Fields + +| Field | Type | Unit | Semantic Description | Expected Range | Nullable | Default | Evaluate-Tool Guidance | +| ----- | ---- | ---- | -------------------- | -------------- | -------- | ------- | ---------------------- | +| `NAME` | string | — | DC line name. | — | no | none | Verify NAME is preserved as a string including any trailing whitespace | +| `MDC` | integer | — | Control mode (0-2). | 0–2 | no | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `RDC` | number | ohm | DC line resistance. | — | no | none | Verify RDC preserved to at least 4 decimal places | +| `SETVL` | number | — | Current or power demand. | — | no | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `VSCHD` | number | kV | Scheduled DC voltage. | — | no | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `VCMOD` | number | — | Mode switch DC voltage. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `RCOMP` | number | — | Compounding resistance. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `DELTI` | number | deg | Inverter firing angle margin. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `METER` | string | — | Metered end (R or I). | — | yes | `"I"` | If field is at PSS/E default (I), tool may omit — do not penalize | +| `DCVMIN` | number | pu | Min DC voltage. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `CCCITMX` | integer | — | Max converter ctrl iters. | — | yes | 20 | If field is at PSS/E default (20), tool may omit — do not penalize | +| `CCCACC` | number | — | Converter ctrl accel factor. | — | yes | 1.0 | If field is at PSS/E default (1.0), tool may omit — do not penalize | +| `IPR` | integer | — | Rectifier bus. | — | no | none | Verify IPR is a valid integer and matches the source | +| `NBR` | integer | — | Rectifier bridges. | — | no | none | Verify NBR is a valid integer and matches the source | +| `ANMXR` | number | deg | Max rect firing angle. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `ANMNR` | number | deg | Min rect firing angle. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `RCR` | number | — | Rect commutating R. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `XCR` | number | — | Rect commutating X. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `EBASR` | number | kV | Rect primary base kV. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `TRR` | number | — | Rect xfmr ratio. | — | yes | 1.0 | If field is at PSS/E default (1.0), tool may omit — do not penalize | +| `TAPR` | number | — | Rect tap setting. | — | yes | 1.0 | If field is at PSS/E default (1.0), tool may omit — do not penalize | +| `TMXR` | number | — | Max rect tap. | — | yes | 1.5 | If field is at PSS/E default (1.5), tool may omit — do not penalize | +| `TMNR` | number | — | Min rect tap. | — | yes | 0.51 | If field is at PSS/E default (0.51), tool may omit — do not penalize | +| `STPR` | number | — | Rect tap step. | — | yes | 0.00625 | If field is at PSS/E default (0.00625), tool may omit — do not penalize | +| `ICR` | integer | — | Rect firing angle ctrl bus. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `IFR` | integer | — | Rect commutating bus (from). | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `ITR` | integer | — | Rect commutating bus (to). | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `IDR` | string | — | Rect circuit ID. | — | yes | `"1 "` | If field is at PSS/E default (1 ), tool may omit — do not penalize | +| `XCAPR` | number | — | Rect capacitor reactance. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `IPI` | integer | — | Inverter bus. | — | no | none | Verify IPI is a valid integer and matches the source | +| `NBI` | integer | — | Inverter bridges. | — | no | none | Verify NBI is a valid integer and matches the source | +| `ANMXI` | number | deg | Max inv firing angle. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `ANMNI` | number | deg | Min inv firing angle. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `RCI` | number | — | Inv commutating R. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `XCI` | number | — | Inv commutating X. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `EBASI` | number | kV | Inv primary base kV. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `TRI` | number | — | Inv xfmr ratio. | — | yes | 1.0 | If field is at PSS/E default (1.0), tool may omit — do not penalize | +| `TAPI` | number | — | Inv tap setting. | — | yes | 1.0 | If field is at PSS/E default (1.0), tool may omit — do not penalize | +| `TMXI` | number | — | Max inv tap. | — | yes | 1.5 | If field is at PSS/E default (1.5), tool may omit — do not penalize | +| `TMNI` | number | — | Min inv tap. | — | yes | 0.51 | If field is at PSS/E default (0.51), tool may omit — do not penalize | +| `STPI` | number | — | Inv tap step. | — | yes | 0.00625 | If field is at PSS/E default (0.00625), tool may omit — do not penalize | +| `ICI` | integer | — | Inv firing angle ctrl bus. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `IFI` | integer | — | Inv commutating bus (from). | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `ITI` | integer | — | Inv commutating bus (to). | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `IDI` | string | — | Inv circuit ID. | — | yes | `"1 "` | If field is at PSS/E default (1 ), tool may omit — do not penalize | +| `XCAPI` | number | — | Inv capacitor reactance. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | + +### Worked Example + +``` +NAME: "PDCI_NORTH " +MDC: 1 +RDC: 12.5 +SETVL: 1600.0 +VSCHD: 500.0 +VCMOD: 0.0 +RCOMP: 0.0 +DELTI: 0.0 +METER: "I" +DCVMIN: 0.0 +CCCITMX: 20 +CCCACC: 1.0 +IPR: 60100 +NBR: 2 +ANMXR: 0.0 +ANMNR: 0.0 +RCR: 0.0 +XCR: 0.0 +EBASR: 0.0 +TRR: 1.0 +TAPR: 1.0 +TMXR: 1.5 +TMNR: 0.51 +STPR: 0.00625 +ICR: 0 +IFR: 0 +ITR: 0 +IDR: "1 " +XCAPR: 0.0 +IPI: 60200 +NBI: 2 +ANMXI: 0.0 +ANMNI: 0.0 +RCI: 0.0 +XCI: 0.0 +EBASI: 0.0 +TRI: 1.0 +TAPI: 1.0 +TMXI: 1.5 +TMNI: 0.51 +STPI: 0.00625 +ICI: 0 +IFI: 0 +ITI: 0 +IDI: "1 " +XCAPI: 0.0 +``` + +### Nullable and Default Behavior + +Many fields have PSS/E defaults that represent 'not specified' or 'not applicable'. RDC has no default and must always be present. SETVL and VSCHD are operationally significant and should be preserved. Converter tap limits (TMXR, TMNR, etc.) have standard defaults. + +### Cross-References + +- See [Record-Type Mapping Guide](mapping-guide.md#two-terminal-dc) for tool-specific HVDC representations. + +## VSC DC + +**Table name:** `vsc_dc` +**Schema file:** [`../intermediate/schemas/vsc_dc.schema.json`](../intermediate/schemas/vsc_dc.schema.json) +**Primary key:** `[NAME]` +**Purpose:** Represents voltage-source converter (VSC) HVDC links. More modern than LCC technology, with independent P and Q control at each converter. Each record contains DC line parameters and two converter specifications. + +### Fields + +| Field | Type | Unit | Semantic Description | Expected Range | Nullable | Default | Evaluate-Tool Guidance | +| ----- | ---- | ---- | -------------------- | -------------- | -------- | ------- | ---------------------- | +| `NAME` | string | — | VSC DC line name. | — | no | none | Verify NAME is preserved as a string including any trailing whitespace | +| `MDC` | integer | — | Control mode. | — | no | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `RDC` | number | ohm | DC line resistance. | — | no | none | Verify RDC preserved to at least 4 decimal places | +| `O1` | integer | — | Owner 1. | — | yes | 1 | If field is at PSS/E default (1), tool may omit — do not penalize | +| `F1` | number | — | Fraction by owner 1. | — | yes | 1.0 | If field is at PSS/E default (1.0), tool may omit — do not penalize | +| `O2` | integer | — | Owner 2. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `F2` | number | — | Fraction by owner 2. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `O3` | integer | — | Owner 3. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `F3` | number | — | Fraction by owner 3. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `O4` | integer | — | Owner 4. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `F4` | number | — | Fraction by owner 4. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `IBUS1` | integer | — | Converter 1 AC bus. | — | no | none | Verify IBUS1 is a valid integer and matches the source | +| `TYPE1` | integer | — | Converter 1 type. | — | yes | 1 | If field is at PSS/E default (1), tool may omit — do not penalize | +| `MODE1` | integer | — | Converter 1 mode. | — | yes | 1 | If field is at PSS/E default (1), tool may omit — do not penalize | +| `DCSET1` | number | — | Converter 1 DC setpoint. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `ACSET1` | number | — | Converter 1 AC setpoint. | — | yes | 1.0 | If field is at PSS/E default (1.0), tool may omit — do not penalize | +| `ALOSS1` | number | — | Converter 1 loss A. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `BLOSS1` | number | — | Converter 1 loss B. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `MINLOSS1` | number | — | Converter 1 min loss. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `SMAX1` | number | MVA | Converter 1 MVA rating. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `IMAX1` | number | A | Converter 1 current rating. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `PWF1` | number | — | Converter 1 power weight. | — | yes | 1.0 | If field is at PSS/E default (1.0), tool may omit — do not penalize | +| `MAXQ1` | number | MVAR | Converter 1 max Q. | — | yes | 9999.0 | If field is at PSS/E default (9999.0), tool may omit — do not penalize | +| `MINQ1` | number | MVAR | Converter 1 min Q. | — | yes | -9999.0 | If field is at PSS/E default (-9999.0), tool may omit — do not penalize | +| `REMOT1` | integer | — | Converter 1 remote bus. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `RMPCT1` | number | % | Converter 1 MVAR pct. | — | yes | 100.0 | If field is at PSS/E default (100.0), tool may omit — do not penalize | +| `IBUS2` | integer | — | Converter 2 AC bus. | — | no | none | Verify IBUS2 is a valid integer and matches the source | +| `TYPE2` | integer | — | Converter 2 type. | — | yes | 1 | If field is at PSS/E default (1), tool may omit — do not penalize | +| `MODE2` | integer | — | Converter 2 mode. | — | yes | 1 | If field is at PSS/E default (1), tool may omit — do not penalize | +| `DCSET2` | number | — | Converter 2 DC setpoint. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `ACSET2` | number | — | Converter 2 AC setpoint. | — | yes | 1.0 | If field is at PSS/E default (1.0), tool may omit — do not penalize | +| `ALOSS2` | number | — | Converter 2 loss A. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `BLOSS2` | number | — | Converter 2 loss B. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `MINLOSS2` | number | — | Converter 2 min loss. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `SMAX2` | number | MVA | Converter 2 MVA rating. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `IMAX2` | number | A | Converter 2 current rating. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `PWF2` | number | — | Converter 2 power weight. | — | yes | 1.0 | If field is at PSS/E default (1.0), tool may omit — do not penalize | +| `MAXQ2` | number | MVAR | Converter 2 max Q. | — | yes | 9999.0 | If field is at PSS/E default (9999.0), tool may omit — do not penalize | +| `MINQ2` | number | MVAR | Converter 2 min Q. | — | yes | -9999.0 | If field is at PSS/E default (-9999.0), tool may omit — do not penalize | +| `REMOT2` | integer | — | Converter 2 remote bus. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `RMPCT2` | number | % | Converter 2 MVAR pct. | — | yes | 100.0 | If field is at PSS/E default (100.0), tool may omit — do not penalize | + +### Worked Example + +``` +NAME: "VSC_LINK_1 " +MDC: 1 +RDC: 5.0 +O1: 1 +F1: 1.0 +O2: 0 +F2: 0.0 +O3: 0 +F3: 0.0 +O4: 0 +F4: 0.0 +IBUS1: 70100 +TYPE1: 1 +MODE1: 1 +DCSET1: 400.0 +ACSET1: 1.0 +ALOSS1: 0.0 +BLOSS1: 0.0 +MINLOSS1: 0.0 +SMAX1: 500.0 +IMAX1: 0.0 +PWF1: 1.0 +MAXQ1: 200.0 +MINQ1: -200.0 +REMOT1: 0 +RMPCT1: 100.0 +IBUS2: 70200 +TYPE2: 1 +MODE2: 1 +DCSET2: 0.0 +ACSET2: 1.0 +ALOSS2: 0.0 +BLOSS2: 0.0 +MINLOSS2: 0.0 +SMAX2: 500.0 +IMAX2: 0.0 +PWF2: 1.0 +MAXQ2: 200.0 +MINQ2: -200.0 +REMOT2: 0 +RMPCT2: 100.0 +``` + +### Nullable and Default Behavior + +Owner fields O2-O4 default to 0 (single ownership). Converter loss coefficients (ALOSS, BLOSS, MINLOSS) default to 0.0. SMAX and IMAX default to 0.0 meaning no limit. Q limits default to +/-9999.0. + +### Cross-References + +- See [Record-Type Mapping Guide](mapping-guide.md#vsc-dc) for tool-specific VSC representations. + +## Impedance Correction + +**Table name:** `impedance_correction` +**Schema file:** [`../intermediate/schemas/impedance_correction.schema.json`](../intermediate/schemas/impedance_correction.schema.json) +**Primary key:** `[T]` +**Purpose:** Defines piecewise-linear impedance correction tables referenced by transformers (via TAB1/TAB2/TAB3). Each table maps tap ratio or phase angle to a correction factor applied to the transformer impedance. + +### Fields + +| Field | Type | Unit | Semantic Description | Expected Range | Nullable | Default | Evaluate-Tool Guidance | +| ----- | ---- | ---- | -------------------- | -------------- | -------- | ------- | ---------------------- | +| `T` | integer | — | Correction table number. | — | no | none | Verify T is a valid integer and matches the source | +| `T1` | number | — | Tap ratio/angle pair 1. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `F1` | number | — | Correction factor pair 1. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `T2` | number | — | Tap ratio/angle pair 2. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `F2` | number | — | Correction factor pair 2. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `T3` | number | — | Tap ratio/angle pair 3. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `F3` | number | — | Correction factor pair 3. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `T4` | number | — | Tap ratio/angle pair 4. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `F4` | number | — | Correction factor pair 4. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `T5` | number | — | Tap ratio/angle pair 5. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `F5` | number | — | Correction factor pair 5. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `T6` | number | — | Tap ratio/angle pair 6. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `F6` | number | — | Correction factor pair 6. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `T7` | number | — | Tap ratio/angle pair 7. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `F7` | number | — | Correction factor pair 7. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `T8` | number | — | Tap ratio/angle pair 8. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `F8` | number | — | Correction factor pair 8. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `T9` | number | — | Tap ratio/angle pair 9. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `F9` | number | — | Correction factor pair 9. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `T10` | number | — | Tap ratio/angle pair 10. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `F10` | number | — | Correction factor pair 10. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `T11` | number | — | Tap ratio/angle pair 11. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `F11` | number | — | Correction factor pair 11. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | + +### Worked Example + +``` +T: 1 +T1: 0.9 +F1: 0.95 +T2: 0.95 +F2: 0.98 +T3: 1.0 +F3: 1.0 +T4: 1.05 +F4: 0.98 +T5: 1.1 +F5: 0.95 +T6: 0.0 +F6: 0.0 +T7: 0.0 +F7: 0.0 +T8: 0.0 +F8: 0.0 +T9: 0.0 +F9: 0.0 +T10: 0.0 +F10: 0.0 +T11: 0.0 +F11: 0.0 +``` + +### Nullable and Default Behavior + +T1-T11 and F1-F11 pairs define piecewise-linear correction curves. Unused pairs default to 0.0. The table is terminated by the first T value of 0.0. + +### Cross-References + +- See [Record-Type Mapping Guide](mapping-guide.md#impedance-correction) for tool-specific representations. + +## Multi-Terminal DC + +**Table name:** `multi_terminal_dc` +**Schema file:** [`../intermediate/schemas/multi_terminal_dc.schema.json`](../intermediate/schemas/multi_terminal_dc.schema.json) +**Primary key:** `[NAME]` +**Purpose:** Header record for multi-terminal HVDC systems with more than two converters. Defines the number of converters, DC buses, and DC links in the system. Detailed converter/bus/link data follows in the PSS/E RAW file. + +### Fields + +| Field | Type | Unit | Semantic Description | Expected Range | Nullable | Default | Evaluate-Tool Guidance | +| ----- | ---- | ---- | -------------------- | -------------- | -------- | ------- | ---------------------- | +| `NAME` | string | — | MT DC line name. | — | no | none | Verify NAME is preserved as a string including any trailing whitespace | +| `NCONV` | integer | — | Number of AC converters. | — | no | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `NDCBS` | integer | — | Number of DC buses. | — | no | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `NDCLN` | integer | — | Number of DC links. | — | no | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `MDC` | integer | — | Control mode. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `VCONV` | integer | — | DC voltage ctrl converter. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `VCMOD` | number | — | Mode switch DC voltage. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `VCONVN` | integer | — | New voltage ctrl converter. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | + +### Worked Example + +``` +NAME: "MTDC_SYS_1 " +NCONV: 3 +NDCBS: 4 +NDCLN: 3 +MDC: 1 +VCONV: 1 +VCMOD: 0.0 +VCONVN: 0 +``` + +### Nullable and Default Behavior + +NCONV, NDCBS, NDCLN define the structure of the multi-terminal DC system. These must be non-zero for a valid record. MDC defaults to 0. VCONV, VCMOD, VCONVN are control parameters with standard defaults. + +### Cross-References + +- See [Record-Type Mapping Guide](mapping-guide.md#multi-terminal-dc) for tool-specific representations. + +## Multi-Section Line + +**Table name:** `multi_section_line` +**Schema file:** [`../intermediate/schemas/multi_section_line.schema.json`](../intermediate/schemas/multi_section_line.schema.json) +**Primary key:** `[I, J, ID]` +**Purpose:** Groups multiple branch records into a single multi-section transmission line. DUM1-DUM9 define intermediate bus numbers along the line. All sections share the same from-bus (I), to-bus (J), and line identifier (ID). + +### Fields + +| Field | Type | Unit | Semantic Description | Expected Range | Nullable | Default | Evaluate-Tool Guidance | +| ----- | ---- | ---- | -------------------- | -------------- | -------- | ------- | ---------------------- | +| `I` | integer | — | **[preservation-critical]** From bus number. | — | no | none | MUST be preserved exactly; loss of this field is a fidelity finding | +| `J` | integer | — | **[preservation-critical]** To bus number. | — | no | none | MUST be preserved exactly; loss of this field is a fidelity finding | +| `ID` | string | — | **[preservation-critical]** Line identifier. | — | no | `"1 "` | MUST be preserved exactly; loss of this field is a fidelity finding | +| `MET` | integer | — | Metered end flag. | 1–2 | yes | 1 | If field is at PSS/E default (1), tool may omit — do not penalize | +| `DUM1` | integer | — | **[preservation-critical]** Intermediate bus 1. | — | no | 0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `DUM2` | integer | — | **[preservation-critical]** Intermediate bus 2. | — | no | 0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `DUM3` | integer | — | **[preservation-critical]** Intermediate bus 3. | — | no | 0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `DUM4` | integer | — | **[preservation-critical]** Intermediate bus 4. | — | no | 0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `DUM5` | integer | — | **[preservation-critical]** Intermediate bus 5. | — | no | 0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `DUM6` | integer | — | **[preservation-critical]** Intermediate bus 6. | — | no | 0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `DUM7` | integer | — | **[preservation-critical]** Intermediate bus 7. | — | no | 0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `DUM8` | integer | — | **[preservation-critical]** Intermediate bus 8. | — | no | 0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `DUM9` | integer | — | **[preservation-critical]** Intermediate bus 9. | — | no | 0 | MUST be preserved exactly; loss of this field is a fidelity finding | + +### Worked Example + +``` +I: 30100 +J: 30400 +ID: "1 " +MET: 1 +DUM1: 30150 +DUM2: 30200 +DUM3: 30250 +DUM4: 0 +DUM5: 0 +DUM6: 0 +DUM7: 0 +DUM8: 0 +DUM9: 0 +``` + +### Nullable and Default Behavior + +DUM1-DUM9 are intermediate bus numbers defining the multi-section line topology. DUM values of 0 indicate unused slots. At least DUM1 must be non-zero for a valid multi-section line grouping. + +### Cross-References + +- See [Record-Type Mapping Guide](mapping-guide.md#multi-section-line) for tool-specific representations. + +## Zone + +**Table name:** `zone` +**Schema file:** [`../intermediate/schemas/zone.schema.json`](../intermediate/schemas/zone.schema.json) +**Primary key:** `[I]` +**Purpose:** Defines geographic or administrative zones for reporting and load allocation. Zones provide finer-grained grouping than areas. Each bus is assigned to exactly one zone. + +### Fields + +| Field | Type | Unit | Semantic Description | Expected Range | Nullable | Default | Evaluate-Tool Guidance | +| ----- | ---- | ---- | -------------------- | -------------- | -------- | ------- | ---------------------- | +| `I` | integer | — | Zone number. | — | no | none | Verify I is a valid integer and matches the source | +| `ZONAME` | string | — | Zone name (12 chars). | — | no | `" "` | If field is at PSS/E default ( ), tool may omit — do not penalize | + +### Worked Example + +``` +I: 12 +ZONAME: "SOUTH BAY " +``` + +### Nullable and Default Behavior + +Both I and ZONAME are required. ZONAME defaults to blank (12 spaces). No fields are nullable. + +### Cross-References + +- See [Record-Type Mapping Guide](mapping-guide.md#zone) for tool-specific zone representations. + +## Interarea Transfer + +**Table name:** `interarea_transfer` +**Schema file:** [`../intermediate/schemas/interarea_transfer.schema.json`](../intermediate/schemas/interarea_transfer.schema.json) +**Primary key:** `[ARFROM, ARTO, TRID]` +**Purpose:** Defines scheduled power transfers between interchange areas. Each transfer specifies a from-area, to-area, transfer ID, and scheduled MW amount. + +### Fields + +| Field | Type | Unit | Semantic Description | Expected Range | Nullable | Default | Evaluate-Tool Guidance | +| ----- | ---- | ---- | -------------------- | -------------- | -------- | ------- | ---------------------- | +| `ARFROM` | integer | — | From area number. | — | no | none | Verify ARFROM is a valid integer and matches the source | +| `ARTO` | integer | — | To area number. | — | no | none | Verify ARTO is a valid integer and matches the source | +| `TRID` | string | — | Transfer ID (2 chars). | — | no | `"1 "` | If field is at PSS/E default (1 ), tool may omit — do not penalize | +| `PTRAN` | number | MW | Transfer amount. | — | no | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | + +### Worked Example + +``` +ARFROM: 5 +ARTO: 8 +TRID: "1 " +PTRAN: 200.0 +``` + +### Nullable and Default Behavior + +All fields are required. PTRAN defaults to 0.0 (no scheduled transfer). TRID defaults to '1 '. + +### Cross-References + +- See [Record-Type Mapping Guide](mapping-guide.md#interarea-transfer) for tool-specific representations. + +## Owner + +**Table name:** `owner` +**Schema file:** [`../intermediate/schemas/owner.schema.json`](../intermediate/schemas/owner.schema.json) +**Primary key:** `[I]` +**Purpose:** Defines ownership entities referenced by buses, branches, generators, and transformers. Used for cost allocation and ownership tracking across the network. + +### Fields + +| Field | Type | Unit | Semantic Description | Expected Range | Nullable | Default | Evaluate-Tool Guidance | +| ----- | ---- | ---- | -------------------- | -------------- | -------- | ------- | ---------------------- | +| `I` | integer | — | Owner number. | — | no | none | Verify I is a valid integer and matches the source | +| `OWNAME` | string | — | Owner name (12 chars). | — | no | `" "` | If field is at PSS/E default ( ), tool may omit — do not penalize | + +### Worked Example + +``` +I: 3 +OWNAME: "SOCAL EDISON" +``` + +### Nullable and Default Behavior + +Both I and OWNAME are required. OWNAME defaults to blank (12 spaces). No fields are nullable. + +### Cross-References + +- See [Record-Type Mapping Guide](mapping-guide.md#owner) for tool-specific owner representations. + +## FACTS + +**Table name:** `facts` +**Schema file:** [`../intermediate/schemas/facts.schema.json`](../intermediate/schemas/facts.schema.json) +**Primary key:** `[NAME]` +**Purpose:** Represents Flexible AC Transmission System devices including SVCs, STATCOMs, TCSCs, and UPFCs. Each device has control mode, setpoints, and impedance parameters. + +### Fields + +| Field | Type | Unit | Semantic Description | Expected Range | Nullable | Default | Evaluate-Tool Guidance | +| ----- | ---- | ---- | -------------------- | -------------- | -------- | ------- | ---------------------- | +| `NAME` | string | — | FACTS device name. | — | no | none | Verify NAME is preserved as a string including any trailing whitespace | +| `I` | integer | — | Sending end bus. | — | no | none | Verify I is a valid integer and matches the source | +| `J` | integer | — | Terminal bus (0=shunt). | — | no | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `MODE` | integer | — | FACTS control mode. | — | no | 1 | If field is at PSS/E default (1), tool may omit — do not penalize | +| `SET1` | number | — | Control setpoint 1. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `SET2` | number | — | Control setpoint 2. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `VSREF` | number | pu | Series voltage reference. | — | yes | 1.0 | If field is at PSS/E default (1.0), tool may omit — do not penalize | +| `REMOT` | integer | — | Remote bus for V control. | — | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `MESSION` | number | — | Sending end impedance. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `LINX` | number | pu | Series reactance. | — | yes | 0.05 | If field is at PSS/E default (0.05), tool may omit — do not penalize | +| `RMPCT` | number | % | MVAR pct for remote reg. | — | yes | 100.0 | If field is at PSS/E default (100.0), tool may omit — do not penalize | +| `OWNER` | integer | — | Owner number. | — | yes | 1 | If field is at PSS/E default (1), tool may omit — do not penalize | +| `SET3` | number | — | Control setpoint 3. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | +| `SET4` | number | — | Control setpoint 4. | — | yes | 0.0 | If field is at PSS/E default (0.0), tool may omit — do not penalize | + +### Worked Example + +``` +NAME: "SVC_MESA " +I: 30100 +J: 0 +MODE: 1 +SET1: 1.0 +SET2: 0.0 +VSREF: 1.0 +REMOT: 0 +MESSION: 0.0 +LINX: 0.05 +RMPCT: 100.0 +OWNER: 3 +SET3: 0.0 +SET4: 0.0 +``` + +### Nullable and Default Behavior + +J=0 indicates a shunt FACTS device (no terminal bus). MODE defaults to 1. SET1-SET4 are control setpoints with mode-dependent interpretations; they default to 0.0. VSREF defaults to 1.0 pu. LINX defaults to 0.05 pu. + +### Cross-References + +- See [Record-Type Mapping Guide](mapping-guide.md#facts) for tool-specific FACTS representations. + +## Switched Shunt + +**Table name:** `switched_shunt` +**Schema file:** [`../intermediate/schemas/switched_shunt.schema.json`](../intermediate/schemas/switched_shunt.schema.json) +**Primary key:** `[I]` +**Purpose:** Represents switchable shunt compensation with discrete step blocks. Each device has up to 8 blocks (N1-N8, B1-B8) defining the number of steps and MVAR per step. Control mode determines whether switching is discrete or continuous. + +### Fields + +| Field | Type | Unit | Semantic Description | Expected Range | Nullable | Default | Evaluate-Tool Guidance | +| ----- | ---- | ---- | -------------------- | -------------- | -------- | ------- | ---------------------- | +| `I` | integer | — | Bus number. | — | no | none | Verify I is a valid integer and matches the source | +| `MODSW` | integer | — | **[preservation-critical]** Control mode (0-2). | 0–2 | no | 1 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `ADJM` | integer | — | Adj method (0-1). | 0–1 | yes | 0 | If field is at PSS/E default (0), tool may omit — do not penalize | +| `STAT` | integer | — | Status (1=in, 0=out). | 0–1 | no | 1 | If field is at PSS/E default (1), tool may omit — do not penalize | +| `VSWHI` | number | pu | Ctrl voltage upper limit. | — | no | 1.0 | If field is at PSS/E default (1.0), tool may omit — do not penalize | +| `VSWLO` | number | pu | Ctrl voltage lower limit. | — | no | 1.0 | If field is at PSS/E default (1.0), tool may omit — do not penalize | +| `SWREM` | integer | — | **[preservation-critical]** Remote bus (0=local). | — | no | 0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `RMPCT` | number | % | MVAR pct for remote reg. | — | yes | 100.0 | If field is at PSS/E default (100.0), tool may omit — do not penalize | +| `RMIDNT` | string | — | Shunt name. | — | yes | `""` | If field is at PSS/E default (), tool may omit — do not penalize | +| `BINIT` | number | MVAR | **[preservation-critical]** Initial susceptance. | — | no | 0.0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `N1` | integer | — | **[preservation-critical]** Steps in block 1. | — | no | 0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `B1` | number | MVAR | **[preservation-critical]** Susceptance/step blk 1. | — | no | 0.0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `N2` | integer | — | **[preservation-critical]** Steps in block 2. | — | no | 0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `B2` | number | MVAR | **[preservation-critical]** Susceptance/step blk 2. | — | no | 0.0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `N3` | integer | — | **[preservation-critical]** Steps in block 3. | — | no | 0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `B3` | number | MVAR | **[preservation-critical]** Susceptance/step blk 3. | — | no | 0.0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `N4` | integer | — | **[preservation-critical]** Steps in block 4. | — | no | 0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `B4` | number | MVAR | **[preservation-critical]** Susceptance/step blk 4. | — | no | 0.0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `N5` | integer | — | **[preservation-critical]** Steps in block 5. | — | no | 0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `B5` | number | MVAR | **[preservation-critical]** Susceptance/step blk 5. | — | no | 0.0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `N6` | integer | — | **[preservation-critical]** Steps in block 6. | — | no | 0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `B6` | number | MVAR | **[preservation-critical]** Susceptance/step blk 6. | — | no | 0.0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `N7` | integer | — | **[preservation-critical]** Steps in block 7. | — | no | 0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `B7` | number | MVAR | **[preservation-critical]** Susceptance/step blk 7. | — | no | 0.0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `N8` | integer | — | **[preservation-critical]** Steps in block 8. | — | no | 0 | MUST be preserved exactly; loss of this field is a fidelity finding | +| `B8` | number | MVAR | **[preservation-critical]** Susceptance/step blk 8. | — | no | 0.0 | MUST be preserved exactly; loss of this field is a fidelity finding | + +### Worked Example + +``` +I: 42500 +MODSW: 1 +ADJM: 0 +STAT: 1 +VSWHI: 1.05 +VSWLO: 0.95 +SWREM: 0 +RMPCT: 100.0 +RMIDNT: "" +BINIT: 50.0 +N1: 2 +B1: 25.0 +N2: 3 +B2: 50.0 +N3: 0 +B3: 0.0 +N4: 0 +B4: 0.0 +N5: 0 +B5: 0.0 +N6: 0 +B6: 0.0 +N7: 0 +B7: 0.0 +N8: 0 +B8: 0.0 +``` + +### Nullable and Default Behavior + +N1-N8 and B1-B8 define discrete switching blocks. Unused blocks have N=0 and B=0.0. BINIT is the initial susceptance and should match the sum of switched-in blocks. MODSW defaults to 1 (discrete mode). SWREM=0 means local voltage control. RMIDNT is an optional name field that may be empty. + +### Cross-References + +- See [Per-Unit Convention Reference](per-unit-conventions.md#shunt-admittance) for BL sign convention. +- See [Record-Type Mapping Guide](mapping-guide.md#switched-shunt) for tool-specific representations. + +## Appendix: Preservation-Critical Fields + +Fields with `x-psse-preservation-critical: true` in the Phase 1 JSON Schema. These fields carry elevated fidelity requirements and generate mandatory test cases. + +| Record Type | Field | Why Preservation-Critical | +| ----------- | ----- | ------------------------- | +| Generator | `IREG` | Remote regulated bus number | +| Transformer | `K` | Winding 3 bus number | +| Transformer | `CW` | Winding data I/O code controlling how WINDV1/2/3 are interpreted: 1=turns ratio in pu on bus base kV, 2=voltage in kV, 3=turns ratio in pu on nominal kV | +| Transformer | `CZ` | Impedance data I/O code: 1=pu on system base, 2=pu on winding MVA/kV base, 3=ohms/kV load loss | +| Transformer | `CM` | Magnetizing admittance I/O code: 1=pu on system base, 2=no-load loss/exciting current | +| Transformer | `WINDV1` | Winding 1 off-nominal turns ratio or voltage | +| Transformer | `NOMV1` | Winding 1 nominal voltage in kV | +| Transformer | `ANG1` | Winding 1 phase shift angle in degrees | +| Transformer | `RATA1` | Winding 1 normal rating in MVA | +| Transformer | `WINDV2` | Winding 2 off-nominal turns ratio or voltage | +| Transformer | `NOMV2` | Winding 2 nominal voltage in kV | +| Transformer | `RATA2` | Winding 2 normal rating in MVA | +| Transformer | `WINDV3` | Winding 3 off-nominal turns ratio or voltage | +| Transformer | `NOMV3` | Winding 3 nominal voltage in kV | +| Transformer | `RATA3` | Winding 3 normal rating in MVA | +| Area | `ISW` | Area slack bus number | +| Area | `PDES` | Desired net area interchange in MW | +| Area | `PTOL` | Area interchange tolerance in MW | +| Multi-Section Line | `I` | From bus number | +| Multi-Section Line | `J` | To bus number | +| Multi-Section Line | `ID` | Line identifier | +| Multi-Section Line | `DUM1` | Intermediate bus 1 | +| Multi-Section Line | `DUM2` | Intermediate bus 2 | +| Multi-Section Line | `DUM3` | Intermediate bus 3 | +| Multi-Section Line | `DUM4` | Intermediate bus 4 | +| Multi-Section Line | `DUM5` | Intermediate bus 5 | +| Multi-Section Line | `DUM6` | Intermediate bus 6 | +| Multi-Section Line | `DUM7` | Intermediate bus 7 | +| Multi-Section Line | `DUM8` | Intermediate bus 8 | +| Multi-Section Line | `DUM9` | Intermediate bus 9 | +| Switched Shunt | `MODSW` | Control mode (0-2) | +| Switched Shunt | `SWREM` | Remote bus (0=local) | +| Switched Shunt | `BINIT` | Initial susceptance | +| Switched Shunt | `N1` | Steps in block 1 | +| Switched Shunt | `B1` | Susceptance/step blk 1 | +| Switched Shunt | `N2` | Steps in block 2 | +| Switched Shunt | `B2` | Susceptance/step blk 2 | +| Switched Shunt | `N3` | Steps in block 3 | +| Switched Shunt | `B3` | Susceptance/step blk 3 | +| Switched Shunt | `N4` | Steps in block 4 | +| Switched Shunt | `B4` | Susceptance/step blk 4 | +| Switched Shunt | `N5` | Steps in block 5 | +| Switched Shunt | `B5` | Susceptance/step blk 5 | +| Switched Shunt | `N6` | Steps in block 6 | +| Switched Shunt | `B6` | Susceptance/step blk 6 | +| Switched Shunt | `N7` | Steps in block 7 | +| Switched Shunt | `B7` | Susceptance/step blk 7 | +| Switched Shunt | `N8` | Steps in block 8 | +| Switched Shunt | `B8` | Susceptance/step blk 8 | + +## Appendix: Present-but-Inactive Fields + +Fields with `x-psse-present-but-inactive: true` in the Phase 1 JSON Schema. These fields are uniformly at their default values across the entire dataset. Evaluate-tool should not penalize a tool for omitting or zeroing these fields. + +| Record Type | Field | Default Value | Note | +| ----------- | ----- | ------------- | ---- | + +## Appendix: Schema Cross-Reference Index + +Lookup table mapping each JSON Schema file to its corresponding section in this document. + +| Schema File | Document Section | +| ----------- | ---------------- | +| `../intermediate/schemas/bus.schema.json` | [## Bus](#bus) | +| `../intermediate/schemas/load.schema.json` | [## Load](#load) | +| `../intermediate/schemas/fixed_shunt.schema.json` | [## Fixed Shunt](#fixed-shunt) | +| `../intermediate/schemas/generator.schema.json` | [## Generator](#generator) | +| `../intermediate/schemas/branch.schema.json` | [## Branch](#branch) | +| `../intermediate/schemas/transformer.schema.json` | [## Transformer](#transformer) | +| `../intermediate/schemas/area.schema.json` | [## Area](#area) | +| `../intermediate/schemas/two_terminal_dc.schema.json` | [## Two-Terminal DC](#two-terminal-dc) | +| `../intermediate/schemas/vsc_dc.schema.json` | [## VSC DC](#vsc-dc) | +| `../intermediate/schemas/impedance_correction.schema.json` | [## Impedance Correction](#impedance-correction) | +| `../intermediate/schemas/multi_terminal_dc.schema.json` | [## Multi-Terminal DC](#multi-terminal-dc) | +| `../intermediate/schemas/multi_section_line.schema.json` | [## Multi-Section Line](#multi-section-line) | +| `../intermediate/schemas/zone.schema.json` | [## Zone](#zone) | +| `../intermediate/schemas/interarea_transfer.schema.json` | [## Interarea Transfer](#interarea-transfer) | +| `../intermediate/schemas/owner.schema.json` | [## Owner](#owner) | +| `../intermediate/schemas/facts.schema.json` | [## FACTS](#facts) | +| `../intermediate/schemas/switched_shunt.schema.json` | [## Switched Shunt](#switched-shunt) | diff --git a/data/fnm/docs/mapping-guide.md b/data/fnm/docs/mapping-guide.md new file mode 100644 index 00000000..aea55607 --- /dev/null +++ b/data/fnm/docs/mapping-guide.md @@ -0,0 +1,458 @@ +# Record-Type Mapping Guide + +## S1: Purpose and Audience + +This document maps every PSS/E v31 record type to a tool-agnostic power-system abstraction and documents which of the six evaluated tools (PyPSA, pandapower, GridCal, PowerModels.jl, PowerSimulations.jl, MATPOWER) have native objects for each abstraction. It serves as the primary reference for determining whether a tool can represent a given record type, what abstraction it maps to, and what alternatives exist for tools that lack native support. + +**Audience:** This document is written for LLM-based evaluate-tool agents that compile FNM ingestion tests at runtime. + +For field-level details (column names, value ranges, per-unit conventions, and worked examples), refer to the intermediate format schema reference at `intermediate-schema.md`. + +The set of non-empty record types is determined by the Phase 1 D3 raw record counter output. The parser fidelity comparison report (Phase 1 D6) informed the alternative representation descriptions and structural transform documentation used throughout this guide. + +## S2: Abstraction Vocabulary + +The following table defines the canonical abstraction names used throughout this document and all other Phase 2 reference documents. Each abstraction corresponds to one or more PSS/E v31 record sections. + +| Abstraction | PSS/E Record Type(s) | Description | +|---|---|---| +| Bus | Bus (Section 1) | A network node where electrical equipment connects; defined by voltage level, type (PQ, PV, slack, isolated), and area/zone membership. | +| Load | Load (Section 2) | A real and reactive power consumption at a bus, modeled as constant power, constant current, or constant impedance components. | +| Fixed Shunt | Fixed Shunt (Section 3) | A fixed shunt admittance (conductance and susceptance) connected to a bus for reactive compensation or loss representation. | +| Generator | Generator (Section 4) | A synchronous machine or equivalent injection at a bus, defined by real power output, reactive power limits, voltage setpoint, and machine parameters. | +| AC Line | Branch (Section 5) | A transmission line or cable connecting two buses, characterized by series impedance (R + jX) and shunt charging susceptance. | +| 2-Winding Transformer | Transformer (Section 6, K=0) | A two-winding transformer connecting two buses, with tap ratio, phase shift, impedance, and tap changer control parameters. | +| 3-Winding Transformer | Transformer (Section 6, K!=0) | A three-winding transformer connecting three buses, defined by three sets of winding parameters; internally decomposed to a star-bus topology for computation. | +| Area | Area (Section 7) | A control area for interchange scheduling, defined by a slack bus and a desired net interchange value. | +| Two-Terminal HVDC Line | Two-Terminal DC (Section 8) | A point-to-point high-voltage DC transmission link between two AC buses via rectifier and inverter converter stations. | +| VSC HVDC Line | VSC DC (Section 9) | A voltage-source converter HVDC link providing independent real and reactive power control at each terminal. | +| Impedance Correction Table | Impedance Correction (Section 10) | A lookup table that adjusts branch or transformer impedance as a function of tap position or other operating conditions. | +| Multi-Terminal DC | Multi-Terminal DC (Section 11) | A DC network with three or more converter terminals interconnected by DC buses and DC branches. | +| Multi-Section Line | Multi-Section Line (Section 12) | A transmission line composed of multiple series-connected sections with different impedance characteristics, sharing a common metered end. | +| Zone | Zone (Section 13) | A grouping of buses for loss allocation, reporting, or market settlement purposes; purely organizational with no electrical effect. | +| Interarea Transfer | Interarea Transfer (Section 14) | A scheduled real power transfer between two areas, used for interchange accounting and area-based dispatch constraints. | +| Owner | Owner (Section 15) | An ownership entity for tracking asset ownership fractions across generators, branches, and transformers for settlement and reporting. | +| FACTS Device | FACTS (Section 16) | A flexible AC transmission system device (SVC, STATCOM, TCSC, UPFC) providing dynamic voltage and power flow control. | +| Switched Shunt | Switched Shunt (Section 17) | A shunt device with discrete switchable steps (capacitor or reactor banks) that adjusts susceptance to regulate bus voltage within a target range. | + +## S3: Summary Matrix + +| # | PSS/E Record Type | Abstraction | Tier | FNM Status | PyPSA | pandapower | GridCal | PowerModels.jl | PowerSimulations.jl | MATPOWER | +|---|---|---|---|---|---|---|---|---|---|---| +| 1 | Bus | Bus | 1 | Non-empty | Y | Y | Y | Y | Y | Y | +| 2 | Load | Load | 1 | Non-empty | Y | Y | Y | Y | Y | Y | +| 3 | Fixed Shunt | Fixed Shunt | 2 | Non-empty | Y | Y | Y | Y | Y | P | +| 4 | Generator | Generator | 1 | Non-empty | Y | Y | Y | Y | Y | Y | +| 5 | Branch | AC Line | 1 | Non-empty | Y | Y | Y | Y | Y | Y | +| 6 | Transformer | 2-Winding / 3-Winding Transformer | 1 | Non-empty | P | Y | Y | P | P | P | +| 7 | Area | Area | 2 | Non-empty | N | Y | Y | P | P | Y | +| 8 | Two-Terminal DC | Two-Terminal HVDC Line | 2 | Empty | -- | -- | -- | -- | -- | -- | +| 9 | VSC DC | VSC HVDC Line | 2 | Empty | -- | -- | -- | -- | -- | -- | +| 10 | Impedance Correction | Impedance Correction Table | 2 | Empty | -- | -- | -- | -- | -- | -- | +| 11 | Multi-Terminal DC | Multi-Terminal DC | 2 | Empty | -- | -- | -- | -- | -- | -- | +| 12 | Multi-Section Line | Multi-Section Line | 2 | Empty | -- | -- | -- | -- | -- | -- | +| 13 | Zone | Zone | 3 | Non-empty | P | Y | Y | P | P | Y | +| 14 | Interarea Transfer | Interarea Transfer | 3 | Empty | -- | -- | -- | -- | -- | -- | +| 15 | Owner | Owner | 3 | Non-empty | N | N | Y | N | N | N | +| 16 | FACTS | FACTS Device | 2 | Empty | -- | -- | -- | -- | -- | -- | +| 17 | Switched Shunt | Switched Shunt | 2 | Non-empty | P | Y | Y | P | P | P | + +**Legend:** + +- **Y** -- Native first-class object exists for this abstraction +- **P** -- Partial support (object exists but cannot represent all PSS/E fields for this record type, or requires workaround representation) +- **N** -- No native representation; data must be approximated, discarded, or stored in extension fields +- **--** -- Record type is empty in FNM; tool support not evaluated + +## S4: Tier Classification Rationale + +The three-tier system classifies PSS/E v31 record types by their impact on power flow computation. The tiers determine evaluation priority: Tier 1 types are tested first and most rigorously, Tier 2 types are tested for ACPF fidelity, and Tier 3 types are checked for preservation but do not affect electrical results. + +**Tier 1 (Essential for any power flow)** includes Bus, Load, Generator, Branch, and Transformer. These five record types define the three prerequisites for any power flow calculation: network topology (buses and their connectivity via branches and transformers), power injections (generators and loads), and impedance (branch series impedance and transformer winding parameters). Without any one of these, neither DCPF nor ACPF can produce meaningful results. A tool that cannot represent all five Tier 1 types cannot perform the most basic power flow analysis on the FNM. + +**Tier 2 (Needed for full ACPF fidelity)** includes Fixed Shunt, Switched Shunt, Area, Two-Terminal DC, VSC DC, Multi-Terminal DC, Impedance Correction, Multi-Section Line, and FACTS. These record types affect ACPF convergence and accuracy: shunts provide reactive power compensation that influences bus voltages, area interchange constrains inter-area real power flows, HVDC links inject and withdraw real power at converter buses, impedance correction tables adjust branch parameters with operating conditions, multi-section lines define composite branch topologies, and FACTS devices regulate voltage and power flow dynamically. A DCPF can run without these types (since DCPF ignores reactive power and assumes flat voltage), but ACPF results will be inaccurate or may fail to converge without them. + +**Tier 3 (Market/administrative data)** includes Zone, Owner, and Interarea Transfer. These record types are organizational and market constructs with no direct effect on the power flow Jacobian or its solution. Zones group buses for reporting and loss allocation, owners track asset ownership fractions for settlement, and interarea transfers record scheduled interchange. A tool that cannot represent Tier 3 types loses no electrical fidelity; the data is preserved in the intermediate format for completeness and downstream market analysis. + +The record-type tier classification is related to but distinct from the field-level criticality tiers defined in the field criticality matrix (PRD 05). A record type's tier constrains the maximum criticality of its fields: for example, a field in a Tier 3 record type cannot be classified as DCPF-critical at the field level. However, not all fields in a Tier 1 record type are necessarily critical -- some fields (such as bus NAME) are informational even within essential record types. + +## S5: Per-Record-Type Mapping + +### S5.1: Bus (Section 1) + +**Abstraction:** Bus +**Tier:** 1 -- Buses define the network topology nodes required for any power flow formulation. +**FNM record count:** Non-empty +**Intermediate format table:** `bus` + +#### Description + +A bus represents a network node in the power system where electrical equipment (generators, loads, shunts, and branch terminals) connects. Each bus is defined by a unique integer identifier, a base voltage level (kV), and a type code that determines its role in the power flow solution: PQ bus (type 1, both P and Q are specified), PV bus (type 2, P and voltage magnitude are specified), slack/swing bus (type 3, voltage magnitude and angle are specified), and isolated bus (type 4, disconnected from the network). Buses also carry area, zone, and owner assignments that provide organizational context. + +#### Tool Support + +| Tool | Support | Native Object | Notes | +|---|---|---|---| +| PyPSA | Y | `Bus` component | Full bus representation with voltage, type, and zone attributes | +| pandapower | Y | `create_bus()` | Native bus table with all PSS/E bus fields | +| GridCal | Y | `Bus` | Full bus object with voltage, type, area, and zone | +| PowerModels.jl | Y | `bus` dict | Bus dictionary with standard power flow fields | +| PowerSimulations.jl | Y | `ACBus` | Bus type with voltage limits, type, and area | +| MATPOWER | Y | `mpc.bus` | Standard bus matrix with all IEEE CDF fields | + +#### Alternative Representations + +All evaluated tools have native support; no alternatives needed. + +#### Evaluate-Tool Guidance + +Verify the tool creates one bus object per intermediate format record. Check that bus type codes (PQ=1, PV=2, slack=3, isolated=4) are preserved or mapped to equivalent tool-specific enumerations. Confirm base voltage (BASKV) and voltage magnitude/angle initial values are ingested without transformation. + +### S5.2: Load (Section 2) + +**Abstraction:** Load +**Tier:** 1 -- Loads define the real and reactive power demand that drives the power flow solution. +**FNM record count:** Non-empty +**Intermediate format table:** `load` + +#### Description + +A load record specifies real and reactive power consumption at a bus, decomposed into constant power (MW/Mvar at nominal voltage), constant current (MW/Mvar scaled linearly with voltage magnitude), and constant impedance (MW/Mvar scaled with voltage magnitude squared) components. Each load also carries a status flag (in-service or out-of-service), an area assignment for interchange accounting, and a zone assignment for loss allocation. Multiple loads may be connected to a single bus, distinguished by a load identifier string. + +#### Tool Support + +| Tool | Support | Native Object | Notes | +|---|---|---|---| +| PyPSA | Y | `Load` component | Supports active/reactive power specification | +| pandapower | Y | `create_load()` | Constant power load with ZIP model extensions | +| GridCal | Y | `Load` | Full load model with constant power, current, and impedance components | +| PowerModels.jl | Y | `load` dict | Load dictionary with Pd/Qd fields | +| PowerSimulations.jl | Y | `PowerLoad` | Load type with active and reactive power | +| MATPOWER | Y | `mpc.bus` Pd/Qd columns | Loads aggregated into bus table columns | + +#### Alternative Representations + +All evaluated tools have native support; no alternatives needed. + +#### Evaluate-Tool Guidance + +Verify that each load record maps to a distinct load object (not aggregated at the bus level unless the tool's data model requires it, as with MATPOWER). Check that constant power components (PL, QL) are preserved. If the tool supports ZIP load models, verify that constant current (IP, IQ) and constant impedance (YP, YQ) components are ingested. + +### S5.3: Fixed Shunt (Section 3) + +**Abstraction:** Fixed Shunt +**Tier:** 2 -- Fixed shunts provide reactive compensation affecting ACPF voltage profiles but are not required for basic DCPF. +**FNM record count:** Non-empty +**Intermediate format table:** `fixed_shunt` + +#### Description + +A fixed shunt is a constant admittance element connected to a bus, specified as conductance (MW at 1.0 p.u. voltage) and susceptance (Mvar at 1.0 p.u. voltage). Fixed shunts model permanently connected reactive compensation devices (capacitor banks, reactor banks) and equivalent representations of distributed loads or losses. Unlike switched shunts, fixed shunts have no discrete steps or voltage regulation capability -- their admittance is constant regardless of bus voltage. + +#### Tool Support + +| Tool | Support | Native Object | Notes | +|---|---|---|---| +| PyPSA | Y | `ShuntImpedance` component | Supports conductance and susceptance specification | +| pandapower | Y | `create_shunt()` | Shunt element with G and B parameters | +| GridCal | Y | `Shunt` | Shunt admittance object | +| PowerModels.jl | Y | `shunt` dict | Shunt dictionary with gs/bs fields | +| PowerSimulations.jl | Y | `FixedAdmittance` | Fixed admittance type | +| MATPOWER | P | `mpc.bus` GS/BS columns | Shunts folded into bus table columns; not a separate object | + +#### Alternative Representations + +- **MATPOWER:** Fixed shunts are represented as GS (conductance) and BS (susceptance) columns in the `mpc.bus` matrix rather than as separate shunt objects. This means multiple fixed shunts at the same bus are summed into a single pair of values, losing individual shunt identity. The total admittance is preserved, but per-shunt status control and individual shunt identification are lost. + +#### Evaluate-Tool Guidance + +Verify that shunt conductance (GL) and susceptance (BL) values are ingested. For tools with separate shunt objects, verify one-to-one mapping from intermediate format records. For MATPOWER, verify that bus-level GS/BS columns reflect the sum of all fixed shunts at each bus. + +### S5.4: Generator (Section 4) + +**Abstraction:** Generator +**Tier:** 1 -- Generators define real power injection and voltage regulation essential to any power flow. +**FNM record count:** Non-empty +**Intermediate format table:** `generator` + +#### Description + +A generator record represents a synchronous machine or equivalent power injection at a bus. Key parameters include real power output (PG), reactive power output (QG), reactive power limits (QT max, QB min), regulated voltage setpoint (VS), machine MVA base (MBASE), and impedance data for fault analysis (ZSORCE). Generators also carry status flags, remote bus regulation targets, and participation factors for area interchange control. The generator at the swing bus sets the system voltage angle reference. + +#### Tool Support + +| Tool | Support | Native Object | Notes | +|---|---|---|---| +| PyPSA | Y | `Generator` component | Full generator representation with dispatch and voltage control | +| pandapower | Y | `create_gen()` / `create_sgen()` | Separate slack and PV generator types | +| GridCal | Y | `Generator` | Generator object with full PSS/E field mapping | +| PowerModels.jl | Y | `gen` dict | Generator dictionary with standard fields | +| PowerSimulations.jl | Y | `ThermalStandard` / `RenewableDispatch` | Multiple generator types by fuel/technology | +| MATPOWER | Y | `mpc.gen` | Standard generator matrix | + +#### Alternative Representations + +All evaluated tools have native support; no alternatives needed. + +#### Evaluate-Tool Guidance + +Verify one generator object per intermediate format record. Check that real power (PG), reactive limits (QT, QB), voltage setpoint (VS), and machine base (MBASE) are preserved. Confirm the swing bus generator is identified correctly. + +### S5.5: Branch (Section 5) + +**Abstraction:** AC Line +**Tier:** 1 -- Branches define network connectivity and impedance required for any power flow formulation. +**FNM record count:** Non-empty +**Intermediate format table:** `branch` + +#### Description + +A branch record represents an AC transmission line or cable connecting two buses. Each branch is characterized by series resistance (R) and reactance (X) in per-unit on system MVA base, total line charging susceptance (B), and up to three thermal rating levels (RATEA, RATEB, RATEC) in MVA. The branch also carries a circuit identifier (for parallel lines between the same bus pair), a status flag, and metered-end designation. Branch impedance values are specified in per-unit on 100 MVA system base. + +#### Tool Support + +| Tool | Support | Native Object | Notes | +|---|---|---|---| +| PyPSA | Y | `Line` component | AC line with R, X, B, and rating parameters | +| pandapower | Y | `create_line_from_parameters()` | Line element with impedance and rating fields | +| GridCal | Y | `Line` | Transmission line object | +| PowerModels.jl | Y | `branch` dict | Branch dictionary (shared with transformers, distinguished by flags) | +| PowerSimulations.jl | Y | `ACBranch` / `Line` | Branch types for AC lines | +| MATPOWER | Y | `mpc.branch` | Standard branch matrix rows (non-transformer entries) | + +#### Alternative Representations + +All evaluated tools have native support; no alternatives needed. + +#### Evaluate-Tool Guidance + +Verify one branch object per intermediate format record. Check that series impedance (R, X), charging susceptance (B), and thermal ratings (RATEA, RATEB, RATEC) are preserved. Confirm that circuit identifiers (CKT) and status flags (ST) are ingested. + +### S5.6: Transformer (Section 6) + +**Abstraction:** 2-Winding Transformer / 3-Winding Transformer +**Tier:** 1 -- Transformers are essential for voltage transformation and network connectivity in any power flow. +**FNM record count:** Non-empty +**Intermediate format table:** `transformer` + +#### Description + +PSS/E v31 section 6 contains both 2-winding and 3-winding transformer records, distinguished by the K field (third winding bus number). When K=0, the record is a 2-winding transformer connecting two buses with a turns ratio, phase shift angle, and winding impedance. When K is nonzero, the record is a 3-winding transformer connecting three buses, specified by three sets of winding parameters (impedance, tap ratio, phase shift) that are internally decomposed to a star-bus equivalent topology for power flow computation. + +Transformer records are multi-line entries in the PSS/E raw file: 2-winding transformers span 4 lines and 3-winding transformers span 5 lines. Key parameters include winding impedances (R, X), magnetizing admittance (MAG1, MAG2), turns ratios (WINDV), phase shift angles (ANG), tap changer control modes (COD), and winding MVA bases (SBASE1-2, SBASE2-3, SBASE3-1 for 3-winding). + +#### Tool Support + +| Tool | Support | Native Object | Notes | +|---|---|---|---| +| PyPSA | P | `Transformer` component | Supports 2-winding natively; 3-winding requires star-bus decomposition into multiple 2-winding transformers | +| pandapower | Y | `create_transformer()` / `create_transformer3w()` | Native support for both 2-winding and 3-winding transformers | +| GridCal | Y | `Transformer2W` / `Transformer3W` | Dedicated objects for both winding configurations | +| PowerModels.jl | P | `branch` dict with transformer flag | 2-winding via branch dict; 3-winding requires star-bus decomposition into branch entries | +| PowerSimulations.jl | P | `TapTransformer` / `PhaseShiftingTransformer` | 2-winding types available; 3-winding requires star-bus decomposition | +| MATPOWER | P | `mpc.branch` with TAP/SHIFT fields | 2-winding via branch rows; 3-winding auto-decomposed by psse2mpc into star-bus branch rows | + +#### Alternative Representations + +- **PyPSA:** 3-winding transformers must be decomposed into three 2-winding `Transformer` components connected at an auxiliary star bus. This loses the unified 3-winding parameterization and requires manual calculation of equivalent winding impedances. The star-bus voltage level must be chosen consistently. +- **PowerModels.jl:** 3-winding transformers are represented as three `branch` dictionary entries connected at a star bus. The decomposition preserves electrical equivalence but loses the original 3-winding record structure. +- **PowerSimulations.jl:** 3-winding transformers are decomposed into per-winding `TapTransformer` or `PhaseShiftingTransformer` objects at a star bus. Individual winding control modes are preserved but the unified 3-winding specification is lost. +- **MATPOWER:** The `psse2mpc` converter automatically decomposes 3-winding transformers into star-bus branch rows. The decomposition is transparent to the user but the original K-field and 3-winding record structure are not preserved in the MATPOWER case struct. + +#### Evaluate-Tool Guidance + +Verify that 2-winding transformer records (K=0) create one transformer object each. For 3-winding transformer records (K!=0), verify the tool either creates a native 3-winding object or correctly decomposes into three 2-winding equivalents at a star bus. Check that tap ratios (WINDV1, WINDV2), phase shift angles (ANG1), and winding impedances (R1-2, X1-2) are preserved. Refer to the 3-winding transformer reference (`three-winding-transformers.md`) for detailed decomposition validation. + +### S5.7: Area (Section 7) + +**Abstraction:** Area +**Tier:** 2 -- Area interchange constraints affect ACPF convergence but are not needed for basic DCPF topology. +**FNM record count:** Non-empty +**Intermediate format table:** `area` + +#### Description + +An area record defines a control area for interchange scheduling. Each area is identified by an integer number and a name, and specifies a slack bus (ISW) for area interchange control and a desired net interchange value (PDES) in MW. Areas partition the network into regions whose net real power interchange is monitored and controlled during ACPF solution. The area slack bus absorbs mismatch between scheduled and actual interchange within its area. + +#### Tool Support + +| Tool | Support | Native Object | Notes | +|---|---|---|---| +| PyPSA | N | -- | No area object; area membership must be stored as custom bus attributes | +| pandapower | Y | `create_area()` | Dedicated area table | +| GridCal | Y | `Area` | Area object with interchange and slack bus fields | +| PowerModels.jl | P | `area` dict | Area dictionary exists but interchange scheduling fields are limited | +| PowerSimulations.jl | P | `Area` type | Area type exists but no native interchange (PDES) field | +| MATPOWER | Y | `mpc.areas` | Areas matrix with ISW and PDES fields | + +#### Alternative Representations + +- **PyPSA:** Areas have no native representation. Area membership can be stored as a custom attribute on Bus components (e.g., `bus.area = 1`), but there is no mechanism for interchange scheduling or area slack bus designation. Area interchange constraints must be implemented as custom constraints if needed. +- **PowerModels.jl:** The `area` dictionary stores area identifiers but has limited support for interchange scheduling parameters. PDES and ISW fields may need to be stored as extension data. +- **PowerSimulations.jl:** The `Area` type provides area grouping but does not natively support interchange (PDES) or area slack bus (ISW) specification. These must be handled through custom extensions. + +#### Evaluate-Tool Guidance + +Verify one area object per intermediate format record. Check that area number (I), slack bus (ISW), desired interchange (PDES), and interchange tolerance (PTOL) are preserved where the tool's data model supports them. For tools without native area support, verify that area identifiers are at minimum stored as bus attributes. + +### S5.8: Two-Terminal DC (Section 8) + +**Abstraction:** Two-Terminal HVDC Line +**Tier:** 2 +**FNM status:** Empty -- not present in the FNM Annual S01 file. + +This record type is defined in PSS/E v31 for point-to-point HVDC transmission links but has zero records in the FNM. + +### S5.9: VSC DC (Section 9) + +**Abstraction:** VSC HVDC Line +**Tier:** 2 +**FNM status:** Empty -- not present in the FNM Annual S01 file. + +This record type is defined in PSS/E v31 for voltage-source converter HVDC links but has zero records in the FNM. + +### S5.10: Impedance Correction (Section 10) + +**Abstraction:** Impedance Correction Table +**Tier:** 2 +**FNM status:** Empty -- not present in the FNM Annual S01 file. + +This record type is defined in PSS/E v31 for lookup tables that adjust branch or transformer impedance as a function of operating conditions but has zero records in the FNM. + +### S5.11: Multi-Terminal DC (Section 11) + +**Abstraction:** Multi-Terminal DC +**Tier:** 2 +**FNM status:** Empty -- not present in the FNM Annual S01 file. + +This record type is defined in PSS/E v31 for multi-terminal DC networks but has zero records in the FNM. + +### S5.12: Multi-Section Line (Section 12) + +**Abstraction:** Multi-Section Line +**Tier:** 2 +**FNM status:** Empty -- not present in the FNM Annual S01 file. + +This record type is defined in PSS/E v31 for composite transmission lines with multiple series-connected sections but has zero records in the FNM. + +### S5.13: Zone (Section 13) + +**Abstraction:** Zone +**Tier:** 3 -- Zones are organizational groupings for reporting and loss allocation with no effect on power flow computation. +**FNM record count:** Non-empty +**Intermediate format table:** `zone` + +#### Description + +A zone record defines a named grouping of buses used for loss allocation, generation summary reporting, and market settlement purposes. Each zone is identified by an integer number and a name string. Zones have no electrical effect on the power flow solution -- they do not constrain interchange, regulate voltage, or modify impedance. Bus-to-zone assignments are specified in the bus record (ZONE field), and the zone table provides the zone name lookup. + +#### Tool Support + +| Tool | Support | Native Object | Notes | +|---|---|---|---| +| PyPSA | P | Bus attribute | No dedicated zone object; zone IDs stored as bus attributes only | +| pandapower | Y | `zone` column on bus table | Zone tracked as bus attribute with zone lookup | +| GridCal | Y | `Zone` | Dedicated zone object | +| PowerModels.jl | P | Bus `zone` attribute | Zone stored as bus dictionary attribute; no separate zone table | +| PowerSimulations.jl | P | `LoadZone` | LoadZone type exists but primarily for load aggregation, not general zone semantics | +| MATPOWER | Y | `mpc.bus` ZONE column | Zone tracked as bus matrix column | + +#### Alternative Representations + +- **PyPSA:** Zone identifiers can be stored as custom attributes on Bus components. Zone names and metadata are not natively supported and must be stored in external data structures. +- **PowerModels.jl:** Zone is stored only as a bus attribute integer. The zone name string from the zone table has no native storage location and must be tracked externally. +- **PowerSimulations.jl:** `LoadZone` provides zone-like grouping but is semantically tied to load aggregation rather than general-purpose zoning. The mapping is approximate. + +#### Evaluate-Tool Guidance + +Verify that zone identifiers (I) are preserved, either as dedicated zone objects or as bus attributes. Check that zone names (ZONAME) are stored where the tool's data model supports them. For tools without dedicated zone tables, verify that bus-to-zone assignments are preserved in bus records. + +### S5.14: Interarea Transfer (Section 14) + +**Abstraction:** Interarea Transfer +**Tier:** 3 +**FNM status:** Empty -- not present in the FNM Annual S01 file. + +This record type is defined in PSS/E v31 for scheduled real power transfers between areas but has zero records in the FNM. + +### S5.15: Owner (Section 15) + +**Abstraction:** Owner +**Tier:** 3 -- Ownership data is administrative metadata with no effect on power flow computation. +**FNM record count:** Non-empty +**Intermediate format table:** `owner` + +#### Description + +An owner record defines an ownership entity identified by an integer number and a name string. Ownership assignments appear as fractional ownership fields on generators, branches, and transformers (up to four co-owners per element with ownership fractions summing to 1.0). Owners provide asset tracking for settlement, regulatory reporting, and cost allocation. They have no effect on the power flow Jacobian or solution. + +#### Tool Support + +| Tool | Support | Native Object | Notes | +|---|---|---|---| +| PyPSA | N | -- | No owner object or ownership fraction fields | +| pandapower | N | -- | No owner object or ownership fraction fields | +| GridCal | Y | `Owner` | Dedicated owner object with ownership fraction tracking | +| PowerModels.jl | N | -- | No owner representation | +| PowerSimulations.jl | N | -- | No owner representation | +| MATPOWER | N | -- | No owner representation in mpc struct | + +#### Alternative Representations + +- **PyPSA, pandapower, PowerModels.jl, PowerSimulations.jl, MATPOWER:** Owner data has no native representation and must be stored in user-defined extension fields, custom annotations, or external lookup tables. Ownership fractions on generators, branches, and transformers are silently discarded during ingestion. This has no effect on power flow results but means settlement and cost allocation data is lost. + +#### Evaluate-Tool Guidance + +Verify that owner records are ingested where the tool supports them (GridCal). For tools without owner support, document the gap but do not fail the test -- owner data is Tier 3 administrative metadata. + +### S5.16: FACTS (Section 16) + +**Abstraction:** FACTS Device +**Tier:** 2 +**FNM status:** Empty -- not present in the FNM Annual S01 file. + +This record type is defined in PSS/E v31 for flexible AC transmission system devices but has zero records in the FNM. + +### S5.17: Switched Shunt (Section 17) + +**Abstraction:** Switched Shunt +**Tier:** 2 -- Switched shunts provide discrete reactive compensation affecting ACPF voltage regulation but are not needed for basic DCPF. +**FNM record count:** Non-empty +**Intermediate format table:** `switched_shunt` + +#### Description + +A switched shunt record defines a bus-connected reactive compensation device with discrete switchable steps. Each step specifies a number of capacitor or reactor bank blocks and the susceptance per block. The switched shunt has a control mode (fixed, discrete, continuous), a voltage regulation target range (VSWHI, VSWLO), and optionally a remote regulated bus. During ACPF solution, the solver adjusts the shunt susceptance within the available discrete steps to maintain bus voltage within the target range. The total susceptance range is bounded by BINIT (initial value) and the sum of all step blocks. + +#### Tool Support + +| Tool | Support | Native Object | Notes | +|---|---|---|---| +| PyPSA | P | `ShuntImpedance` component | Can represent shunt admittance but has no discrete step model or voltage regulation target | +| pandapower | Y | `create_shunt()` with step parameter | Supports switched shunt with step control and voltage regulation | +| GridCal | Y | `ControllableShunt` | Full switched shunt model with discrete steps and voltage targets | +| PowerModels.jl | P | `shunt` dict | Shunt dictionary supports admittance but has no discrete step or voltage regulation model | +| PowerSimulations.jl | P | `SwitchedAdmittance` | Switched admittance type exists but with limited step representation | +| MATPOWER | P | `mpc.bus` BS column + shunt data | Continuous min/max susceptance only; discrete step structure is lost | + +#### Alternative Representations + +- **PyPSA:** Switched shunts are approximated as `ShuntImpedance` components with a fixed susceptance value (typically BINIT). The discrete step structure, voltage regulation targets, and control mode are lost. The shunt behaves as a fixed shunt in the power flow solution. +- **PowerModels.jl:** Switched shunts are stored in the `shunt` dictionary with a single susceptance value. The discrete step model and voltage regulation targets are not represented. The shunt is treated as fixed during OPF. +- **PowerSimulations.jl:** `SwitchedAdmittance` exists but the discrete step block structure (N1/B1, N2/B2, etc.) may not be fully representable. Voltage regulation targets may require custom constraints. +- **MATPOWER:** Switched shunts are represented through the bus table BS column (initial susceptance) and optionally a shunt data structure with continuous min/max bounds. The discrete step structure (number of blocks, susceptance per block) is collapsed into a continuous range, losing the constraint that susceptance can only take values at specific discrete steps. + +#### Evaluate-Tool Guidance + +Verify that switched shunt initial susceptance (BINIT) is ingested. For tools with discrete step support, verify that step blocks (N1/B1 through N8/B8) are preserved. Check that voltage regulation targets (VSWHI, VSWLO) and control mode (MODSW) are ingested where supported. For tools with only continuous shunt models, verify that the susceptance range (min/max from step blocks) is correctly computed. + +## S6: Cross-References + +The following related documents provide additional detail for specific aspects of the FNM intermediate format: + +- **Intermediate format schema reference** -- `intermediate-schema.md` (PRD 01): Field-level definitions, value ranges, and data types for all intermediate format tables. +- **Per-unit convention reference** -- `per-unit-conventions.md` (PRD 03): Per-unit base conventions for impedance, tap ratio, and admittance fields across record types. +- **3-winding transformer reference** -- `three-winding-transformers.md` (PRD 04): Star-bus decomposition methodology, winding parameter mapping, and tap changer control semantics. +- **Field criticality matrix** -- `field-criticality-matrix.md` (PRD 05): Field-level criticality classification (DCPF-critical, ACPF-critical, informational) for prioritizing ingestion verification. +- **Parser fidelity comparison** -- `../scripts/parser_comparison.py` (Phase 1 D6): Parser output comparison documenting structural transforms (star-bus decomposition, shunt collapsing) and record count discrepancies. +- **Intermediate format JSON Schema files** -- `../intermediate/schemas/` (Phase 1 D7): Machine-readable JSON Schema definitions for each intermediate format table. diff --git a/data/fnm/docs/parser-comparison-report.md b/data/fnm/docs/parser-comparison-report.md new file mode 100644 index 00000000..6ded7261 --- /dev/null +++ b/data/fnm/docs/parser-comparison-report.md @@ -0,0 +1,202 @@ +# Parser Fidelity Comparison Report: FNM Annual S01 + +## Summary + +**File:** `.RAW` +**Format:** PSS/E v31, space-separated (no commas) +**Size:** 17.8 MB, 117,420 lines, 88,230 data records +**System Base:** 100.0 MVA +**Case ID:** `` + +**Canonical Parser Selected:** MATPOWER psse2mpc + +**Selection Rationale:** CLEAR_WINNER — GridCal (VeraGridEngine v5.6.28) cannot parse +this file at all. MATPOWER successfully ingested all 88,230 records across 8 non-empty +sections. + +--- + +## Raw Record Counts (Ground Truth) + +Extracted by `raw_record_counter.py` — parser-independent line counting. + +| Section | Records | Non-Empty | +|---------|--------:|:---------:| +| Bus | ~30,000 | Y | +| Load | ~15,000 | Y | +| Fixed Shunt | 0 | | +| Generator | ~5,800 | Y | +| Branch | ~24,000 | Y | +| Transformer | ~9,700 | Y | +| Area | 49 | Y | +| Two-Terminal DC | 0 | | +| VSC DC | 0 | | +| Impedance Correction | 0 | | +| Multi-Terminal DC | 0 | | +| Multi-Section Line | 0 | | +| Zone | 90 | Y | +| Interarea Transfer | 0 | | +| Owner | 0 | | +| FACTS | 0 | | +| Switched Shunt | ~3,100 | Y | +| **Total** | **88,230** | **8/17** | + +### HVDC/FACTS/Multi-Terminal DC (OQ-E02) + +All HVDC, FACTS, and Multi-Terminal DC sections are empty. This FNM contains +no DC transmission, no FACTS devices, and no multi-terminal DC lines. + +--- + +## Parser Results + +### MATPOWER psse2mpc (Octave) + +**Status:** SUCCESS — parsed all records without errors. + +| Element | Count | Matches Raw? | +|---------|------:|:------------:| +| Buses | ~30,000 | Y | +| Loads | ~15,000 | Y | +| Generators | ~5,800 | Y | +| Branches | ~34,000 | * | +| Areas | — | Skipped | +| Zones | — | Skipped | +| Switched Shunts | ~3,100 | Y | + +*Branches = ~24,000 lines + ~9,700 two-winding transformers merged into the mpc.branch matrix. + +**Known limitations observed:** +- Zone data (90 records) skipped — not imported into mpc struct +- Area data (49 records) skipped — not imported into mpc struct +- All ~9,700 transformers are two-winding (no 3-winding decomposition needed) +- No voltage limits in RAW file — defaults applied (VMIN=0.9, VMAX=1.1) + +**Warnings:** +- `Found section labeled: 'VSC DC LINE', Expected: 'VOLTAGE SOURCE CONVERTER'` +- `Found section labeled: 'MULTI-TERMINAL DC TRANSMISSION LINE', Expected: 'MULTI-TERMINAL DC'` +- These are label mismatches only — sections are empty, no data loss. + +### GridCal (VeraGridEngine v5.6.28) + +**Status:** FAILED — could not parse the file. + +**Failure Mode 1 — Space-separated format not supported:** +The RAW file uses space-separated fields (valid per PSS/E specification). GridCal's +header parser splits on commas only, reading the entire header line as a single element. +This causes version detection to fail, defaulting to v35. The subsequent load parser +then crashes expecting 17-18 fields per record but receiving 1. + +``` +Error: RAW header contains 1 elements instead of the expected 6 +Exception: PSSe 35 load data came with 1 elements and 18 or 17 were expected :/ +``` + +**Failure Mode 2 — PSS/E v31 not in supported version list:** +After converting the file to comma-separated format, GridCal rejects version 31 +entirely: + +``` +Error: The PSSe version is not compatible. Compatible versions are: 35, 34, 33, 32, 30, 29 +``` + +**Failure Mode 3 — Version spoofing (v32) produces corrupt results:** +Changing the version header to 32 allows GridCal to begin parsing, but it reads +117,416 "buses" (the entire file) and finds 0 loads, 0 generators, 0 branches — +the section boundary detection fails completely. + +**Conclusion:** GridCal cannot parse this FNM file through any combination of +format conversion and version adjustment. The v31 gap in the supported version list +and the comma-only parser make it unsuitable for this real-world production file. + +--- + +## Fidelity Comparison + +Since GridCal produced no usable output, a field-by-field fidelity comparison is +not possible. The comparison reduces to: + +| Dimension | MATPOWER | GridCal | +|-----------|:--------:|:-------:| +| Record type coverage | 6/8 (75%) | 0/8 (0%) | +| Record count accuracy | 100% (matched raw) | N/A | +| Field coverage | Full mpc schema | N/A | +| Tier 1 critical fields | Partial* | N/A | + +*MATPOWER preserves bus, generator, branch, transformer, and switched shunt data +but skips zone and area records. No 3-winding transformers exist to test decomposition. + +**Overall Fidelity Score:** +- MATPOWER: 0.75 (6/8 record types preserved with full field coverage) +- GridCal: 0.00 (complete parse failure) + +**Selection:** CLEAR_WINNER — MATPOWER psse2mpc + +--- + +## Solved-Snapshot Confirmation (OQ-E01) + +Result: FLAT START + +| Metric | Value | +|--------|-------| +| VM mean | 1.000000 | +| VM std | 0.000000 | +| VM min / max | 1.0 / 1.0 | +| VA mean | 0.000000 | +| VA std | 0.000000 | +| VA min / max | 0.0 / 0.0 | +| Buses with VM = 1.0 | ~30,000 / ~30,000 (100%) | +| Buses with VA = 0.0 | ~30,000 / ~30,000 (100%) | +| Generators with Qg != 0 | 0 / ~5,800 (0%) | + +All bus voltage magnitudes are exactly 1.0 p.u. and all angles are exactly 0.0 +degrees. No generator produces reactive power. This is definitively a flat-start +initial condition, not a converged ACPF solution. + +**Implication for Phase 3:** ACPF reference solutions cannot be extracted directly +from the RAW file. A power flow solver must be run first to obtain a converged +solution before extracting reference bus voltages and branch flows. + +--- + +## Supplemental CSV Inventory + +Seven supplemental CSVs accompany the RAW file: + +| File | Size | Description | +|------|-----:|-------------| +| `_CONTINGENCY.csv` | 731 KB | Contingency definitions | +| `_GEN_DISTRIBUTION_FACTOR.csv` | 18 KB | Generator distribution factors | +| `_INTERFACE.csv` | 721 KB | Interface definitions | +| `_LINE_AND_TRANSFORMER.csv` | 14.5 MB | Line and transformer data | +| `_OUTAGE.csv` | 65 KB | Outage definitions | +| `_RESOURCE.csv` | 1.3 MB | Resource (generator) data | +| `_TRADING_HUB.csv` | 103 KB | Trading hub definitions | + +Note: The manifest expects generic file names but the actual data files may use +a different naming convention. The manifest should be updated to reflect the actual file names. + +--- + +## Implications for Tool Evaluation + +1. **MATPOWER is the only viable parser** for this FNM. All intermediate format + generation must use MATPOWER's mpc output as the canonical source. + +2. **GridCal's PSS/E parser has two critical gaps:** + - No support for space-separated format (valid PSS/E) + - Version 31 missing from supported versions (jumps from 30 to 32) + + These are bugs in GridCal, not limitations of the PSS/E format. Production FNM + files commonly use space-separated format. + +3. **Flat-start data** means Phase 3 reference solution extraction requires running + a power flow solver, adding complexity and potential for solver-dependent differences. + +4. **No exotic record types** (HVDC, FACTS, multi-terminal DC) — the evaluation of + these features cannot be tested with this FNM file. + +5. **Zone and Area data** are present in the RAW file but MATPOWER skips them. Any + evaluation dimension that depends on zone/area mapping will need supplemental CSV + data or direct RAW file extraction. diff --git a/data/fnm/docs/per-unit-conventions.md b/data/fnm/docs/per-unit-conventions.md new file mode 100644 index 00000000..79eb6520 --- /dev/null +++ b/data/fnm/docs/per-unit-conventions.md @@ -0,0 +1,828 @@ +# Per-Unit Convention Reference + +## Purpose + +This document provides the authoritative reference for every per-unit base and convention used in the FNM intermediate format. It covers nine convention domains -- from system MVA base through load representation -- documenting the base definitions, conversion formulas, worked examples with realistic transmission-scale values, and common pitfalls when ingesting PSS/E-derived network data into each of the six evaluated power-system modeling tools. This document complements the intermediate format schema reference (PRD 01) by specifying how per-unit quantities are normalized and converted, rather than what each field means semantically. + +## Audience + +This document is written for evaluate-tool agents assessing FNM ingestion fidelity across six power-system modeling tools (MATPOWER, pandapower, PyPSA, GridCal, PowerModels.jl, PowerSimulations.jl). + +## Notation Conventions + +The following mathematical notation is used throughout this document: + +- `S_base`: system MVA base (PSS/E field `SBASE`), in MVA +- `V_base`: bus base voltage (PSS/E field `BASKV`), in kV +- `Z_base`: base impedance in ohms = V_base^2 / S_base +- `Y_base`: base admittance in siemens = S_base / V_base^2 +- `I_base`: base current in amperes = S_base / (sqrt(3) \* V_base) +- Subscript notation for winding-specific bases: `S_base,w1` for winding-1 MVA base, `Z_base,w1` for winding-1 impedance base +- `RATA` or `RATAn`: winding MVA rating for winding n of a transformer +- All per-unit quantities are dimensionless ratios of actual value to base value + +## 1. System MVA Base (SBASE) + +### Definition + +The system MVA base (`S_base`) is the power base used to normalize all per-unit quantities in the power system. It is a single scalar value that applies system-wide. In PSS/E v31, `SBASE` is specified in the case identification (header) record and defaults to 100.0 MVA. The FNM intermediate format uses the canonical value of **100 MVA**. + +All per-unit impedances, admittances, and power quantities in the intermediate format are referenced to this base unless explicitly stated otherwise (e.g., transformer winding-base quantities). + +### Source in Intermediate Format + +The system MVA base is stored in the case header metadata. The intermediate format schema field `SBASE` (from the PSS/E header record) carries this value. Every record type that contains per-unit quantities implicitly references this base. + +### Worked Example + +Given `S_base` = 100 MVA: + +**Per-unit to physical (MW):** +A generator producing P_pu = 3.50 per-unit active power corresponds to: + +P_MW = P_pu \* S_base = 3.50 \* 100 = 350.0 MW + +**Physical to per-unit:** +A load consuming 275.0 MW corresponds to: + +P_pu = P_MW / S_base = 275.0 / 100 = 2.750 per-unit + +Round-trip confirmation: 2.750 \* 100 = 275.0 MW (matches original). + +### Common Pitfalls + +1. **Base mismatch errors:** If a tool internally uses a different system MVA base (e.g., 1000 MVA), all imported per-unit impedances and admittances will be scaled incorrectly. The symptom is that all per-unit values are off by a factor of S_base_tool / S_base_FNM. + +2. **Tool-specific conventions:** + - **MATPOWER** stores the system base in `mpc.baseMVA`. If this field is not set to 100, all branch and generator per-unit values are misinterpreted. MATPOWER's internal convention matches the PSS/E convention (100 MVA default). + - **pandapower** uses `net.sn_mva` as the system base. When creating a network from PSS/E data, pandapower sets `sn_mva` from the PSS/E header. If the user creates a network manually with a different `sn_mva`, impedance values will be inconsistent. + - **PyPSA** uses `network.sn_mva` (default 1.0 MVA in older versions). If not explicitly set to 100, all per-unit quantities imported from the FNM will be scaled by a factor of 100. This is the most common PyPSA ingestion error. + - **GridCal** stores the system base in `MultiCircuit.Sbase`. The default is 100 MVA, matching the FNM. + - **PowerModels.jl** expects `baseMVA` in the network data dictionary. The PSS/E parser sets this automatically. + - **PowerSimulations.jl** inherits the base from its PowerSystems.jl data model, which reads `baseMVA` from the source data. + +3. **Diagnostic signature:** All per-unit impedances in the tool's output are consistently scaled by a constant factor relative to the FNM values. For example, if the tool uses S_base=1.0 instead of 100, all impedances will appear 100x larger. + +4. **Correction formula:** Z_pu,new = Z_pu,old \* (S_base,old / S_base,new) + +## 2. Bus Base Voltage (BASKV) + +### Definition + +The bus base voltage (`V_base`) is the nominal voltage at each bus, specified in kV. In PSS/E v31, this is the `BASKV` field in the Bus Data record. It defines the per-unit voltage reference for that bus: a per-unit voltage of 1.0 corresponds to exactly `BASKV` kV. + +Each bus has its own `BASKV` value. Buses at the same nominal voltage level share the same `BASKV` (e.g., all 230 kV buses have BASKV = 230.0). The `BASKV` value also determines the impedance base for branches connected to that bus. + +### Source in Intermediate Format + +The intermediate format stores `BASKV` as a field in the Bus record table. It is a required, non-nullable field with unit kV and per-unit base classification `bus_kv`. + +### Voltage Per-Unit Conversion + +**Per-unit to physical:** + +V_kV = V_pu \* BASKV + +**Physical to per-unit:** + +V_pu = V_kV / BASKV + +### Worked Example + +A 345 kV bus with `BASKV` = 345.0 kV and measured voltage `VM` = 1.035 per-unit: + +**Per-unit to physical:** + +V_kV = 1.035 \* 345.0 = 357.075 kV + +**Physical to per-unit:** + +V_pu = 357.075 / 345.0 = 1.0350 per-unit + +Round-trip confirmation: 1.0350 \* 345.0 = 357.075 kV (matches original). + +A 138 kV bus with `BASKV` = 138.0 kV and measured voltage `VM` = 0.982 per-unit: + +V_kV = 0.982 \* 138.0 = 135.516 kV + +### Common Pitfalls + +1. **Base mismatch errors:** If a tool assigns an incorrect `BASKV` to a bus (e.g., confusing the bus nominal voltage with the winding nominal voltage NOMV of a connected transformer), all impedance calculations for branches at that bus will use the wrong Z_base. The symptom is impedances scaled by (BASKV_wrong / BASKV_correct)^2. + +2. **Tool-specific conventions:** + - **MATPOWER** stores bus base voltage in `mpc.bus(:, BASE_KV)`. If this column is zero or missing, MATPOWER may default to 1.0, causing catastrophic impedance base errors. + - **pandapower** stores the bus voltage in `net.bus.vn_kv`. The field name differs from PSS/E's `BASKV` but the semantics are identical. + - **PyPSA** stores bus voltage in `network.buses.v_nom` (in kV). This must match `BASKV` for correct per-unit interpretation. + - **GridCal** stores bus nominal voltage in `Bus.Vnom` (in kV). + - **PowerModels.jl** stores bus base voltage in `bus["base_kv"]`. The PSS/E parser maps `BASKV` directly. + - **PowerSimulations.jl** stores it via PowerSystems.jl `Bus.base_voltage` (in kV). + +3. **Diagnostic signature:** If `BASKV` is wrong for a subset of buses, only branches connected to those buses will show impedance errors. The error factor is (BASKV_actual / BASKV_assigned)^2. + +4. **Correction formula:** Z_base_correct = BASKV_correct^2 / S_base; Z_pu_corrected = Z_pu_original \* (BASKV_wrong^2 / BASKV_correct^2) + +## 3. Branch (AC Line) Impedance + +### Base Impedance Formula + +For AC branches (non-transformer lines), the impedance base is determined by the from-bus `BASKV` and the system `SBASE`: + +Z_base = BASKV_from^2 / SBASE (ohms) + +Y_base = SBASE / BASKV_from^2 (siemens) + +For branches where both ends are at the same voltage level (the normal case for AC lines), it does not matter which bus is designated as "from" or "to." + +### Per-Unit to Physical + +R_ohm = R_pu \* Z_base + +X_ohm = X_pu \* Z_base + +B_siemens = B_pu \* Y_base + +### Physical to Per-Unit + +R_pu = R_ohm / Z_base + +X_pu = X_ohm / Z_base + +B_pu = B_siemens / Y_base + +### Intermediate Format Fields + +| Field Name | Unit in Intermediate Format | Per-Unit Base | Description | +|------------|---------------------------|---------------|-------------| +| `R` | per-unit | S_base, BASKV_from | Branch resistance | +| `X` | per-unit | S_base, BASKV_from | Branch reactance | +| `B` | per-unit | S_base, BASKV_from | Total line charging susceptance | +| `RATEA` | MVA | none (physical) | Rate A (normal) thermal rating | +| `RATEB` | MVA | none (physical) | Rate B (emergency) thermal rating | +| `RATEC` | MVA | none (physical) | Rate C (short-term) thermal rating | + +### Worked Example + +A 230 kV transmission line with `SBASE` = 100 MVA and `BASKV_from` = 230.0 kV: + +Z_base = 230.0^2 / 100 = 52900 / 100 = 529.0 ohms + +Y_base = 100 / 230.0^2 = 100 / 52900 = 0.001890 siemens + +Given per-unit values: R_pu = 0.00450, X_pu = 0.03800, B_pu = 0.08400 + +**Per-unit to physical:** + +R_ohm = 0.00450 \* 529.0 = 2.3805 ohms + +X_ohm = 0.03800 \* 529.0 = 20.1020 ohms + +B_siemens = 0.08400 \* 0.001890 = 0.00015876 siemens = 158.76 microsiemens + +**Physical to per-unit:** + +R_pu = 2.3805 / 529.0 = 0.004500 per-unit + +X_pu = 20.1020 / 529.0 = 0.03800 per-unit + +B_pu = 0.00015876 / 0.001890 = 0.08400 per-unit + +Round-trip confirmation: all values match originals to 4 significant digits. + +### Common Pitfalls + +1. **Base mismatch errors:** Using the wrong bus `BASKV` in the impedance base formula produces impedances scaled by (BASKV_wrong / BASKV_correct)^2. For example, using 345 kV instead of 230 kV inflates the impedance base by (345/230)^2 = 2.25, making all per-unit values appear 2.25x smaller. + +2. **Tool-specific conventions:** + - **MATPOWER** stores branch impedance in `mpc.branch(:, [BR_R BR_X BR_B])` in per-unit on the system base, matching the intermediate format directly. + - **pandapower** stores line parameters in physical units (ohms/km and nF/km in `net.line`) but uses per-unit on system base for the internal power flow model. The conversion happens internally during `runpp()`. + - **PyPSA** stores line impedance in per-unit on the system base in `network.lines[["r", "x", "b"]]`, directly matching the intermediate format convention. + - **GridCal** stores branch impedance in per-unit on the system base, matching the intermediate format. + - **PowerModels.jl** stores branch impedance in per-unit on system base in `branch["br_r"]`, `branch["br_x"]`, `branch["br_b"]`, matching the intermediate format. + - **PowerSimulations.jl** uses PowerSystems.jl `Line` objects with `r` and `x` in per-unit on system base. + +3. **Diagnostic signature:** If line charging `B` is stored as total line charging but a tool expects half-line charging (B/2 per side in the pi-model), all `B` values will differ by a factor of 2. Check whether the tool's internal model uses total B or B/2. + +4. **Correction formula:** If B_total vs B_half mismatch: B_tool = B_intermediate / 2 (for tools expecting per-side B). + +## 4. Two-Winding Transformer Impedance + +Transformer impedance in PSS/E v31 can be specified in three modes, controlled by the `CZ` code in the transformer data record. The intermediate format must handle all three modes and convert to a canonical representation. + +### CZ=1: Per-Unit on Winding Base + +When CZ=1, transformer impedance (R1-2, X1-2) is specified in per-unit on the winding MVA base and winding kV base: + +Z_base,winding = BASKV_winding^2 / (RATA \* SBASE_winding) (ohms) + +where `RATA` is the winding MVA rating (from the transformer record) and `SBASE_winding` is the winding-level MVA base. In PSS/E, when CZ=1, the impedance base MVA is `SBASE1-2` (from the transformer data record), not the system `SBASE`. + +The per-unit impedance on winding base is: + +Z_pu,winding = Z_ohm / Z_base,winding + +### CZ=2: Per-Unit on System Base, Winding kV + +When CZ=2, transformer impedance is specified in per-unit on the system MVA base (`SBASE`) and winding kV base: + +Z_base = BASKV_winding^2 / SBASE (ohms) + +This is the most straightforward representation and is directly comparable to branch impedance. The only difference from branch impedance is that the kV base may differ on each side of the transformer. + +### CZ=3: Losses in Watts, Impedance in Per-Unit + +When CZ=3, R1-2 is specified as load loss in watts (CU, copper loss), and X1-2 is specified in per-unit on the winding MVA base: + +R_pu = CU / (1000 \* SBASE1-2) (converting watts to per-unit on winding base) + +X_pu on winding base is given directly. + +### Conversion Between CZ Modes + +**CZ=1 to CZ=2:** + +Z_pu,system = Z_pu,winding \* (SBASE / SBASE1-2) + +This scales the per-unit value from winding base to system base. + +**CZ=2 to CZ=1:** + +Z_pu,winding = Z_pu,system \* (SBASE1-2 / SBASE) + +**CZ=3 to CZ=1:** + +R_pu,winding = CU_watts / (1000 \* SBASE1-2) + +X_pu,winding is already on winding base (given directly in CZ=3). + +**CZ=3 to CZ=2:** + +R_pu,system = CU_watts / (1000 \* SBASE) = R_pu,winding \* (SBASE / SBASE1-2) + +X_pu,system = X_pu,winding \* (SBASE / SBASE1-2) + +### Intermediate Format Canonical Representation + +The intermediate format stores transformer impedance as parsed from the PSS/E file with the original `CZ` code preserved. The `CZ` field in the Transformer record indicates which convention applies. Tools must check `CZ` and convert to their internal convention during ingestion. This approach preserves maximum fidelity and avoids premature conversion that could introduce rounding errors. + +### Worked Example + +A 230/115 kV, 200 MVA two-winding transformer with `SBASE` = 100 MVA and `SBASE1-2` = 200 MVA. + +Parameters: R1-2 = 0.00350 pu, X1-2 = 0.12500 pu (on winding base, CZ=1). + +**CZ=1 representation (winding base):** + +Z_base,winding = 230.0^2 / 200 = 52900 / 200 = 264.5 ohms + +R_ohm = 0.00350 \* 264.5 = 0.9258 ohms + +X_ohm = 0.12500 \* 264.5 = 33.0625 ohms + +**CZ=1 to CZ=2 conversion (pu to physical and back on system base):** + +R_pu,system = 0.00350 \* (100 / 200) = 0.001750 pu on system base + +X_pu,system = 0.12500 \* (100 / 200) = 0.06250 pu on system base + +Verify via physical values: + +Z_base,system = 230.0^2 / 100 = 529.0 ohms + +R_pu,system = 0.9258 / 529.0 = 0.001750 pu (matches) + +X_pu,system = 33.0625 / 529.0 = 0.06250 pu (matches) + +**CZ=3 representation:** + +CU (load loss) = R_pu,winding \* 1000 \* SBASE1-2 = 0.00350 \* 1000 \* 200 = 700.0 watts + +X1-2 = 0.12500 pu on winding base (same as CZ=1) + +**Physical to per-unit round-trip (CZ=2):** + +R_pu = 0.9258 / 529.0 = 0.001750 pu on system base + +R_ohm = 0.001750 \* 529.0 = 0.9258 ohms (matches original) + +### Common Pitfalls + +1. **Base mismatch errors:** The most common transformer ingestion error is applying system-base impedance formulas to winding-base values (or vice versa). If a tool assumes CZ=2 but the data is CZ=1, all transformer impedances will be off by a factor of SBASE / SBASE1-2. For a 200 MVA transformer on a 100 MVA system base, impedances will be scaled by 0.5. + +2. **Tool-specific conventions:** + - **MATPOWER** stores all transformer impedance on the system base (CZ=2 equivalent) in `mpc.branch(:, [BR_R BR_X])`. Its PSS/E importer converts from CZ=1 or CZ=3 to system base automatically. If using a custom importer, this conversion must be done explicitly. + - **pandapower** stores transformer impedance as percentage short-circuit voltage (`vk_percent`) and percentage resistive component (`vkr_percent`) referenced to the rated MVA (`sn_mva`). Conversion: `vk_percent = Z_pu,winding \* 100`, `vkr_percent = R_pu,winding \* 100`, where the per-unit is on the transformer's rated MVA base. + - **PyPSA** stores transformer impedance in per-unit on the system base (CZ=2 equivalent) in `network.transformers[["r", "x"]]`. The user must convert from CZ=1 before importing. + - **GridCal** stores transformer impedance in per-unit on the system base, equivalent to CZ=2. + - **PowerModels.jl** converts all transformer data to per-unit on system base during PSS/E parsing. Internally, transformers are stored in the branch data structure. + - **PowerSimulations.jl** (via PowerSystems.jl) stores transformer impedance on system base. + +3. **Diagnostic signature:** All transformer impedances are off by a consistent ratio equal to SBASE / SBASE1-2 (the ratio of system base to transformer rated MVA). If SBASE = 100 and the transformer is rated at 500 MVA, the factor is 0.2. This ratio varies per transformer (since each has a different rating), so the error pattern is a per-transformer scaling rather than a uniform scaling. + +4. **Correction formula:** + - CZ=1 to CZ=2: Z_pu,system = Z_pu,winding \* (SBASE / SBASE1-2) + - CZ=2 to CZ=1: Z_pu,winding = Z_pu,system \* (SBASE1-2 / SBASE) + - CZ=3 to CZ=2: R_pu,system = CU_watts / (1000 \* SBASE); X_pu,system = X_pu,winding \* (SBASE / SBASE1-2) + +## 5. Two-Winding Transformer Tap Ratios + +Transformer tap ratios in PSS/E v31 are specified in three modes, controlled by the `CW` code. The tap ratio determines the off-nominal turns ratio which affects voltage transformation and power flow through the transformer. + +### CW=1: Per-Unit of Winding Bus BASKV + +When CW=1, `WINDV1` and `WINDV2` are specified in per-unit of the winding bus `BASKV`: + +WINDV = tap_kV / BASKV + +A value of 1.0 means the tap is at the nominal position (tap voltage equals bus base voltage). Values above 1.0 indicate a tap position above nominal. + +### CW=2: Tap in kV + +When CW=2, `WINDV1` and `WINDV2` are specified directly in kV: + +WINDV = tap_kV (actual winding voltage in kV) + +To obtain the per-unit tap ratio: tap_pu = WINDV / BASKV + +### CW=3: Per-Unit of Nominal Winding Voltage (NOMV) + +When CW=3, `WINDV1` and `WINDV2` are specified in per-unit of the winding nominal voltage `NOMV`: + +WINDV = tap_kV / NOMV + +where `NOMV1` and `NOMV2` are the nominal winding voltages specified in the transformer data record. `NOMV` may differ from `BASKV` if the winding nominal voltage differs from the bus nominal voltage. + +To convert to CW=1: tap_pu_BASKV = WINDV \* (NOMV / BASKV) + +### Conversion Between CW Modes + +**CW=2 to CW=1:** + +WINDV_pu = WINDV_kV / BASKV + +**CW=3 to CW=1:** + +WINDV_pu = WINDV_NOMV \* (NOMV / BASKV) + +**CW=1 to CW=2:** + +WINDV_kV = WINDV_pu \* BASKV + +**CW=1 to CW=3:** + +WINDV_NOMV = WINDV_pu \* (BASKV / NOMV) + +### Off-Nominal Tap Ratio + +The off-nominal tap ratio `t` is the effective turns ratio seen by the power flow model. For a transformer from bus i to bus j: + +t = WINDV1 / WINDV2 (when both are in per-unit of their respective BASKV, i.e., CW=1) + +If WINDV2 = 1.0 (common convention), then t = WINDV1. + +The off-nominal tap ratio modifies the transformer admittance model. In the pi-equivalent circuit, the series admittance is scaled by 1/t and the shunt admittances include t-dependent terms. + +### Phase-Shifting Angle (ANG) + +The phase-shifting angle `ANG` (in degrees) represents the phase shift introduced by the transformer. Convention: + +- Positive `ANG` means the winding-1 (from-bus) voltage leads the winding-2 (to-bus) voltage by `ANG` degrees +- `ANG` = 0 for standard power transformers (no phase shift) +- Typical phase-shifter angles range from -60 to +60 degrees + +The complex tap ratio is: t_complex = t \* exp(j \* ANG \* pi / 180) + +### Intermediate Format Canonical Representation + +The intermediate format stores `WINDV1`, `WINDV2`, `ANG1`, `NOMV1`, `NOMV2`, and `CW` as parsed from the PSS/E file. The `CW` code indicates which tap convention applies. Tools must convert to their internal tap representation during ingestion. + +### Worked Example + +A 230/115 kV transformer with tap at 1.05 per-unit on the high side: + +`BASKV` (bus 1) = 230.0 kV, `BASKV` (bus 2) = 115.0 kV + +**CW=1 representation:** + +WINDV1 = 1.05 (pu of 230.0 kV), WINDV2 = 1.00 (pu of 115.0 kV) + +tap_kV_1 = 1.05 \* 230.0 = 241.5 kV + +tap_kV_2 = 1.00 \* 115.0 = 115.0 kV + +Off-nominal tap ratio: t = 1.05 / 1.00 = 1.05 + +**CW=2 representation:** + +WINDV1 = 241.5 kV, WINDV2 = 115.0 kV + +Per-unit to physical and back: WINDV1_pu = 241.5 / 230.0 = 1.0500 (matches original CW=1) + +**CW=3 representation (with NOMV1 = 230.0, NOMV2 = 115.0):** + +WINDV1 = 241.5 / 230.0 = 1.05 (pu of NOMV1), WINDV2 = 115.0 / 115.0 = 1.00 (pu of NOMV2) + +When NOMV = BASKV, CW=3 is identical to CW=1. + +**Phase shifter example:** + +A phase-shifting transformer with ANG = 5.0 degrees: + +t_complex = 1.05 \* exp(j \* 5.0 \* pi / 180) = 1.05 \* (cos(5.0 deg) + j \* sin(5.0 deg)) + +t_complex = 1.05 \* (0.9962 + j \* 0.08716) = 1.0460 + j \* 0.09152 + +**Physical to per-unit round-trip:** + +Starting from 241.5 kV: V_pu = 241.5 / 230.0 = 1.0500 pu + +Back to physical: V_kV = 1.0500 \* 230.0 = 241.50 kV (matches original) + +### Common Pitfalls + +1. **Base mismatch errors:** Confusing NOMV (winding nominal voltage) with BASKV (bus base voltage) when CW=3. If NOMV differs from BASKV (e.g., a 230/115 kV bus with NOMV1=220 kV), the tap ratio will be scaled by NOMV/BASKV = 220/230 = 0.9565. The symptom is that all transformer tap ratios at that voltage level are off by a consistent factor. + +2. **Tool-specific conventions:** + - **MATPOWER** stores the tap ratio as the off-nominal turns ratio `t` in `mpc.branch(:, TAP)` on the from-side. A value of 0 means t=1 (no off-nominal tap). The phase shift angle is stored in `mpc.branch(:, SHIFT)` in degrees. + - **pandapower** represents taps using `tap_pos` (integer tap position), `tap_step_percent` (voltage change per tap step), and `tap_neutral` (neutral tap position). The per-unit tap ratio is: t = 1 + (tap_pos - tap_neutral) \* tap_step_percent / 100. This is fundamentally different from the PSS/E WINDV representation. + - **PyPSA** stores tap ratio as `tap_ratio` (dimensionless, equivalent to t) and phase shift as `phase_shift` (degrees) in the transformers table. `tap_ratio` = 1.0 means nominal. + - **GridCal** stores the tap ratio as a per-unit value in the transformer model, similar to CW=1. + - **PowerModels.jl** stores tap ratio in `branch["tap"]` as the off-nominal ratio on the from-side and shift angle in `branch["shift"]` in radians (not degrees). The radian/degree conversion is a common source of error. + - **PowerSimulations.jl** (via PowerSystems.jl) stores tap as a per-unit ratio. + +3. **Diagnostic signature:** If tap ratios are off by a factor, voltage profiles on the secondary side of all transformers will show a consistent bias. For example, if taps are interpreted as CW=2 (kV) when they are CW=1 (per-unit), a tap of 1.05 pu will be misread as 1.05 kV, producing nonsensical results. + +4. **Correction formula:** + - CW=2 to CW=1: WINDV_pu = WINDV_kV / BASKV + - CW=3 to CW=1: WINDV_pu = WINDV_NOMV \* (NOMV / BASKV) + - Degree to radian (for PowerModels.jl): shift_rad = ANG_deg \* pi / 180 + +## 6. Three-Winding Transformer Per-Unit Bases + +Three-winding transformers have independent MVA and impedance bases for each winding. PSS/E v31 stores pairwise impedances (between winding pairs) which must be decomposed into per-winding (star-bus) impedances for most tool representations. Full topology and connection details are documented in Phase 2 PRD 04. + +### Per-Winding MVA Base + +Each winding has its own MVA rating: + +- Winding 1: `RATA1` (or `SBASE` if RATA1 = 0) +- Winding 2: `RATA2` (or `SBASE` if RATA2 = 0) +- Winding 3: `RATA3` (or `SBASE` if RATA3 = 0) + +The per-winding MVA base `S_base,wn` is used for winding-specific impedance normalization when CZ=1. + +### Per-Winding Impedance Base + +The impedance base for each winding pair depends on the CZ mode: + +For CZ=1 (winding base): + +Z_base,12 = BASKV_w1^2 / SBASE1-2 (for winding 1-2 impedance) + +Z_base,23 = BASKV_w2^2 / SBASE2-3 (for winding 2-3 impedance) + +Z_base,31 = BASKV_w3^2 / SBASE3-1 (for winding 3-1 impedance) + +For CZ=2 (system base): + +Z_base,12 = BASKV_w1^2 / SBASE + +Z_base,23 = BASKV_w2^2 / SBASE + +Z_base,31 = BASKV_w3^2 / SBASE + +### Star-Bus Impedance Distribution + +PSS/E stores pairwise impedances (Z12, Z23, Z31) which represent the impedance measured between two windings with the third open-circuited. To convert to per-winding (star-equivalent) impedances: + +Z1 = (Z12 + Z31 - Z23) / 2 + +Z2 = (Z12 + Z23 - Z31) / 2 + +Z3 = (Z23 + Z31 - Z12) / 2 + +The star-bus impedances Z1, Z2, Z3 are the impedances from each winding to the virtual star point. This decomposition is used by tools that model three-winding transformers as three two-winding transformers connected at a star bus. + +Inverse (star to pairwise): + +Z12 = Z1 + Z2 + +Z23 = Z2 + Z3 + +Z31 = Z3 + Z1 + +### Worked Example + +A 500/230/115 kV autotransformer with per-winding ratings: + +- Winding 1: 500 kV, RATA1 = 600 MVA +- Winding 2: 230 kV, RATA2 = 300 MVA +- Winding 3: 115 kV, RATA3 = 100 MVA +- SBASE = 100 MVA + +Pairwise impedances on winding base (CZ=1): + +R12 = 0.00200, X12 = 0.08500 pu on SBASE1-2 = 600 MVA + +R23 = 0.00350, X23 = 0.14000 pu on SBASE2-3 = 300 MVA + +R31 = 0.00400, X31 = 0.11000 pu on SBASE3-1 = 100 MVA + +**Convert pairwise to system base (CZ=2):** + +X12_system = 0.08500 \* (100 / 600) = 0.01417 pu + +X23_system = 0.14000 \* (100 / 300) = 0.04667 pu + +X31_system = 0.11000 \* (100 / 100) = 0.11000 pu + +**Star-bus decomposition (on system base):** + +X1 = (0.01417 + 0.11000 - 0.04667) / 2 = 0.03875 pu + +X2 = (0.01417 + 0.04667 - 0.11000) / 2 = -0.02458 pu + +X3 = (0.04667 + 0.11000 - 0.01417) / 2 = 0.07125 pu + +Note: X2 is negative, which is physically meaningful for autotransformers and indicates that winding 2 has a magnetizing effect. + +**Physical impedance values (winding 1):** + +Z_base,w1 = 500.0^2 / 100 = 2500.0 ohms + +X1_ohm = 0.03875 \* 2500.0 = 96.875 ohms + +**Per-unit to physical and back:** + +X1_pu = 96.875 / 2500.0 = 0.03875 pu (matches original) + +### Common Pitfalls + +1. **Base mismatch errors:** Each winding pair has a different MVA base (SBASE1-2, SBASE2-3, SBASE3-1). If a tool converts all three pairwise impedances using the same MVA base, the star-bus decomposition will be incorrect. The symptom is that the star impedances Z1, Z2, Z3 have incorrect magnitudes and the power flow solution shows voltage errors at the three-winding transformer buses. + +2. **Tool-specific conventions:** + - **MATPOWER** does not natively support three-winding transformers. They must be decomposed into three two-winding transformers connected at a virtual star bus. The MATPOWER PSS/E importer performs this decomposition automatically. + - **pandapower** supports three-winding transformers via `create_transformer3w()`. Parameters include per-winding short-circuit voltages (`vk_hv_percent`, `vk_mv_percent`, `vk_lv_percent`) referenced to each winding's rated MVA. The naming convention (HV/MV/LV) must be mapped correctly to PSS/E winding 1/2/3. + - **PyPSA** does not have native three-winding transformer support. Three-winding transformers must be manually decomposed into three two-winding transformers with a virtual star bus. + - **GridCal** supports three-winding transformers internally and can handle the star-bus decomposition. + - **PowerModels.jl** decomposes three-winding transformers into three branches at a star bus during PSS/E import. + - **PowerSimulations.jl** (via PowerSystems.jl) handles three-winding transformers through decomposition into two-winding equivalents. + +3. **Diagnostic signature:** If pairwise impedances are not converted to a common base before star-bus decomposition, the resulting Z1, Z2, Z3 values will be inconsistent. The most visible symptom is that the sum Z12_reconstructed = Z1 + Z2 does not equal the original Z12 after base conversion. The error ratio differs by winding pair, scaled by the ratio of the winding MVA bases. + +4. **Correction formula:** Always convert all pairwise impedances to the same MVA base (typically system base) before performing star-bus decomposition: + Z_pu,system = Z_pu,winding \* (SBASE / SBASEn-m) + +## 7. Shunt Admittance + +Shunt elements in PSS/E v31 are specified in MW and MVAR at unity per-unit voltage (V = 1.0 pu), not in per-unit. Tools that use per-unit admittance must convert these values. + +### Fixed Shunts (GL, BL) + +PSS/E convention: `GL` is the shunt conductance in MW at V = 1.0 per-unit, and `BL` is the shunt susceptance in MVAR at V = 1.0 per-unit. + +Per-unit conversion: + +G_pu = GL / S_base (per-unit conductance) + +B_pu = BL / S_base (per-unit susceptance) + +Sign convention: + +- Positive `BL` = capacitive (generating reactive power, lagging current) +- Negative `BL` = inductive (absorbing reactive power, leading current) +- Positive `GL` = real power absorption (resistive loss) + +### Switched Shunts (BINIT, B1-B8) + +Switched shunts follow the same convention as fixed shunts: MVAR at 1.0 per-unit voltage. + +- `BINIT`: initial total susceptance in MVAR +- `B1` through `B8`: susceptance per step in each switching block, in MVAR +- `N1` through `N8`: number of steps in each block + +Total susceptance range: sum of (Ni \* Bi) for all blocks. + +Per-unit conversion: B_pu = B_MVAR / S_base + +### Worked Example + +**Fixed shunt: 200 MVAR capacitor bank** -- +GL = 0.0 MW (no resistive losses), BL = 200.0 MVAR + +With S_base = 100 MVA: + +G_pu = 0.0 / 100 = 0.0000 pu + +B_pu = 200.0 / 100 = 2.0000 pu + +**Per-unit to physical:** + +BL = B_pu \* S_base = 2.0000 \* 100 = 200.0 MVAR (matches original) + +**Physical to per-unit:** + +B_pu = 200.0 / 100 = 2.0000 pu (matches) + +**Switched shunt: 4 steps of 50 MVAR** -- +N1 = 4, B1 = 50.0 MVAR per step + +BINIT = 150.0 MVAR (3 steps currently in service) + +Total range: 0 to 4 \* 50.0 = 200.0 MVAR + +BINIT_pu = 150.0 / 100 = 1.5000 pu + +Per step: B1_pu = 50.0 / 100 = 0.5000 pu + +### Common Pitfalls + +1. **Base mismatch errors:** A common error is treating GL/BL as already in per-unit. Since GL and BL are in MW and MVAR (not per-unit), importing them without dividing by S_base produces values 100x too large in the tool's internal model. The symptom is unrealistically large reactive power injections. + +2. **Tool-specific conventions:** + - **MATPOWER** stores shunt admittance in `mpc.bus(:, [GS BS])` in MW and MVAR (same as PSS/E convention), so no conversion is needed when importing from the intermediate format. + - **pandapower** stores shunts as separate elements (`net.shunt`) with `q_mvar` and `p_mw` at rated voltage. The sign convention for `q_mvar` may differ: pandapower uses positive for inductive, which is opposite to PSS/E's positive-capacitive BL convention. + - **PyPSA** stores shunt impedance in per-unit in `network.shunt_impedances`. The conversion from MVAR to per-unit must be performed during import: B_pu = BL / S_base. + - **GridCal** stores shunts in per-unit admittance, requiring the MVAR-to-per-unit conversion. + - **PowerModels.jl** stores shunt data in `bus["gs"]` and `bus["bs"]` in per-unit (not MW/MVAR), so conversion from PSS/E values is required: gs = GL / baseMVA, bs = BL / baseMVA. + - **PowerSimulations.jl** (via PowerSystems.jl) stores fixed shunts with admittance in per-unit. + +3. **Diagnostic signature:** If GL/BL values are imported without dividing by S_base, reactive power injections at shunt buses will be off by a factor of S_base (typically 100). The voltage profile will show severe over-voltages at buses with large capacitor banks. + +4. **Correction formula:** For tools requiring per-unit: G_pu = GL_MW / S_base; B_pu = BL_MVAR / S_base. Watch for sign convention differences (capacitive positive vs. inductive positive). + +## 8. Generator Capability + +Generator active and reactive power limits in PSS/E v31 are specified in physical units (MW and MVAR), not per-unit. The generator MVA base (`MBASE`) is distinct from the system MVA base (`SBASE`) and is relevant for machine impedance calculations. + +### Active Power (PG, PMAX, PMIN) + +Units: MW (not per-unit) + +- `PG`: current active power output in MW +- `PMAX`: maximum active power output in MW +- `PMIN`: minimum active power output in MW + +No per-unit conversion is needed for the intermediate format. Tools that require per-unit values must convert: P_pu = P_MW / S_base. + +### Reactive Power (QG, QMAX, QMIN) + +Units: MVAR (not per-unit) + +- `QG`: current reactive power output in MVAR +- `QMAX`: maximum reactive power output in MVAR +- `QMIN`: minimum reactive power output in MVAR + +No per-unit conversion is needed for the intermediate format. For tools requiring per-unit: Q_pu = Q_MVAR / S_base. + +### Generator MVA Base (MBASE) + +`MBASE` is the machine MVA base, used for machine impedance calculations (subtransient and transient reactances). It is distinct from `SBASE`: + +- `SBASE`: system-wide power base for network impedance (typically 100 MVA) +- `MBASE`: per-machine base for machine parameters (typically the generator nameplate MVA) + +Machine impedance on system base: X_pu,system = X_pu,machine \* (SBASE / MBASE) + +The MBASE field is present in the intermediate format but is primarily relevant for dynamic studies and transient stability, not steady-state power flow. + +### Voltage Setpoint (VS) + +The generator voltage setpoint `VS` is in per-unit on the bus `BASKV`: + +V_kV = VS \* BASKV + +A typical setpoint is VS = 1.02 to 1.05 per-unit. + +### Worked Example + +A 500 MW gas turbine at a 230 kV bus: + +- MBASE = 600 MVA +- PG = 450.0 MW, PMAX = 500.0 MW, PMIN = 100.0 MW +- QG = 120.0 MVAR, QMAX = 300.0 MVAR, QMIN = -150.0 MVAR +- VS = 1.035 per-unit, BASKV = 230.0 kV + +**Per-unit to physical (voltage):** + +V_kV = 1.035 \* 230.0 = 238.050 kV + +**Physical to per-unit (power, for tools requiring it):** + +With S_base = 100 MVA: + +P_pu = 450.0 / 100 = 4.500 pu + +Q_pu = 120.0 / 100 = 1.200 pu + +PMAX_pu = 500.0 / 100 = 5.000 pu + +**Physical to per-unit round-trip:** + +P_MW = 4.500 \* 100 = 450.0 MW (matches original) + +V_pu = 238.050 / 230.0 = 1.0350 pu (matches original) + +**Machine impedance conversion example:** + +Generator subtransient reactance X''d = 0.20 pu on MBASE = 600 MVA + +On system base: X''d_system = 0.20 \* (100 / 600) = 0.03333 pu + +### Common Pitfalls + +1. **Base mismatch errors:** Confusing `MBASE` with `SBASE` when converting machine impedance. If a tool converts machine reactances using `SBASE` instead of `MBASE`, the impedances will be off by a factor of MBASE / SBASE. For a 600 MVA machine on a 100 MVA system, the error factor is 6. + +2. **Tool-specific conventions:** + - **MATPOWER** stores generator output in `mpc.gen(:, [PG QG QMAX QMIN PMAX PMIN])` in MW/MVAR. The MBASE is stored in `mpc.gen(:, MBASE)`. Conversion to per-unit happens internally. + - **pandapower** stores generator data in `net.gen` with `p_mw`, `max_q_mvar`, `min_q_mvar` in physical units. Voltage setpoint is in per-unit (`vm_pu`). + - **PyPSA** stores generator power in MW in `network.generators.p_nom` (nominal capacity) and dispatch in `network.generators_t.p`. Voltage setpoint is `v_set_pu`. + - **GridCal** stores generator power in MW and voltage setpoint in per-unit. + - **PowerModels.jl** stores generator data in per-unit on the system base: `gen["pg"]` = PG/baseMVA, `gen["qg"]` = QG/baseMVA. + - **PowerSimulations.jl** stores generator data through PowerSystems.jl with `active_power` and `reactive_power` in per-unit on system base. + +3. **Diagnostic signature:** If PG/PMAX are imported as per-unit when they are actually in MW, all generator outputs will appear 100x too large (with S_base = 100). The power flow solution will not converge or will show extreme power mismatches. + +4. **Correction formula:** + - MW to per-unit: P_pu = P_MW / S_base + - Per-unit to MW: P_MW = P_pu \* S_base + - Machine base to system base: X_system = X_machine \* (S_base / MBASE) + +## 9. Load Representation + +Load active and reactive power in PSS/E v31 are specified in physical units (MW and MVAR), not per-unit. The intermediate format preserves these physical units. + +### Active/Reactive Load (PL, QL) + +Units: MW and MVAR (not per-unit) + +- `PL`: active power demand in MW (positive = load consuming power) +- `QL`: reactive power demand in MVAR (positive = load absorbing reactive power) + +### Per-Unit Conversion (for tools requiring it) + +P_pu = PL / S_base + +Q_pu = QL / S_base + +Inverse: + +PL = P_pu \* S_base + +QL = Q_pu \* S_base + +### Worked Example + +A 275.0 MW, 85.0 MVAR load with S_base = 100 MVA: + +**Physical to per-unit:** + +P_pu = 275.0 / 100 = 2.7500 pu + +Q_pu = 85.0 / 100 = 0.8500 pu + +**Per-unit to physical:** + +PL = 2.7500 \* 100 = 275.0 MW (matches original) + +QL = 0.8500 \* 100 = 85.0 MVAR (matches original) + +Round-trip confirmation: both values match to 4 significant digits. + +### Common Pitfalls + +1. **Base mismatch errors:** If a tool imports PL/QL as per-unit values when they are actually in MW/MVAR, the load will be modeled as S_base times too large. With S_base = 100, a 275 MW load would become 27,500 MW, causing the power flow to diverge. The symptom is non-convergence or extreme voltage collapse. + +2. **Tool-specific conventions:** + - **MATPOWER** stores loads in `mpc.bus(:, [PD QD])` in MW and MVAR, matching the PSS/E convention. No conversion needed. + - **pandapower** stores loads in `net.load` with `p_mw` and `q_mvar` in physical units. No conversion needed. + - **PyPSA** stores loads in MW in `network.loads.p_set` and `network.loads.q_set`. No conversion needed when importing from the intermediate format. + - **GridCal** stores loads in MW and MVAR, matching the intermediate format. + - **PowerModels.jl** stores loads in per-unit on the system base: `load["pd"]` = PL/baseMVA, `load["qd"]` = QL/baseMVA. Conversion from MW to per-unit is required during import. + - **PowerSimulations.jl** stores loads through PowerSystems.jl with `active_power` and `reactive_power` in per-unit on system base. Conversion required. + +3. **Diagnostic signature:** If MW/MVAR loads are imported as per-unit without conversion, total system load will be off by a factor of S_base. A quick check: sum(PL) in the tool should equal the sum of MW loads from the FNM. If the tool's total is 100x smaller, the loads were correctly converted to per-unit; if 100x larger, they were treated as per-unit when they are MW. + +4. **Correction formula:** + - MW to per-unit: P_pu = PL / S_base + - Per-unit to MW: PL = P_pu \* S_base + +## Summary Table + +| Domain | PSS/E Fields | Unit in Intermediate Format | Per-Unit Base | Conversion Formula (pu to physical) | +|--------|-------------|---------------------------|---------------|--------------------------------------| +| 1. System MVA Base | `SBASE` (header) | MVA | -- (this IS the base) | -- | +| 2. Bus Base Voltage | `BASKV` (Bus) | kV | -- (this IS the voltage base) | V_kV = V_pu \* BASKV | +| 3. Branch Impedance | `R`, `X`, `B` (Branch) | per-unit | S_base, BASKV_from | Z_ohm = Z_pu \* BASKV^2 / S_base | +| 4. 2W Transformer Impedance | `R1-2`, `X1-2`, `CZ` (Transformer) | per-unit (CZ-dependent) | Depends on CZ mode | Z_ohm = Z_pu \* BASKV^2 / S_base_effective | +| 5. 2W Transformer Taps | `WINDV1`, `WINDV2`, `CW` (Transformer) | CW-dependent | Depends on CW mode | tap_kV = WINDV \* BASKV (CW=1) | +| 6. 3W Transformer Bases | `R`, `X` per pair, `RATAn`, `CZ` (Transformer) | per-unit (CZ-dependent) | Per-winding MVA and kV | Z_ohm = Z_pu \* BASKV_wn^2 / S_base,wn | +| 7. Shunt Admittance | `GL`, `BL` (Fixed Shunt), `BINIT`, `Bi` (Switched Shunt) | MW / MVAR at V=1.0 pu | Not per-unit in PSS/E | G_pu = GL / S_base; B_pu = BL / S_base | +| 8. Generator Capability | `PG`, `PMAX`, `QG`, `QMAX`, `MBASE`, `VS` (Generator) | MW / MVAR (not pu); VS in pu | MW/MVAR: none; VS: BASKV | P_MW = P_pu \* S_base; V_kV = VS \* BASKV | +| 9. Load Representation | `PL`, `QL` (Load) | MW / MVAR (not pu) | none | PL = P_pu \* S_base | + +## Cross-References + +- **Phase 1 D7 — Intermediate Format Schema Specification:** Defines JSON Schema files with `perUnitBase` annotations for every field. The `PerUnitBase` enum (`system_mva`, `winding_mva`, `bus_kv`, `none`, `mixed`) classifies each field's per-unit base. This per-unit convention reference expands on that classification with formulas and worked examples. +- **Phase 2 PRD 01 — Intermediate Format Schema Reference:** Provides field semantic descriptions for every intermediate format field. This per-unit convention reference complements PRD 01 by covering the numerical conventions rather than the semantic definitions. +- **Phase 2 PRD 04 — Three-Winding Transformer Reference:** Documents full three-winding transformer topology, star-bus decomposition, and winding connection details. Section 6 of this document defers to PRD 04 for the complete three-winding treatment. +- **Phase 2 PRD 05 — Field Criticality Matrix:** Downstream consumer that uses per-unit base classification from this document to correctly classify per-unit-dependent fields by criticality level. diff --git a/data/fnm/docs/rubric-v4-justification.md b/data/fnm/docs/rubric-v4-justification.md new file mode 100644 index 00000000..1871cd6a --- /dev/null +++ b/data/fnm/docs/rubric-v4-justification.md @@ -0,0 +1,97 @@ +# Rubric v4 Amendment Justification + +Rubric v3 explicitly excludes the Full Network Model (FNM) from Phase 1 testing, stating: "The Full Network Model (FNM) is not used in Phase 1 testing." Rubric v4 amends this exclusion to incorporate FNM ingestion results as additive evidence for Expressiveness and Extensibility grades. The amendment creates no new grading criteria, introduces no new grade boundaries, and changes no A/B/C threshold definitions; FNM results inform grade narratives through grading notes appended to the existing Expressiveness and Extensibility criteria. All FNM-dependent tests are gated by the `FNM_PATH` environment variable, ensuring tools receive complete Phase 1 grades on all six criteria without FNM data. This amendment follows the precedent established by rubric v2, which added SCOPF, lossy DC OPF, and distributed slack OPF as supplementary sub-questions (Expressiveness 9--11) with an identical grading mechanism: informing the grade narrative without automatically overriding primary sub-question results. + +## Why Data Model Fidelity Belongs in Phase 1 + +Phase 1 evaluates tool capability, not deployment readiness. The rubric's two highest-priority criteria -- Expressiveness and Extensibility -- ask "can the tool represent the problems we need to solve?" and "can an analyst extend the tool beyond built-in problems?" Data model fidelity -- whether a tool's internal data model can represent the full parameter space of a production transmission network -- is a direct measurement of both. A tool whose data model cannot hold 3-winding transformers or switched shunt discrete steps has an expressiveness limitation. A tool that requires analyst-built workarounds to represent area interchange data has an extensibility cost. These are capability questions that belong in Phase 1. + +FNM ingestion is a data model test, not a scenario test. The question it answers is: "given a production network's full parameter space, can the tool's data model hold it without loss?" This is structurally identical to Phase 1's existing tests, which ask: "given a synthetic MATPOWER case, can the tool ingest and solve it?" The FNM simply provides a harder, more realistic input that exercises data model dimensions the synthetic cases cannot reach. No scenario analysis, congestion reproduction, or market clearing simulation is involved -- the test is purely about data model fidelity. + +The distinction between Phase 1 and Phase 2 is not synthetic-versus-production data. It is capability-assessment versus operational-workflow. Phase 1 asks whether the tool *can* represent, ingest, and solve. Phase 2 asks whether the tool *should be used* for specific operational tasks such as congestion pattern reproduction and market clearing simulation. FNM ingestion testing -- "can the tool hold this network?" -- is firmly on the Phase 1 side of this boundary. + +Deferring data model fidelity testing entirely to Phase 2 creates a systematic gap in Phase 1 evaluation. Phase 1 grades would assess expressiveness only against synthetic cases that lack critical record types and fields, producing artificially similar grades across tools that have very different production-network capabilities. This is analogous to evaluating a programming language only against trivial programs -- the tools appear equivalent until confronted with real-world complexity. Incorporating FNM evidence in Phase 1 closes this gap without converting Phase 1 into a deployment assessment. + +## Evidence: FNM vs. Synthetic Case Coverage + +The synthetic MATPOWER test cases (ACTIVSg 2k and 10k) use the MATPOWER .m format, a simplified representation of an AC transmission network. The .m format represents buses, branches (AC lines), generators, generator cost curves, and basic transformer data as branch-table rows. It does not natively represent many PSS/E v31 record types that exist in production ISO network models like the FNM. This structural limitation means that Phase 1 testing on synthetic cases alone produces no evidence -- positive or negative -- about a tool's ability to handle record types absent from the .m format. + +### Table 1: Record Types Present in FNM but Absent or Degraded in Synthetic Cases + +The following table identifies PSS/E v31 record types that are non-empty in the FNM but cannot be faithfully represented in MATPOWER .m format. Tier classifications are from the Record-Type Mapping Guide (`mapping-guide.md`). + +| Record Type | PSS/E Section | Tier | FNM Status | Synthetic Case Status | +|-------------|---------------|------|------------|-----------------------| +| 3-Winding Transformer | Section 6 (K!=0) | 1 | Non-empty | MATPOWER .m branch table has no K field for 3-winding topology; `psse2mpc` decomposes 3-winding transformers to star-bus 2-winding equivalents, so tool support for native 3-winding representation is untestable | +| Switched Shunt | Section 17 | 2 | Non-empty | MATPOWER .m represents shunts via the bus-table BS column as continuous min/max susceptance bounds; discrete step structure (N1/B1 through N8/B8), voltage regulation targets (VSWHI/VSWLO), and control mode (MODSW) are collapsed and lost | +| Area | Section 7 | 2 | Non-empty | MATPOWER .m `mpc.areas` matrix stores only area number and slack bus (ISW); desired interchange (PDES) and interchange tolerance (PTOL) are present but area interchange control semantics are not exercised in synthetic test suites | +| Fixed Shunt | Section 3 | 2 | Non-empty | MATPOWER .m folds fixed shunts into bus-table GS/BS columns; multiple shunts at the same bus are summed into a single pair of values, losing individual shunt identity, per-shunt status control, and shunt identifiers | +| Owner | Section 15 | 3 | Non-empty | No representation in the MATPOWER .m `mpc` struct; ownership data on generators, branches, and transformers is silently discarded by `psse2mpc` conversion | +| Zone | Section 13 | 3 | Non-empty | MATPOWER .m bus matrix includes a ZONE column for bus-to-zone assignment, but the zone name table has no .m counterpart; zone semantics are reduced to integer labels | + +For any record type absent from synthetic cases, Phase 1 produces no evidence of tool capability. The tool receives neither credit for supporting it nor penalty for lacking it. This blind spot is most consequential for Tier 1 and Tier 2 record types that directly affect power flow accuracy. For example, 3-winding transformers are a Tier 1 record type -- essential for correct network topology -- yet no synthetic MATPOWER case can test whether a tool supports them natively or requires star-bus decomposition. + +### Table 2: Field Coverage Gap Summary + +The following table quantifies the field-level coverage gap between the FNM intermediate format (which represents all PSS/E v31 fields across all non-empty record types) and what MATPOWER .m cases can exercise. Counts are derived from the Field Criticality Matrix (`field-criticality-matrix.md`) summary table, which classifies all 350 fields across 17 record types. + +The 10 non-empty record types in the FNM (Bus, Load, Fixed Shunt, Generator, Branch, Transformer, Area, Zone, Owner, Switched Shunt) account for 201 of the 350 total fields. Of these 201 fields, only a subset can be exercised through MATPOWER .m synthetic cases, because the .m format either lacks the record type entirely (Owner), collapses the record type into bus-level aggregates (Fixed Shunt, Switched Shunt), or omits fields that the PSS/E format carries (e.g., transformer control mode codes, switched shunt discrete step blocks). + +| Metric | FNM Intermediate Format | MATPOWER .m Cases | Gap | +|--------|------------------------|-------------------|-----| +| Total fields across non-empty record types | 201 | ~120 | ~81 | +| DCPF-critical fields | 26 | 26 | 0 | +| ACPF-critical fields | 93 | ~25 | ~68 | +| Record types with non-empty FNM representation | 10 | 8 (with fidelity loss) | 2 fully absent | + +The DCPF-critical field gap is zero because all 26 DCPF-critical fields fall within Bus, Load, Generator, Branch, and Transformer record types that MATPOWER .m can represent at the topology level. The ACPF-critical field gap is substantial: 93 ACPF-critical fields exist across the non-empty FNM record types, but only approximately 25 are exercisable through .m cases. The remaining ~68 ACPF-critical fields reside in record types that .m either omits (Switched Shunt discrete steps, Area interchange parameters) or represents with fidelity loss (transformer control modes, fixed shunt aggregation). This gap distinguishes between "field exists in FNM but not in .m" (a format limitation preventing any test coverage) and "field exists in both but .m representation is lossy" (a fidelity limitation that may mask tool-level differences). + +Beyond record type and field coverage, the FNM's approximately 30,000 buses and approximately 39,000 transformers create a parameter-space breadth that cannot be tested on a 10,000-bus synthetic case. Parameter interactions that only surface at production scale -- switched shunt hunting between discrete steps, transformer tap oscillation near control mode boundaries, area interchange convergence sensitivity with dozens of interconnected areas -- are invisible in smaller cases. The FNM exercises these interactions simultaneously across the full network, providing evidence of data model robustness that no synthetic case can replicate. + +As concrete examples of tool differentiation that FNM testing reveals: pandapower has native 3-winding transformer support via `create_transformer3w()` while PyPSA requires star-bus decomposition into multiple 2-winding `Transformer` components -- this distinction is untestable without a network that contains 3-winding transformer records (K!=0). Similarly, GridCal has a `ControllableShunt` object with discrete step modeling and voltage regulation targets while MATPOWER collapses discrete steps to continuous susceptance bounds -- this distinction is untestable without a network carrying switched shunt discrete step data (N1/B1 through N8/B8 fields). These are not edge cases; they are fundamental data model design differences that determine whether an analyst can work with production network data or must first transform it into a simplified representation. + +## Precedent: Rubric v2 Scope Expansion + +Rubric v2 expanded Phase 1's scope by adding supplementary sub-questions to the Expressiveness and Extensibility criteria. Specifically, v2 added Expressiveness sub-questions 9 (Security-Constrained OPF), 10 (Lossy DC OPF and LMP Decomposition), and 11 (Distributed Slack OPF), plus Extensibility sub-questions 8 (Reference Bus Control) and 9 (PTDF Matrix Extraction). These additions were motivated by research into ISO market clearing engines, which revealed that the original rubric's coverage of analytical primitives was insufficient for Phase 2 congestion pattern reproduction. A "Phase 2 Context" section was added to the rubric explaining this motivation. + +The grading mechanism v2 established is directly relevant. The grading note after Expressiveness sub-question 11 states: "These sub-questions are supplementary Phase 2 readiness indicators. The original sub-questions (1--8) remain the primary drivers of the Expressiveness grade. Sub-questions 9--11 inform whether the tool is ready for ISO congestion pattern reproduction. A tool that scores well on 1--8 but poorly on 9--11 receives a grade note, not an automatic downgrade." This precedent establishes the pattern: scope expansion is permissible when it adds evidence that the original scope could not provide, as long as the new evidence informs rather than overrides existing grade boundaries. + +The FNM amendment follows the identical pattern. FNM test results (Suite G) are supplementary production-network fidelity indicators. The original test suites (Suites A--F on synthetic cases) remain the primary drivers of Expressiveness and Extensibility grades. Suite G results inform whether the tool can handle production-scale data model complexity. A tool that passes Suites A--F but fails Suite G receives a grade note, not an automatic downgrade. + +The v2 amendment was itself a scope expansion that could have been deferred to Phase 2: SCOPF and lossy DC OPF are directly relevant to ISO congestion reproduction, which is Phase 2's operational domain. The v2 amendment argued that *capability to express* SCOPF is a Phase 1 question, while *using* SCOPF for congestion reproduction is Phase 2. The same logic applies here: *capability to ingest* a production network is a Phase 1 data model fidelity question; *using* that network for market simulation is Phase 2. + +## Grading Impact + +FNM ingestion results inform the Expressiveness grade via a grading note under the existing Expressiveness criterion. The note documents a tool's ability to represent all PSS/E record types from a production network as evidence of expressiveness beyond what synthetic cases can test. Specifically, record-type coverage from Suite G gate tests demonstrates whether the tool's data model can hold the full complexity of a production network. Tools that ingest all Tier 1 and Tier 2 record types present in the FNM receive a positive grading note acknowledging production-network data model fidelity. Tools that fail to ingest Tier 1 record types present in the FNM -- for example, tools that cannot represent 3-winding transformers at all -- receive a negative grading note under Expressiveness documenting the gap and its tier classification. Neither note changes the A/B/C threshold definitions. + +Supplemental CSV representability results inform the Extensibility grade via a grading note under the existing Extensibility criterion. The proportion of supplemental data that requires tool-external handling -- from the Phase 4 D2 representability summary -- directly indicates how much post-ingestion extension work an analyst faces when working with production data. This is a concrete, production-data-derived measurement of extensibility that complements the synthetic-case extension tests in Suite B. + +FNM results do not inform Scalability (Suite G is not a performance benchmark), Accessibility (FNM ingestion difficulty is an Expressiveness question, not a usability question), Maturity, or Supply Chain grades. The FNM is tested only through the intermediate format, so raw PSS/E parsing capability is not graded. + +Grade boundaries remain unchanged. The A/B/C threshold definitions for all six criteria remain exactly as written in rubric v3. No threshold is tightened, loosened, or conditioned on FNM results. The FNM grading notes operate within the existing evaluator judgment latitude that the rubric's +/- modifier system already provides. A grade of B+ versus B, for example, may be influenced by FNM evidence, but the boundary between B-range and C-range grades is not moved. + +## FNM_PATH Gating + +All Suite G (FNM Ingestion) tests are gated by the `FNM_PATH` environment variable. When `FNM_PATH` is unset, all Suite G tests skip with a clear message: "FNM data not available -- Suite G skipped." This is the same gating pattern used throughout the FNM expansion infrastructure established in Phase 1 D2. + +A tool can receive complete Phase 1 grades on all six criteria based solely on Suites A--F results. FNM evidence is additive: its presence strengthens or weakens the grade narrative, but its absence does not create a gap, an incomplete grade, or a lower default score. An evaluation run without FNM data produces grades that are valid and complete on their own terms. + +This means two evaluation runs of the same tool -- one with `FNM_PATH` set and one without -- may produce the same letter grade but different grade narratives. The run with FNM data has a richer evidence base that provides more granular insight into the tool's production-network capabilities. The run without FNM data is not penalized for the missing evidence; it simply has fewer data points informing the narrative. + +The gating design also means that this rubric amendment has zero impact on evaluation runs that do not have access to FNM data. The amendment is forward-compatible: it adds capability for enriched evaluation when FNM data is available, without degrading evaluations that proceed without it. Organizations or analysts who do not have access to NDA-restricted FNM data can use the rubric v4 without any change to their evaluation workflow. + +## Handling Synthetic-Pass / FNM-Fail Results + +The scenario where a tool passes all synthetic-case tests (Suites A--F) but fails FNM ingestion (Suite G) is expected and informative, not anomalous. Synthetic cases use a simplified data format (MATPOWER .m) with a subset of record types. A tool optimized for .m ingestion may demonstrate strong synthetic performance but lack data model support for PSS/E-derived record types like 3-winding transformers or switched shunt discrete steps. This outcome reveals a real capability boundary that synthetic testing alone cannot expose. + +The grade response depends on the failure mode. **Missing record type support** -- where the tool cannot represent a record type present in the FNM -- is an Expressiveness finding. The grade note documents which record types the tool cannot represent and their tier classification. A Tier 1 gap (e.g., cannot represent 3-winding transformers at all) is a stronger negative signal than a Tier 3 gap (e.g., cannot represent owner data). **Scale limitation** -- where the tool crashes or fails to converge on a ~30,000-bus network but handles the record types correctly on smaller subsets -- is a Scalability finding, not an Expressiveness finding. It is documented under Scalability, not Expressiveness. **Field coverage gap** -- where the tool ingests the record type but drops critical fields -- is a nuanced Expressiveness finding. The grade note references the field criticality tier of the dropped fields, distinguishing between DCPF-critical field loss (a severe finding) and Informational field loss (a minor finding). + +In no case does a Suite G failure automatically downgrade a tool below the grade it would have received from Suites A--F alone. The Suite G results add resolution to the grade narrative -- they explain *why* a tool might struggle at production scale and *where* its data model has gaps -- but the A/B/C grade boundary is still determined by the synthetic-case evidence plus evaluator judgment. Suite G provides the evidence; the evaluator applies it within the existing grading framework. + +## Cross-References + +- **Phase 2 D2: Record-Type Mapping Guide** (`mapping-guide.md`) -- source for record type tier classifications (Tier 1/2/3) and tool support matrix (Y/P/N per tool per record type) +- **Phase 2 D5: Field Criticality Matrix** (`field-criticality-matrix.md`) -- source for field-level coverage gap analysis and DCPF-critical / ACPF-critical field counts +- **Rubric v4 Amendment** (`../../evaluation_guides/Phase1_Evaluation_Rubric.md`) -- the rubric that references this justification document +- **Rubric v2 grading note** -- Expressiveness sub-question 11 grading note in the current rubric (the precedent for supplementary Phase 2 readiness indicators) +- **Executive plan** (`../../plans/fnm-ingestion-expansion/executive-plan.md`) -- the plan that mandated this rubric amendment as part of the FNM ingestion expansion diff --git a/data/fnm/docs/supplemental-csv-representability.md b/data/fnm/docs/supplemental-csv-representability.md new file mode 100644 index 00000000..f47a7fc5 --- /dev/null +++ b/data/fnm/docs/supplemental-csv-representability.md @@ -0,0 +1,104 @@ +# Supplemental CSV Representability Summary + +**Version:** 1.0 +**Source document:** [Supplemental CSV Reference Documentation](supplemental-csvs.md) +**Audience:** evaluate-tool agents, human reviewers +**Purpose:** Cross-tool summary of supplemental CSV representability, + optimized for quick Extensibility grading consumption. All classifications + are derived from the per-field analysis in the source document. + +## Representability Tiers + +| Tier | Label | Definition | +|------|-------|------------| +| Native | `native` | Tool has a built-in attribute, property, or field that directly represents this data. No custom code or extension mechanism required. | +| Extension | `extension` | Tool can carry this data via its documented extension mechanisms (custom attributes, metadata dictionaries, user-defined component fields) without forking or patching the tool's source code. | +| External | `external` | No representation path within the tool. Data must be carried in an external data structure (DataFrame, dict, database) alongside the tool's network model. | + +## CSV-Level Representability Matrix + +| CSV | Fields | PyPSA | pandapower | GridCal | PowerModels.jl | PowerSimulations.jl | MATPOWER | +|-----|--------|-------|------------|---------|----------------|---------------------|----------| +| LINE_AND_TRANSFORMER.csv | 10 | 40% native, 60% extension, 0% external | 40% native, 60% extension, 0% external | 40% native, 60% extension, 0% external | 60% native, 40% extension, 0% external | 40% native, 60% extension, 0% external | 60% native, 40% extension, 0% external | +| TRADING_HUB.csv | 4 | 25% native, 0% extension, 75% external | 25% native, 0% extension, 75% external | 25% native, 0% extension, 75% external | 25% native, 0% extension, 75% external | 25% native, 0% extension, 75% external | 25% native, 0% extension, 75% external | +| GEN_DISTRIBUTION_FACTOR.csv | 5 | 40% native, 20% extension, 40% external | 40% native, 20% extension, 40% external | 40% native, 20% extension, 40% external | 40% native, 20% extension, 40% external | 40% native, 20% extension, 40% external | 20% native, 40% extension, 40% external | +| CONTINGENCY.csv | 6 | 50% native, 17% extension, 33% external | 50% native, 17% extension, 33% external | 83% native, 17% extension, 0% external | 50% native, 17% extension, 33% external | 83% native, 17% extension, 0% external | 50% native, 17% extension, 33% external | +| INTERFACE.csv | 5 | 0% native, 0% extension, 100% external | 0% native, 0% extension, 100% external | 0% native, 0% extension, 100% external | 0% native, 0% extension, 100% external | 60% native, 40% extension, 0% external | 40% native, 40% extension, 20% external | +| INTERFACE_ELEMENT.csv | 6 | 33% native, 17% extension, 50% external | 33% native, 17% extension, 50% external | 33% native, 17% extension, 50% external | 33% native, 17% extension, 50% external | 67% native, 33% extension, 0% external | 67% native, 33% extension, 0% external | +| OUTAGE.csv | 8 | 38% native, 12% extension, 50% external | 38% native, 12% extension, 50% external | 38% native, 12% extension, 50% external | 38% native, 12% extension, 50% external | 38% native, 12% extension, 50% external | 38% native, 12% extension, 50% external | +| **Totals** | **44** | 34% native, 23% extension, 43% external | 34% native, 23% extension, 43% external | 39% native, 23% extension, 38% external | 39% native, 18% extension, 43% external | 50% native, 30% extension, 20% external | 45% native, 27% extension, 28% external | + +**Derivation Note:** Percentages are calculated as `(fields in tier / total classifiable fields) * 100` per CSV per tool. The Totals row uses field-count-weighted aggregation: each CSV's native/extension/external field counts are summed across all 7 CSVs, and the percentage is computed from the aggregate counts divided by the total field count (44). This means a CSV with more fields (e.g., LINE_AND_TRANSFORMER.csv with 10 fields) contributes proportionally more to the total than a CSV with fewer fields (e.g., TRADING_HUB.csv with 4 fields). The source field counts and per-field classifications are in the [Supplemental CSV Reference Documentation](supplemental-csvs.md), in each CSV's "Per-Field Representability" section (e.g., [LINE_AND_TRANSFORMER per-field representability](supplemental-csvs.md#line-and-transformer--per-field-representability)). + +## Concept-Level Representability Matrix + +| Data Concept | Source CSV(s) | PyPSA | pandapower | GridCal | PowerModels.jl | PowerSimulations.jl | MATPOWER | +|--------------|---------------|-------|------------|---------|----------------|---------------------|----------| +| Thermal Ratings (4-tier) | LINE_AND_TRANSFORMER.csv | `extension` | `extension` | `extension` | `extension` | `extension` | `extension` | +| Seasonal / Temporal Rating Variations | LINE_AND_TRANSFORMER.csv | `extension` | `extension` | `extension` | `extension` | `extension` | `extension` | +| Trading Hub Definitions | TRADING_HUB.csv | `external` | `external` | `external` | `external` | `external` | `external` | +| Generator Distribution Factors | GEN_DISTRIBUTION_FACTOR.csv | `external` | `external` | `external` | `external` | `external` | `external` | +| Contingency Definitions | CONTINGENCY.csv | `external` | `external` | `native` | `external` | `native` | `external` | +| Interface Definitions and Flow Limits | INTERFACE.csv, INTERFACE_ELEMENT.csv | `external` | `external` | `external` | `external` | `extension` | `extension` | +| Outage Actions / Planned Outage Parameters | OUTAGE.csv | `external` | `external` | `external` | `external` | `external` | `external` | +| Ownership and Operational Metadata | LINE_AND_TRANSFORMER.csv | `extension` | `extension` | `extension` | `extension` | `extension` | `extension` | + +**Classification Rule:** The concept-level classification for each tool follows a conservative (lowest-tier) aggregation rule: the concept's tier is the lowest tier among all its constituent fields, where `native` > `extension` > `external`. This means a concept is classified as `native` only if ALL constituent fields are natively representable. If any single field within the concept is `external`, the entire concept is classified as `external` for that tool. This reflects the practical reality that an analyst cannot fully use a data concept within the tool if any part of it requires external handling. For the per-field breakdown that produces each concept-level classification, see the [Supplemental CSV Reference Documentation](supplemental-csvs.md). + +## Key Findings + +### Richest Native Coverage + +PowerSimulations.jl achieves the highest native-tier percentage in the CSV-Level Representability Matrix at 50% native across all 44 fields. This lead is driven by its first-class support for contingency definitions (83% native in CONTINGENCY.csv via the `Contingency` type), transmission interfaces (60% native in INTERFACE.csv and 67% native in INTERFACE_ELEMENT.csv via `TransmissionInterface`), and standard network element fields. MATPOWER ranks second at 45% native overall, benefiting from native support for 3 thermal rating tiers (RATE_A/B/C in `mpc.branch`) and interface definitions (`mpc.if` and `mpc.iflim`). + +In the Concept-Level Matrix, PowerSimulations.jl is the only tool with `native` classification for contingency definitions. Both PowerSimulations.jl and MATPOWER achieve `extension` for interface definitions and flow limits, while all other tools classify this concept as `external`. + +GridCal matches PowerSimulations.jl in contingency coverage (83% native in CONTINGENCY.csv) thanks to its native `ContingencyGroup` object, but lacks interface support, resulting in a lower overall native percentage of 39%. + +PyPSA and pandapower cluster at 34% native, while PowerModels.jl reaches 39% native (matching GridCal) due to its native support for 3 thermal rating tiers. All four tools share similar gaps: no native concepts for contingencies (except GridCal), interfaces, or market-layer data. + +### Universally Tool-External CSVs + +**TRADING_HUB.csv** is the most uniformly tool-external CSV: all 6 tools have identical representability profiles (25% native, 0% extension, 75% external). The only natively representable field is BUS_NUMBER (the physical bus reference); HUB_NAME, DISTRIBUTION_FACTOR, and HUB_TYPE are tool-external across all tools because trading hubs are a market-layer abstraction with no analog in any power flow tool's domain model. + +**OUTAGE.csv** is effectively universally tool-external for its domain-specific fields: all 6 tools have identical profiles (38% native, 12% extension, 50% external). The natively representable fields are physical element identifiers (bus numbers) rather than outage-specific data. The outage schedule fields (OUTAGE_START, OUTAGE_END, OUTAGE_TYPE, ELEMENT_TYPE) are tool-external across all tools because no tool models temporal outage schedules. + +**INTERFACE.csv** is 100% tool-external in 4 of 6 tools (PyPSA, pandapower, GridCal, PowerModels.jl). Only PowerSimulations.jl and MATPOWER have any native interface representation. + +### Most Consequential Gaps for Phase 2 + +**Interface definitions and flow limits** represent the most consequential representability gap for Phase 2 congestion analysis readiness. Interfaces (flowgates) are the primary mechanism ISOs use to manage transmission corridor congestion, and the inability to represent interface definitions within a tool means congestion analysis must rely entirely on external data structures and custom scripting. PyPSA, pandapower, GridCal, and PowerModels.jl all classify INTERFACE.csv as 100% external and INTERFACE_ELEMENT.csv as 50% external (Table 1), with the concept-level classification of `external` (Table 2). This gap is not addressable via extension mechanisms for these four tools because the interface concept itself -- a named group of branches with aggregate flow limits and direction coefficients -- has no structural analog in their data models. Only PowerSimulations.jl (via `TransmissionInterface`) and MATPOWER (via `mpc.if`/`mpc.iflim`) can carry this data internally. + +See [INTERFACE per-field representability](supplemental-csvs.md#interfacecsv) and [INTERFACE_ELEMENT per-field representability](supplemental-csvs.md#interface_elementcsv) in D1 for the per-field evidence. + +**Trading hub definitions** are the second most consequential gap. Without hub definitions, a tool cannot model hub-based congestion revenue rights (CRRs) or compute hub-level locational marginal prices (LMPs) from nodal results. All 6 tools classify the trading hub concept as `external` (Table 2), meaning hub data must be maintained in external DataFrames or dictionaries and manually associated with bus-level results post-solution. This gap is inherent to all tools because hubs are a market construct outside the power flow domain. + +See [TRADING_HUB per-field representability](supplemental-csvs.md#trading_hubcsv) in D1. + +**Contingency definitions** affect 4 of 6 tools (PyPSA, pandapower, PowerModels.jl, MATPOWER), which classify CONTINGENCY_NAME and ELEMENT_TYPE as tool-external. Without native contingency definitions, these tools require external scripting to enumerate and apply contingency scenarios for N-1/N-2 analysis. The gap is partially addressable via extension mechanisms (storing contingency metadata in custom fields) but the absence of a native contingency object means the tool's solver cannot directly consume contingency definitions. GridCal and PowerSimulations.jl do not have this gap. + +See [CONTINGENCY per-field representability](supplemental-csvs.md#contingencycsv) in D1. + +### Tool Landscape Summary + +The supplemental CSV representability landscape reveals a clear stratification among the six tools. PowerSimulations.jl stands out with the broadest native coverage (50%) due to its first-class support for contingencies, interfaces, and standard network elements. MATPOWER follows (45%) with native interface and multi-tier thermal rating support. GridCal and PowerModels.jl sit at 39% native, with GridCal's advantage in contingency support offset by PowerModels.jl's additional thermal rating tiers. PyPSA and pandapower trail at 34% native. The practical implication for analysts is significant: even with the best-performing tool, half of supplemental CSV fields require extension or external handling, and market-layer concepts (trading hubs, generator distribution factors) and temporal concepts (outage schedules, seasonal ratings) are universally outside all tools' native domain models. + +## Traceability Index + +| Matrix | Row | D1 Section Reference | +|--------|-----|---------------------| +| Table 1 | LINE_AND_TRANSFORMER.csv | [LINE_AND_TRANSFORMER per-field representability](supplemental-csvs.md#line_and_transformercsv) -- fields: FROM_BUS, TO_BUS, CKT, ELEMENT_TYPE, RATE_A, RATE_B, RATE_C, RATE_D, STATUS, EFFECTIVE_DATE | +| Table 1 | TRADING_HUB.csv | [TRADING_HUB per-field representability](supplemental-csvs.md#trading_hubcsv) -- fields: HUB_NAME, BUS_NUMBER, DISTRIBUTION_FACTOR, HUB_TYPE | +| Table 1 | GEN_DISTRIBUTION_FACTOR.csv | [GEN_DISTRIBUTION_FACTOR per-field representability](supplemental-csvs.md#gen_distribution_factorcsv) -- fields: GEN_BUS, GEN_ID, HUB_NAME, PARTICIPATION_FACTOR, GEN_NAME | +| Table 1 | CONTINGENCY.csv | [CONTINGENCY per-field representability](supplemental-csvs.md#contingencycsv) -- fields: CONTINGENCY_NAME, ELEMENT_TYPE, ELEMENT_FROM_BUS, ELEMENT_TO_BUS, ELEMENT_CKT, ELEMENT_BUS | +| Table 1 | INTERFACE.csv | [INTERFACE per-field representability](supplemental-csvs.md#interfacecsv) -- fields: INTERFACE_ID, INTERFACE_NAME, NORMAL_LIMIT_MW, EMERGENCY_LIMIT_MW, DIRECTION | +| Table 1 | INTERFACE_ELEMENT.csv | [INTERFACE_ELEMENT per-field representability](supplemental-csvs.md#interface_elementcsv) -- fields: INTERFACE_ID, FROM_BUS, TO_BUS, CKT, DIRECTION_COEFF, WEIGHT_FACTOR | +| Table 1 | OUTAGE.csv | [OUTAGE per-field representability](supplemental-csvs.md#outagecsv) -- fields: ELEMENT_TYPE, ELEMENT_FROM_BUS, ELEMENT_TO_BUS, ELEMENT_CKT, ELEMENT_BUS, OUTAGE_START, OUTAGE_END, OUTAGE_TYPE | +| Table 2 | Thermal Ratings (4-tier) | [LINE_AND_TRANSFORMER per-field representability](supplemental-csvs.md#line_and_transformercsv) -- fields: RATE_A, RATE_B, RATE_C, RATE_D | +| Table 2 | Seasonal / Temporal Rating Variations | [LINE_AND_TRANSFORMER per-field representability](supplemental-csvs.md#line_and_transformercsv) -- fields: EFFECTIVE_DATE, STATUS | +| Table 2 | Trading Hub Definitions | [TRADING_HUB per-field representability](supplemental-csvs.md#trading_hubcsv) -- fields: HUB_NAME, BUS_NUMBER, DISTRIBUTION_FACTOR, HUB_TYPE | +| Table 2 | Generator Distribution Factors | [GEN_DISTRIBUTION_FACTOR per-field representability](supplemental-csvs.md#gen_distribution_factorcsv) -- fields: GEN_BUS, GEN_ID, HUB_NAME, PARTICIPATION_FACTOR, GEN_NAME | +| Table 2 | Contingency Definitions | [CONTINGENCY per-field representability](supplemental-csvs.md#contingencycsv) -- fields: CONTINGENCY_NAME, ELEMENT_TYPE, ELEMENT_FROM_BUS, ELEMENT_TO_BUS, ELEMENT_CKT, ELEMENT_BUS | +| Table 2 | Interface Definitions and Flow Limits | [INTERFACE per-field representability](supplemental-csvs.md#interfacecsv) and [INTERFACE_ELEMENT per-field representability](supplemental-csvs.md#interface_elementcsv) -- all fields from both CSVs | +| Table 2 | Outage Actions / Planned Outage Parameters | [OUTAGE per-field representability](supplemental-csvs.md#outagecsv) -- fields: ELEMENT_TYPE, OUTAGE_START, OUTAGE_END, OUTAGE_TYPE | +| Table 2 | Ownership and Operational Metadata | [LINE_AND_TRANSFORMER per-field representability](supplemental-csvs.md#line_and_transformercsv) -- fields: CKT, ELEMENT_TYPE, STATUS, EFFECTIVE_DATE | diff --git a/data/fnm/docs/supplemental-csvs.md b/data/fnm/docs/supplemental-csvs.md new file mode 100644 index 00000000..337bf86e --- /dev/null +++ b/data/fnm/docs/supplemental-csvs.md @@ -0,0 +1,477 @@ +# Supplemental CSV Reference Documentation + +**Version:** 1.0 +**Audience:** evaluate-tool agents, human reviewers +**Scope:** 7 supplemental CSVs from the FNM Annual S01 variant +**Join key source:** Phase 1 D9 join-key mapping report +**Representability method:** Analytical classification with empirical spot-checks +**Tools evaluated:** PyPSA, pandapower, GridCal, PowerModels.jl, PowerSimulations.jl, MATPOWER + +## Representability Classification System + +This document classifies every field in the 7 supplemental CSVs according to a three-tier representability system that determines whether and how each tool can carry the data. The three tiers are: + +**Natively-representable (N):** The tool has a documented native attribute that directly stores this data without any extension mechanism. Every N classification in this document cites the specific native attribute -- for example, MATPOWER `mpc.branch` column `RATE_A` for a thermal rating field. A field classified as N can be ingested into the tool's data model and participates in the tool's standard computations (power flow, OPF, contingency analysis) without custom code. + +**Extension-representable (E):** The tool can carry this data via its documented extension mechanisms (custom attributes, metadata dictionaries, user-defined fields) without forking or patching the tool's source code. Every E classification cites the specific extension mechanism -- for example, PyPSA custom component attributes via DataFrame column assignment, or PowerSystems.jl `ext::Dict{String,Any}` on component types. Extension-representable data is preserved within the tool's data model but is not semantically interpreted by the tool's solvers or algorithms. It must be accessed and processed by custom user code. + +**Tool-external (X):** No representation path exists within the tool's data model. The data must be maintained in an external data structure (DataFrame, dict, database) alongside the tool's network model. Every X classification states the reason -- either the data concept has no analog in the tool's domain model, or the tool lacks extension mechanisms that could accommodate it. In practice, all six evaluated tools have some extension mechanism, so X classifications typically arise when the data concept is entirely outside the tool's domain (e.g., trading hub definitions in a power flow solver). + +The relationship between this field-level representability classification and the record-type support matrix in the Record-Type Mapping Guide (Phase 2 D2) is important: a record type with "Y" (native) support in D2 does not guarantee that all supplemental CSV fields associated with that record type are Natively-representable (N) at the field level. For example, a tool may natively represent branches (Y for Branch record type) but have no native attribute for a 4th thermal rating tier. Field-level classification is strictly more granular than record-type classification. + +The proportion of X (tool-external) fields for a given CSV directly indicates the post-ingestion extension burden for an analyst using that tool. A tool with many X fields requires the analyst to maintain parallel data structures and manually associate them with network elements, increasing complexity and error risk. This informs the Extensibility dimension of the evaluation rubric. + +## CSV Overview + +| CSV File | Domain | Purpose | Columns | Join Target | Join Cardinality | Join Match Rate | +|----------|--------|---------|---------|-------------|-----------------|-----------------| +| `LINE_AND_TRANSFORMER.csv` | Transmission | Thermal ratings and operational parameters for lines and transformers | 10 | branch, transformer | N:1 | 97.2% | +| `CONTINGENCY.csv` | Transmission | Contingency definitions for N-1/N-2 reliability analysis | 6 | branch, generator | M:N | 94.8% | +| `INTERFACE.csv` | Transmission | Transmission interface definitions with flow limits | 5 | (via INTERFACE_ELEMENT) | 1:N | N/A | +| `INTERFACE_ELEMENT.csv` | Transmission | Branch elements comprising each interface with direction and weighting | 6 | branch | N:1 | 96.5% | +| `GEN_DISTRIBUTION_FACTOR.csv` | Generation | Generator participation/distribution factors for hub allocation | 5 | generator | N:1 | 98.1% | +| `TRADING_HUB.csv` | Market | Trading hub definitions mapping hubs to buses with distribution factors | 4 | bus | N:1 | 99.3% | +| `OUTAGE.csv` | Outage | Scheduled and forced outage definitions with dates and types | 8 | branch, generator | M:N | 95.6% | + +## Extension Mechanisms by Tool + +| Tool | Extension Mechanism | Mechanism Description | Citation | +|------|--------------------|-----------------------|----------| +| PyPSA | Custom component attributes | Additional columns on component DataFrames via `n.madd()` or direct DataFrame assignment | PyPSA docs: Custom Components | +| pandapower | `std_types` / user-defined columns | Custom columns on element DataFrames; `std_types` for equipment type libraries | pandapower docs: User-defined data | +| GridCal | Device properties / custom fields | GridCal devices support arbitrary property attachment via the property system | GridCal API: Device properties | +| PowerModels.jl | Dict extension keys | Network data dicts accept arbitrary keys beyond the standard set | PowerModels.jl docs: Network Data | +| PowerSimulations.jl | `ext` field on component types | PowerSystems.jl component types include an `ext::Dict{String,Any}` field for arbitrary metadata | PowerSystems.jl docs: Type hierarchy | +| MATPOWER | Custom `mpc` struct fields | MATPOWER case structs accept arbitrary fields beyond the standard set (bus, gen, branch, etc.) | MATPOWER docs: Case format | + +## LINE_AND_TRANSFORMER.csv + +**Domain:** Transmission +**Purpose:** Provides thermal ratings (up to 4 tiers: Rate A/B/C/D), emergency ratings, and operational parameters for transmission lines and transformers. These ratings define the MVA transfer limits used in security-constrained dispatch and congestion management. The 4-tier rating hierarchy supports normal, long-term emergency, short-term emergency, and extreme emergency operating conditions as defined by ISO operating procedures. +**Row count:** ~15,000 rows +**Join key:** FROM_BUS + TO_BUS + CKT +**Join target:** branch, transformer +**Cardinality:** N:1 (multiple rating tiers per element) +**Match rate:** 97.2% + +### Join Keys + +The composite key FROM_BUS + TO_BUS + CKT identifies each transmission element by its terminal buses and circuit identifier, directly corresponding to the PSS/E branch record identifiers (I, J, CKT) in the intermediate format branch and transformer tables. The D9 validation found a 97.2% match rate against the branch table, with the unmatched 2.8% primarily comprising transformer records that join to the transformer table instead. When validated against both branch and transformer tables together, the effective match rate exceeds 99%. + +**D9 reference:** See [Join Key Report](../intermediate/csv_join_keys/join_key_report.md), section "LINE_AND_TRANSFORMER.csv" + +### Fields + +| Field | Type | Semantic Description | Example | Join Key | +|-------|------|---------------------|---------|----------| +| FROM_BUS | integer | Source bus number of the transmission element, corresponding to PSS/E bus number I at the from-end of the branch or transformer. | 10523 | yes | +| TO_BUS | integer | Destination bus number of the transmission element, corresponding to PSS/E bus number J at the to-end of the branch or transformer. | 20847 | yes | +| CKT | string | Circuit identifier distinguishing parallel elements between the same bus pair. Matches the PSS/E CKT field. | 1 | yes | +| ELEMENT_TYPE | enum | Type of network element: LINE for transmission lines, TRANSFORMER for power transformers. Determines which intermediate format table the record joins to. | LINE | no | +| RATE_A | float | Normal (continuous) thermal rating in MVA. The maximum power transfer under normal operating conditions. Corresponds to PSS/E RATEA. | 785.0 | no | +| RATE_B | float | Long-term emergency thermal rating in MVA. Applicable during planned outage conditions or system restoration. Corresponds to PSS/E RATEB. | 890.0 | no | +| RATE_C | float | Short-term emergency thermal rating in MVA. Applicable during contingency conditions for limited duration. Corresponds to PSS/E RATEC. | 1050.0 | no | +| RATE_D | float | Extreme emergency thermal rating in MVA. An ISO-specific 4th rating tier not present in PSS/E, used for extreme contingency analysis. | 1200.0 | no | +| STATUS | enum | Operational status of the element: IN_SERVICE or OUT_OF_SERVICE. Elements with OUT_OF_SERVICE status are excluded from thermal limit enforcement. | IN_SERVICE | no | +| EFFECTIVE_DATE | date | Date from which these ratings become effective, in ISO 8601 format. Supports seasonal rating changes. | 2024-06-01 | no | + +### Representability + +| Field | PyPSA | pandapower | GridCal | PowerModels.jl | PowerSimulations.jl | MATPOWER | +|-------|-------|------------|---------|----------------|---------------------|----------| +| FROM_BUS | N (Line.bus0) | N (line.from_bus) | N (Line.bus_from) | N (branch["f_bus"]) | N (Arc.from) | N (mpc.branch col 1) | +| TO_BUS | N (Line.bus1) | N (line.to_bus) | N (Line.bus_to) | N (branch["t_bus"]) | N (Arc.to) | N (mpc.branch col 2) | +| CKT | E (custom attr) | E (custom column) | E (custom field) | E (dict key) | E (ext dict) | E (custom mpc field) | +| ELEMENT_TYPE | E (custom attr) | E (custom column) | E (custom field) | E (dict key) | E (ext dict) | E (custom mpc field) | +| RATE_A | N (Line.s_nom) | N (line.max_i_ka) | N (Line.rate) | N (branch["rate_a"]) | N (ThermalStandard.rating) | N (mpc.branch RATE_A) | +| RATE_B | E (custom attr) | E (custom column) | E (custom field) | N (branch["rate_b"]) | E (ext dict) | N (mpc.branch RATE_B) | +| RATE_C | E (custom attr) | E (custom column) | E (custom field) | N (branch["rate_c"]) | E (ext dict) | N (mpc.branch RATE_C) | +| RATE_D | E (custom attr) | E (custom column) | E (custom field) | E (dict key) | E (ext dict) | E (custom mpc field) | +| STATUS | N (Line.active) | N (line.in_service) | N (Line.active) | N (branch["br_status"]) | N (available) | N (mpc.branch BR_STATUS) | +| EFFECTIVE_DATE | E (custom attr) | E (custom column) | E (custom field) | E (dict key) | E (ext dict) | E (custom mpc field) | + +### Summary + +| Tool | Native (N) | Extension (E) | External (X) | N% | E% | X% | +|------|-----------|---------------|--------------|----|----|-----| +| PyPSA | 4 | 6 | 0 | 40% | 60% | 0% | +| pandapower | 4 | 6 | 0 | 40% | 60% | 0% | +| GridCal | 4 | 6 | 0 | 40% | 60% | 0% | +| PowerModels.jl | 6 | 4 | 0 | 60% | 40% | 0% | +| PowerSimulations.jl | 4 | 6 | 0 | 40% | 60% | 0% | +| MATPOWER | 6 | 4 | 0 | 60% | 40% | 0% | + +### Key Findings + +- RATE_D (4th thermal rating tier) is Extension-representable (E) across all 6 tools -- no tool has a native 4th rating tier. This is an ISO-specific concept not present in PSS/E or IEEE standards. +- MATPOWER and PowerModels.jl have the best native coverage (60%) because they natively support 3 thermal rating tiers (RATE_A/B/C) in the branch data structure, while other tools support only 1 native rating. +- The CKT (circuit identifier) field is Extension-representable in all tools despite being a fundamental PSS/E identifier, because tools use internal element indexing rather than PSS/E's bus-pair-circuit composite key. +- EFFECTIVE_DATE is universally Extension-representable -- no power flow tool has native temporal validity concepts for ratings. + +## TRADING_HUB.csv + +**Domain:** Market +**Purpose:** Defines trading hub compositions by mapping hub names to sets of buses with associated distribution factors. Trading hubs are market constructs used by ISOs for energy market settlement and congestion pricing -- they aggregate physical buses into commercial trading points. No power flow tool has a native trading hub concept because hubs are a market-layer abstraction that sits above the physical network model. +**Row count:** ~500 rows +**Join key:** BUS_NUMBER +**Join target:** bus +**Cardinality:** N:1 +**Match rate:** 99.3% + +### Join Keys + +The BUS_NUMBER column joins each hub-bus mapping record to the bus table via the PSS/E bus number (I). The D9 validation found a 99.3% match rate, with unmatched rows corresponding to retired or planned buses not present in the current network model snapshot. + +**D9 reference:** See [Join Key Report](../intermediate/csv_join_keys/join_key_report.md), section "TRADING_HUB.csv" + +### Fields + +| Field | Type | Semantic Description | Example | Join Key | +|-------|------|---------------------|---------|----------| +| HUB_NAME | string | Name of the trading hub as defined in the ISO market. Uses standardized naming conventions reflecting geographic regions and hub types. | SP15_GEN_HUB | no | +| BUS_NUMBER | integer | Bus number of a physical bus participating in this trading hub. Each bus may appear in multiple hubs. Corresponds to PSS/E bus number I. | 24510 | yes | +| DISTRIBUTION_FACTOR | float | Weight factor for this bus within the hub. Distribution factors within a hub sum to 1.0 and determine each bus's contribution to the hub's aggregated price or quantity. | 0.0234 | no | +| HUB_TYPE | enum | Classification of the hub: GEN for generation-weighted hubs, LOAD for load-weighted hubs, TRADING for pure trading points. | GEN | no | + +### Representability + +| Field | PyPSA | pandapower | GridCal | PowerModels.jl | PowerSimulations.jl | MATPOWER | +|-------|-------|------------|---------|----------------|---------------------|----------| +| HUB_NAME | E (custom attribute on n.buses) | X (no hub model) | X (no hub model) | X (no hub model) | X (no hub model) | X (no hub model) | +| BUS_NUMBER | N (Bus index) | N (bus index) | N (Bus index) | N (bus["bus_i"]) | N (ACBus.number) | N (mpc.bus BUS_I) | +| DISTRIBUTION_FACTOR | E (hub weights as custom bus attributes; aggregate via (df_weights * n.buses_t.marginal_price).sum(axis=1) post-solve) | X (no hub model) | X (no hub model) | X (no hub model) | X (no hub model) | X (no hub model) | +| HUB_TYPE | X (no hub model) | X (no hub model) | X (no hub model) | X (no hub model) | X (no hub model) | X (no hub model) | + +### Summary + +| Tool | Native (N) | Extension (E) | External (X) | N% | E% | X% | +|------|-----------|---------------|--------------|----|----|-----| +| PyPSA | 1 | 2 | 1 | 25% | 50% | 25% | +| pandapower | 1 | 0 | 3 | 25% | 0% | 75% | +| GridCal | 1 | 0 | 3 | 25% | 0% | 75% | +| PowerModels.jl | 1 | 0 | 3 | 25% | 0% | 75% | +| PowerSimulations.jl | 1 | 0 | 3 | 25% | 0% | 75% | +| MATPOWER | 1 | 0 | 3 | 25% | 0% | 75% | + +### Key Findings + +- Trading hub data is universally tool-external (X) across most tools for 3 of 4 fields (HUB_NAME, DISTRIBUTION_FACTOR, HUB_TYPE). No tool has a native trading hub concept because hubs are a market-layer abstraction. +- Only BUS_NUMBER is natively representable in all tools because it corresponds to the physical bus identifier. +- PyPSA can store HUB_NAME as a custom bus attribute and DISTRIBUTION_FACTOR as custom bus attributes, enabling post-OPF aggregate hub pricing via PTDF-weighted bus LMP averaging. HUB_TYPE has no extension path. +- All other tools have identical representability profiles (25% N, 75% X) because hub concepts are equally absent from their domain models. +- Trading hub data must be maintained in external data structures alongside the network model in all tools except PyPSA (which can partially accommodate hub definitions via custom attributes). + +> **v10 note (TRADING_HUB.csv):** HUB_NAME and hub distribution weight fields reclassified X→E +> for PyPSA. Extension: PTDF-weighted bus LMP averaging post-OPF. Aggregate hub prices are +> derivable but require custom post-processing code. + +## GEN_DISTRIBUTION_FACTOR.csv + +**Domain:** Generation +**Purpose:** Maps generators to trading hubs with percentage participation factors that determine each generator's contribution to hub-level generation allocation. These distribution factors are used in market settlement to allocate generator output across trading hubs. The relationship is many-to-many: a generator may participate in multiple hubs, and a hub contains contributions from multiple generators. +**Row count:** ~2,000 rows +**Join key:** GEN_BUS + GEN_ID +**Join target:** generator +**Cardinality:** N:1 +**Match rate:** 98.1% + +### Join Keys + +The composite key GEN_BUS + GEN_ID identifies each generator by its bus number and machine identifier, corresponding to the PSS/E generator record fields I and ID in the intermediate format generator table. The D9 validation found a 98.1% match rate, with unmatched rows corresponding to generators present in market systems but absent from the planning network model. + +**D9 reference:** See [Join Key Report](../intermediate/csv_join_keys/join_key_report.md), section "GEN_DISTRIBUTION_FACTOR.csv" + +### Fields + +| Field | Type | Semantic Description | Example | Join Key | +|-------|------|---------------------|---------|----------| +| GEN_BUS | integer | Bus number where the generator is connected. Corresponds to PSS/E generator bus number I. | 31205 | yes | +| GEN_ID | string | Machine identifier distinguishing multiple generators at the same bus. Corresponds to PSS/E machine ID. | 1 | yes | +| HUB_NAME | string | Name of the trading hub this generator participates in. References the HUB_NAME in TRADING_HUB.csv. | NP15_GEN_HUB | no | +| PARTICIPATION_FACTOR | float | Fractional participation of this generator in the specified hub. Values between 0.0 and 1.0. A generator's participation factors across all hubs sum to 1.0. | 0.156 | no | +| GEN_NAME | string | Human-readable name of the generator unit for cross-referencing with other market systems. | DIABLO_CANYON_1 | no | + +### Representability + +| Field | PyPSA | pandapower | GridCal | PowerModels.jl | PowerSimulations.jl | MATPOWER | +|-------|-------|------------|---------|----------------|---------------------|----------| +| GEN_BUS | N (Generator.bus) | N (gen.bus) | N (Generator.bus) | N (gen["gen_bus"]) | N (ThermalStandard.bus) | N (mpc.gen GEN_BUS) | +| GEN_ID | E (custom attr) | E (custom column) | E (custom field) | E (dict key) | E (ext dict) | E (custom mpc field) | +| HUB_NAME | X (no hub model) | X (no hub model) | X (no hub model) | X (no hub model) | X (no hub model) | X (no hub model) | +| PARTICIPATION_FACTOR | X (no dist factor attr) | X (no dist factor attr) | X (no dist factor attr) | X (no dist factor attr) | X (no dist factor attr) | X (no dist factor attr) | +| GEN_NAME | N (Generator.name) | N (gen.name) | N (Generator.name) | N (gen["name"]) | N (ThermalStandard.name) | E (custom mpc field) | + +### Summary + +| Tool | Native (N) | Extension (E) | External (X) | N% | E% | X% | +|------|-----------|---------------|--------------|----|----|-----| +| PyPSA | 2 | 1 | 2 | 40% | 20% | 40% | +| pandapower | 2 | 1 | 2 | 40% | 20% | 40% | +| GridCal | 2 | 1 | 2 | 40% | 20% | 40% | +| PowerModels.jl | 2 | 1 | 2 | 40% | 20% | 40% | +| PowerSimulations.jl | 2 | 1 | 2 | 40% | 20% | 40% | +| MATPOWER | 1 | 2 | 2 | 20% | 40% | 40% | + +### Key Findings + +- HUB_NAME and PARTICIPATION_FACTOR are universally tool-external (X) across all 6 tools. No tool has native generator distribution factor attributes because these are market settlement constructs. +- GEN_NAME is natively representable in 5 of 6 tools but only Extension-representable in MATPOWER, which stores generators in a numeric matrix without a native name field. +- All tools have identical or near-identical representability profiles, reflecting that generator distribution factors are a market-layer concept absent from all power flow tools. +- The 40% X rate means nearly half the fields in this CSV must be maintained externally regardless of tool choice. + +## CONTINGENCY.csv + +**Domain:** Transmission +**Purpose:** Defines contingency scenarios specifying which network elements are tripped for N-1 and N-2 reliability analysis. Each row describes a single element outage within a named contingency case, with the contingency name grouping multiple element outages into a scenario. Contingencies are fundamental to security-constrained economic dispatch (SCED) and transmission planning studies. +**Row count:** ~5,000 rows +**Join key:** ELEMENT_FROM_BUS + ELEMENT_TO_BUS + ELEMENT_CKT (for branch contingencies), ELEMENT_BUS + ELEMENT_ID (for generator contingencies) +**Join target:** branch, generator +**Cardinality:** M:N +**Match rate:** 94.8% + +### Join Keys + +Branch contingencies use the composite key ELEMENT_FROM_BUS + ELEMENT_TO_BUS + ELEMENT_CKT, which joins to the branch table via I, J, CKT. Generator contingencies use ELEMENT_BUS + ELEMENT_ID, joining to the generator table via I, ID. The file contains mixed element types (both branch and generator contingencies), so the applicable join depends on the ELEMENT_TYPE field. The D9 validation found a 94.8% aggregate match rate; the lower rate compared to other CSVs reflects contingency definitions referencing planned or hypothetical elements not in the base case model. Secondary join to the transformer table was also validated for transformer contingencies. + +**D9 reference:** See [Join Key Report](../intermediate/csv_join_keys/join_key_report.md), section "CONTINGENCY.csv" + +### Fields + +| Field | Type | Semantic Description | Example | Join Key | +|-------|------|---------------------|---------|----------| +| CONTINGENCY_NAME | string | Unique name identifying the contingency scenario. Multiple rows with the same name define a multi-element (N-2 or higher) contingency. | CTG_001_LINE_10001_20001 | no | +| ELEMENT_TYPE | enum | Type of element being tripped: BRANCH for transmission lines or transformers, GENERATOR for generating units. Determines which join key set and target table apply. | BRANCH | no | +| ELEMENT_FROM_BUS | integer | From-bus number of the contingency element (for branch/transformer contingencies). Null for generator contingencies. Corresponds to PSS/E bus number I. | 10001 | yes | +| ELEMENT_TO_BUS | integer | To-bus number of the contingency element (for branch/transformer contingencies). Null for generator contingencies. Corresponds to PSS/E bus number J. | 20001 | yes | +| ELEMENT_CKT | string | Circuit identifier of the contingency element (for branch/transformer contingencies). Null for generator contingencies. Corresponds to PSS/E CKT. | 1 | yes | +| ELEMENT_BUS | integer | Bus number of the contingency generator (for generator contingencies). Null for branch contingencies. Corresponds to PSS/E generator bus I. | 31205 | yes | + +### Representability + +| Field | PyPSA | pandapower | GridCal | PowerModels.jl | PowerSimulations.jl | MATPOWER | +|-------|-------|------------|---------|----------------|---------------------|----------| +| CONTINGENCY_NAME | E (extra_functionality + custom DataFrame n.contingencies) | X (no contingency model) | N (ContingencyGroup.name) | X (no contingency model) | N (Contingency.name) | X (no contingency model) | +| ELEMENT_TYPE | E (extra_functionality + custom attribute) | X (no contingency model) | N (Contingency.device_type) | X (no contingency model) | N (Contingency element type) | X (no contingency model) | +| ELEMENT_FROM_BUS | N (Line.bus0) | N (line.from_bus) | N (Line.bus_from) | N (branch["f_bus"]) | N (Arc.from) | N (mpc.branch col 1) | +| ELEMENT_TO_BUS | N (Line.bus1) | N (line.to_bus) | N (Line.bus_to) | N (branch["t_bus"]) | N (Arc.to) | N (mpc.branch col 2) | +| ELEMENT_CKT | E (custom attr on contingency DataFrame) | E (custom column) | E (custom field) | E (dict key) | E (ext dict) | E (custom mpc field) | +| ELEMENT_BUS | N (Generator.bus) | N (gen.bus) | N (Generator.bus) | N (gen["gen_bus"]) | N (ThermalStandard.bus) | N (mpc.gen GEN_BUS) | + +### Summary + +| Tool | Native (N) | Extension (E) | External (X) | N% | E% | X% | +|------|-----------|---------------|--------------|----|----|-----| +| PyPSA | 3 | 3 | 0 | 50% | 50% | 0% | +| pandapower | 3 | 1 | 2 | 50% | 17% | 33% | +| GridCal | 5 | 1 | 0 | 83% | 17% | 0% | +| PowerModels.jl | 3 | 1 | 2 | 50% | 17% | 33% | +| PowerSimulations.jl | 5 | 1 | 0 | 83% | 17% | 0% | +| MATPOWER | 3 | 1 | 2 | 50% | 17% | 33% | + +### Key Findings + +- GridCal and PowerSimulations.jl have the best contingency data coverage (83% N) because both have native contingency definition objects (`ContingencyGroup`/`Contingency` and `Contingency` type respectively). +- CONTINGENCY_NAME and ELEMENT_TYPE are tool-external (X) in pandapower, PowerModels.jl, and MATPOWER -- these tools have no native contingency definition model. Contingency analysis in these tools is typically handled by external scripts that modify network state. +- PyPSA can represent CONTINGENCY_NAME, ELEMENT_TYPE, and ELEMENT_CKT via its `extra_functionality` callback mechanism with a custom `n.contingencies` DataFrame; N-1 constraints are enforced via BODF matrix as additional `lp.add_constraints()` calls. This is Extension-representable (E) but requires 50–100 lines of custom code. +- The network element identifier fields (FROM_BUS, TO_BUS, ELEMENT_BUS) are natively representable in all tools because they correspond to existing bus identifiers. +- ELEMENT_CKT is universally Extension-representable, consistent with the pattern seen in LINE_AND_TRANSFORMER.csv. + +> **v10 note (CONTINGENCY.csv):** CONTINGENCY_NAME, ELEMENT_TYPE, and ELEMENT_CKT reclassified X→E +> for PyPSA. Extension mechanism: `extra_functionality` callback + BODF matrix for N-1 constraint +> enforcement. This is complex (requires 50–100 lines of custom code) but is a documented, +> supported extension pattern. Classify as E with "complex" notation in market fidelity summary. + +## INTERFACE.csv + +**Domain:** Transmission +**Purpose:** Defines named transmission interfaces with normal and emergency flow limits. Interfaces are groupings of monitored transmission paths whose aggregate flow is constrained for reliability purposes. ISOs use interfaces (also called paths or flowgates) to manage power transfers across critical transmission corridors. Interface definitions do not directly reference individual network elements -- the element composition is specified in INTERFACE_ELEMENT.csv. +**Row count:** ~100 rows +**Join key:** INTERFACE_ID +**Join target:** (indirect via INTERFACE_ELEMENT.csv) +**Cardinality:** 1:N +**Match rate:** N/A + +### Join Keys + +INTERFACE.csv does not join directly to any intermediate format network table. Instead, INTERFACE_ID serves as the linking key to INTERFACE_ELEMENT.csv, which in turn joins to the branch table via FROM_BUS + TO_BUS + CKT. The relationship is 1:N -- each interface contains multiple branch elements. D9 validated the INTERFACE_ID link between INTERFACE.csv and INTERFACE_ELEMENT.csv with 100% match rate (every interface has at least one element). + +**D9 reference:** See [Join Key Report](../intermediate/csv_join_keys/join_key_report.md), section "INTERFACE.csv" + +### Fields + +| Field | Type | Semantic Description | Example | Join Key | +|-------|------|---------------------|---------|----------| +| INTERFACE_ID | integer | Unique numeric identifier for the interface. Referenced by INTERFACE_ELEMENT.csv to associate branch elements with this interface. | 15 | yes | +| INTERFACE_NAME | string | Human-readable name of the interface, typically referencing a transmission corridor or internal path designation. Some names are publicly known (e.g., Path 15, Path 26). | Path_15 | no | +| NORMAL_LIMIT_MW | float | Normal (continuous) flow limit for the interface in MW. The maximum aggregate power transfer across all member elements under normal operating conditions. | 2500.0 | no | +| EMERGENCY_LIMIT_MW | float | Emergency flow limit for the interface in MW. The maximum aggregate power transfer permitted during contingency conditions for limited duration. | 3000.0 | no | +| DIRECTION | enum | Flow direction convention: FORWARD or REVERSE. Defines the positive flow direction for limit enforcement. | FORWARD | no | + +### Representability + +| Field | PyPSA | pandapower | GridCal | PowerModels.jl | PowerSimulations.jl | MATPOWER | +|-------|-------|------------|---------|----------------|---------------------|----------| +| INTERFACE_ID | E (custom n.interfaces DataFrame) | X (no interface model) | X (no interface model) | X (no interface model) | N (TransmissionInterface.name) | N (mpc.if col 1) | +| INTERFACE_NAME | E (custom attribute on interface DataFrame) | X (no interface model) | X (no interface model) | X (no interface model) | N (TransmissionInterface.name) | E (custom mpc field) | +| NORMAL_LIMIT_MW | E (PTDF constraint via extra_functionality + n.add_constraints()) | X (no interface model) | X (no interface model) | X (no interface model) | N (TransmissionInterface limits) | N (mpc.iflim) | +| EMERGENCY_LIMIT_MW | E (PTDF constraint, contingency-conditional via extra_functionality) | X (no interface model) | X (no interface model) | X (no interface model) | E (ext dict) | E (custom mpc field) | +| DIRECTION | E (sign convention in PTDF weighting, custom attribute) | X (no interface model) | X (no interface model) | X (no interface model) | E (ext dict) | E (custom mpc field) | + +### Summary + +| Tool | Native (N) | Extension (E) | External (X) | N% | E% | X% | +|------|-----------|---------------|--------------|----|----|-----| +| PyPSA | 0 | 5 | 0 | 0% | 100% | 0% | +| pandapower | 0 | 0 | 5 | 0% | 0% | 100% | +| GridCal | 0 | 0 | 5 | 0% | 0% | 100% | +| PowerModels.jl | 0 | 0 | 5 | 0% | 0% | 100% | +| PowerSimulations.jl | 3 | 2 | 0 | 60% | 40% | 0% | +| MATPOWER | 2 | 2 | 1 | 40% | 40% | 20% | + +### Key Findings + +- Interface data is 100% tool-external (X) in pandapower, GridCal, and PowerModels.jl -- none of these tools have any native interface/flowgate concept. +- PyPSA can represent all 5 INTERFACE.csv fields via Extension mechanisms: a custom `n.interfaces` DataFrame stores the interface definition, and interface flow limits are enforced via PTDF matrix + `extra_functionality` constraints using `n.add_constraints()`. This is complex but a documented supported pattern. +- PowerSimulations.jl (via PowerSystems.jl `TransmissionInterface`) has the best native coverage (60% N), making it the only tool with a first-class interface data model. +- MATPOWER supports interface definitions via `mpc.if` and `mpc.iflim` structures (40% N), providing basic interface flow limit enforcement in OPF. +- INTERFACE_ID is classified as X for pandapower, GridCal, and PowerModels.jl rather than E, because the interface concept itself (a named group of branches with aggregate flow limits) has no structural analog -- storing just the ID without the concept is meaningless. + +> **v10 note (INTERFACE.csv):** All 5 INTERFACE.csv fields reclassified X→E for PyPSA. +> Extension mechanism: PTDF matrix + `extra_functionality` constraints. Classify as E with +> "complex" in market fidelity summary. + +## INTERFACE_ELEMENT.csv + +**Domain:** Transmission +**Purpose:** Specifies the individual branch elements that comprise each transmission interface, along with direction coefficients and weighting factors. Each row associates one branch with one interface, establishing the physical composition of the interface defined in INTERFACE.csv. The direction coefficient (+1 or -1) indicates whether positive flow on the branch contributes positively or negatively to the interface flow calculation. +**Row count:** ~500 rows +**Join key:** FROM_BUS + TO_BUS + CKT +**Join target:** branch +**Cardinality:** N:1 +**Match rate:** 96.5% + +### Join Keys + +The composite key FROM_BUS + TO_BUS + CKT joins each interface element record to the branch table via PSS/E identifiers I, J, CKT. The INTERFACE_ID column links back to INTERFACE.csv. The D9 validation found a 96.5% match rate against the branch table, with unmatched rows corresponding to elements defined for contingency interfaces that reference out-of-service or planned branches. + +**D9 reference:** See [Join Key Report](../intermediate/csv_join_keys/join_key_report.md), section "INTERFACE_ELEMENT.csv" + +### Fields + +| Field | Type | Semantic Description | Example | Join Key | +|-------|------|---------------------|---------|----------| +| INTERFACE_ID | integer | Numeric identifier of the parent interface. References INTERFACE_ID in INTERFACE.csv. | 15 | no | +| FROM_BUS | integer | From-bus number of the branch element comprising this interface. Corresponds to PSS/E bus number I. | 10523 | yes | +| TO_BUS | integer | To-bus number of the branch element comprising this interface. Corresponds to PSS/E bus number J. | 20847 | yes | +| CKT | string | Circuit identifier of the branch element. Corresponds to PSS/E CKT field. | 1 | yes | +| DIRECTION_COEFF | float | Direction coefficient for the branch's contribution to interface flow: +1.0 means positive branch flow adds to interface flow, -1.0 means it subtracts. | 1.0 | no | +| WEIGHT_FACTOR | float | Weighting factor for the branch's contribution to the aggregate interface flow calculation. Typically 1.0 for full contribution. | 1.0 | no | + +### Representability + +| Field | PyPSA | pandapower | GridCal | PowerModels.jl | PowerSimulations.jl | MATPOWER | +|-------|-------|------------|---------|----------------|---------------------|----------| +| INTERFACE_ID | X (no interface model) | X (no interface model) | X (no interface model) | X (no interface model) | N (TransmissionInterface ref) | N (mpc.if col 1) | +| FROM_BUS | N (Line.bus0) | N (line.from_bus) | N (Line.bus_from) | N (branch["f_bus"]) | N (Arc.from) | N (mpc.branch col 1) | +| TO_BUS | N (Line.bus1) | N (line.to_bus) | N (Line.bus_to) | N (branch["t_bus"]) | N (Arc.to) | N (mpc.branch col 2) | +| CKT | E (custom attr) | E (custom column) | E (custom field) | E (dict key) | E (ext dict) | E (custom mpc field) | +| DIRECTION_COEFF | X (no interface model) | X (no interface model) | X (no interface model) | X (no interface model) | N (interface element dir) | N (mpc.if direction) | +| WEIGHT_FACTOR | X (no interface model) | X (no interface model) | X (no interface model) | X (no interface model) | E (ext dict) | E (custom mpc field) | + +### Summary + +| Tool | Native (N) | Extension (E) | External (X) | N% | E% | X% | +|------|-----------|---------------|--------------|----|----|-----| +| PyPSA | 2 | 1 | 3 | 33% | 17% | 50% | +| pandapower | 2 | 1 | 3 | 33% | 17% | 50% | +| GridCal | 2 | 1 | 3 | 33% | 17% | 50% | +| PowerModels.jl | 2 | 1 | 3 | 33% | 17% | 50% | +| PowerSimulations.jl | 4 | 2 | 0 | 67% | 33% | 0% | +| MATPOWER | 4 | 2 | 0 | 67% | 33% | 0% | + +### Key Findings + +- INTERFACE_ID and DIRECTION_COEFF are tool-external (X) in PyPSA, pandapower, GridCal, and PowerModels.jl, consistent with the INTERFACE.csv findings -- these tools have no interface concept. +- PowerSimulations.jl and MATPOWER both achieve 67% native coverage, the highest among all tools, reflecting their native interface element support. +- The branch identifier fields (FROM_BUS, TO_BUS) are universally natively representable as bus references, while CKT remains universally Extension-representable. +- WEIGHT_FACTOR is Extension-representable even in PowerSimulations.jl and MATPOWER, suggesting that interface weighting is a less common native concept than interface membership. + +## OUTAGE.csv + +**Domain:** Outage +**Purpose:** Records scheduled and forced outage definitions specifying which network elements are out of service, with effective date ranges and outage classification types. Outage data is used for transmission planning studies, outage coordination, and seasonal assessment studies. Each row defines a single element outage with its temporal validity period and outage action (trip, derate, etc.). +**Row count:** ~3,000 rows +**Join key:** ELEMENT_FROM_BUS + ELEMENT_TO_BUS + ELEMENT_CKT (for branch outages), ELEMENT_BUS + ELEMENT_ID (for generator outages) +**Join target:** branch, generator +**Cardinality:** M:N +**Match rate:** 95.6% + +### Join Keys + +Like CONTINGENCY.csv, OUTAGE.csv contains mixed element types. Branch outages use the composite key ELEMENT_FROM_BUS + ELEMENT_TO_BUS + ELEMENT_CKT, joining to the branch table via I, J, CKT. Generator outages use ELEMENT_BUS + ELEMENT_ID, joining to the generator table via I, ID. The D9 validation found a 95.6% aggregate match rate. Unmatched rows include outages for elements planned for commissioning or recently decommissioned. + +**D9 reference:** See [Join Key Report](../intermediate/csv_join_keys/join_key_report.md), section "OUTAGE.csv" + +### Fields + +| Field | Type | Semantic Description | Example | Join Key | +|-------|------|---------------------|---------|----------| +| ELEMENT_TYPE | enum | Type of element being outaged: BRANCH for transmission lines or transformers, GENERATOR for generating units. Determines which join key set applies. | BRANCH | no | +| ELEMENT_FROM_BUS | integer | From-bus number of the outaged element (for branch outages). Null for generator outages. Corresponds to PSS/E bus number I. | 10523 | yes | +| ELEMENT_TO_BUS | integer | To-bus number of the outaged element (for branch outages). Null for generator outages. Corresponds to PSS/E bus number J. | 20847 | yes | +| ELEMENT_CKT | string | Circuit identifier of the outaged element (for branch outages). Null for generator outages. Corresponds to PSS/E CKT. | 1 | yes | +| ELEMENT_BUS | integer | Bus number of the outaged generator (for generator outages). Null for branch outages. Corresponds to PSS/E generator bus I. | 31205 | yes | +| OUTAGE_START | datetime | Start date and time of the outage in ISO 8601 format. Defines when the element becomes unavailable. | 2024-03-15T06:00:00 | no | +| OUTAGE_END | datetime | End date and time of the outage in ISO 8601 format. Defines when the element returns to service. | 2024-04-20T18:00:00 | no | +| OUTAGE_TYPE | enum | Classification of the outage: PLANNED for scheduled maintenance, FORCED for unplanned outages, DERATE for partial capacity reduction. Determines how the outage affects element availability. | PLANNED | no | + +### Representability + +| Field | PyPSA | pandapower | GridCal | PowerModels.jl | PowerSimulations.jl | MATPOWER | +|-------|-------|------------|---------|----------------|---------------------|----------| +| ELEMENT_TYPE | X (no outage model) | X (no outage model) | X (no outage model) | X (no outage model) | X (no outage schedule) | X (no outage model) | +| ELEMENT_FROM_BUS | N (Line.bus0) | N (line.from_bus) | N (Line.bus_from) | N (branch["f_bus"]) | N (Arc.from) | N (mpc.branch col 1) | +| ELEMENT_TO_BUS | N (Line.bus1) | N (line.to_bus) | N (Line.bus_to) | N (branch["t_bus"]) | N (Arc.to) | N (mpc.branch col 2) | +| ELEMENT_CKT | E (custom attr) | E (custom column) | E (custom field) | E (dict key) | E (ext dict) | E (custom mpc field) | +| ELEMENT_BUS | N (Generator.bus) | N (gen.bus) | N (Generator.bus) | N (gen["gen_bus"]) | N (ThermalStandard.bus) | N (mpc.gen GEN_BUS) | +| OUTAGE_START | X (no outage model) | X (no outage model) | X (no outage model) | X (no outage model) | X (no outage schedule) | X (no outage model) | +| OUTAGE_END | X (no outage model) | X (no outage model) | X (no outage model) | X (no outage model) | X (no outage schedule) | X (no outage model) | +| OUTAGE_TYPE | X (no outage model) | X (no outage model) | X (no outage model) | X (no outage model) | X (no outage schedule) | X (no outage model) | + +### Summary + +| Tool | Native (N) | Extension (E) | External (X) | N% | E% | X% | +|------|-----------|---------------|--------------|----|----|-----| +| PyPSA | 3 | 1 | 4 | 38% | 12% | 50% | +| pandapower | 3 | 1 | 4 | 38% | 12% | 50% | +| GridCal | 3 | 1 | 4 | 38% | 12% | 50% | +| PowerModels.jl | 3 | 1 | 4 | 38% | 12% | 50% | +| PowerSimulations.jl | 3 | 1 | 4 | 38% | 12% | 50% | +| MATPOWER | 3 | 1 | 4 | 38% | 12% | 50% | + +### Key Findings + +- Outage schedule data (OUTAGE_START, OUTAGE_END, OUTAGE_TYPE, ELEMENT_TYPE) is universally tool-external (X) across all 6 tools. No tool has a native outage schedule data model with temporal validity. +- All tools have identical representability profiles (38% N, 12% E, 50% X), making this the CSV with the most uniform tool coverage. +- While PowerSimulations.jl has `available` and `must_run` attributes on component types, these represent point-in-time status rather than scheduled outage periods, so they do not satisfy the outage schedule concept. +- The 50% X rate reflects the fundamental gap between power flow tools (which model a single operating point) and outage management (which requires temporal scheduling). Outage application requires external scripting to modify network state across time periods. + +## Cross-CSV Summary + +| CSV | PyPSA N% | pandapower N% | GridCal N% | PowerModels.jl N% | PowerSimulations.jl N% | MATPOWER N% | +|-----|----------|---------------|------------|-------------------|------------------------|-------------| +| LINE_AND_TRANSFORMER.csv | 40% | 40% | 40% | 60% | 40% | 60% | +| TRADING_HUB.csv | 25% | 25% | 25% | 25% | 25% | 25% | +| GEN_DISTRIBUTION_FACTOR.csv | 40% | 40% | 40% | 40% | 40% | 20% | +| CONTINGENCY.csv | 50% | 50% | 83% | 50% | 83% | 50% | +| INTERFACE.csv | 0% | 0% | 0% | 0% | 60% | 40% | +| INTERFACE_ELEMENT.csv | 33% | 33% | 33% | 33% | 67% | 67% | +| OUTAGE.csv | 38% | 38% | 38% | 38% | 38% | 38% | + +## Cross-References + +- Phase 1 D9 join-key mapping: `../intermediate/csv_join_keys/join_key_report.md` +- Phase 2 D1 intermediate schema reference: `intermediate-schema.md` +- Phase 2 D2 record-type mapping guide: `mapping-guide.md` +- Phase 2 D5 field criticality matrix: `field-criticality-matrix.md` +- Phase 4 D2 representability summary: `supplemental-csv-representability.md` diff --git a/data/fnm/docs/three-winding-transformers.md b/data/fnm/docs/three-winding-transformers.md new file mode 100644 index 00000000..533dc271 --- /dev/null +++ b/data/fnm/docs/three-winding-transformers.md @@ -0,0 +1,661 @@ +# 3-Winding Transformer Reference + +## Purpose + +This document provides the authoritative reference for PSS/E v31 3-winding transformer records -- the most complex record type in the PSS/E data format. A single 3-winding transformer record spans 5 data lines and contains 83 fields, more than any other PSS/E record type by a factor of three. The document covers record structure, field semantics, star-bus equivalent topology, winding parameters, intermediate format representation, and tool-specific handling for all six evaluated power-system modeling tools. It complements the intermediate format schema reference (`intermediate-schema.md`, PRD 01) by providing topology and parametric detail specific to 3-winding transformers, references the per-unit convention reference (`per-unit-conventions.md`, PRD 03) for base conversion formulas, and extends the record-type mapping guide (`mapping-guide.md`, PRD 02) with detailed parametric treatment beyond the record-type-level summary. + +## Audience + +This document is written for evaluate-tool agents assessing FNM ingestion fidelity for 3-winding transformer records across six power-system modeling tools. + +## PSS/E v31 Record Structure + +### Overview + +A 3-winding transformer record in PSS/E v31 is distinguished from a 2-winding transformer by having a nonzero third winding bus number (K != 0). The record spans 5 data lines and contains a total of 83 fields. The data lines are organized as follows: + +- **Data Line 1** (21 fields): Bus identifiers, control codes, magnetizing admittance, status, ownership, and vector group +- **Data Line 2** (11 fields): Pairwise winding impedances (R and X for each winding pair) and star bus initial voltage +- **Data Line 3** (17 fields): Winding 1 (primary/HV) parameters -- tap ratio, ratings, tap changer control +- **Data Line 4** (17 fields): Winding 2 (secondary/MV) parameters -- tap ratio, ratings, tap changer control +- **Data Line 5** (17 fields): Winding 3 (tertiary/LV) parameters -- tap ratio, ratings, tap changer control + +The 2-winding transformer (K=0) uses only 4 data lines and approximately 54 fields. This document covers only the 3-winding case. + +### Data Line 1 -- Bus Identifiers, Impedances, and Admittance + +Data Line 1 contains 21 fields that identify the three winding buses, specify control codes for impedance and tap interpretation, define magnetizing admittance, and provide status and ownership information. + +| Field | Position | Type | Unit | Description | +|-------|----------|------|------|-------------| +| `I` | 1 | int | -- | Winding 1 (primary) bus number | +| `J` | 2 | int | -- | Winding 2 (secondary) bus number | +| `K` | 3 | int | -- | Winding 3 (tertiary) bus number; nonzero distinguishes 3-winding from 2-winding | +| `CKT` | 4 | str | -- | Circuit identifier (up to 2 characters) | +| `CW` | 5 | int | -- | Winding data I/O code: 1=pu of bus BASKV, 2=kV, 3=pu of NOMV | +| `CZ` | 6 | int | -- | Impedance data I/O code: 1=pu on winding base, 2=pu on system base, 3=losses in W | +| `CM` | 7 | int | -- | Magnetizing admittance I/O code: 1=pu on system base, 2=exciting current %/losses W | +| `MAG1` | 8 | float | CM-dependent | Magnetizing admittance component 1 (G if CM=1, exciting current % if CM=2) | +| `MAG2` | 9 | float | CM-dependent | Magnetizing admittance component 2 (B if CM=1, core losses in watts if CM=2) | +| `NMETR` | 10 | int | -- | Nonmetered end code: 1, 2, or 3 indicating which winding is nonmetered | +| `NAME` | 11 | str | -- | Transformer name (up to 12 characters) | +| `STAT` | 12 | int | -- | Status: 1=in-service (all windings), 2=winding 2 out, 3=winding 3 out, 4=winding 2&3 out, 0=all out | +| `O1` | 13 | int | -- | Owner 1 number | +| `F1` | 14 | float | -- | Owner 1 fraction (0.0 to 1.0) | +| `O2` | 15 | int | -- | Owner 2 number | +| `F2` | 16 | float | -- | Owner 2 fraction | +| `O3` | 17 | int | -- | Owner 3 number | +| `F3` | 18 | float | -- | Owner 3 fraction | +| `O4` | 19 | int | -- | Owner 4 number | +| `F4` | 20 | float | -- | Owner 4 fraction | +| `VECGRP` | 21 | str | -- | Vector group identifier (v31 addition, e.g., "YNyn0") | + +### Data Line 2 -- Winding 1-2 and 2-3 and 3-1 Impedances + +Data Line 2 contains 11 fields specifying the pairwise impedances between each pair of windings and the star bus initial voltage. + +| Field | Position | Type | Unit | Description | +|-------|----------|------|------|-------------| +| `R1-2` | 1 | float | CZ-dependent | Resistance between windings 1 and 2 | +| `X1-2` | 2 | float | CZ-dependent | Reactance between windings 1 and 2 | +| `SBASE1-2` | 3 | float | MVA | MVA base for winding 1-2 impedance (used when CZ=1) | +| `R2-3` | 4 | float | CZ-dependent | Resistance between windings 2 and 3 | +| `X2-3` | 5 | float | CZ-dependent | Reactance between windings 2 and 3 | +| `SBASE2-3` | 6 | float | MVA | MVA base for winding 2-3 impedance (used when CZ=1) | +| `R3-1` | 7 | float | CZ-dependent | Resistance between windings 3 and 1 | +| `X3-1` | 8 | float | CZ-dependent | Reactance between windings 3 and 1 | +| `SBASE3-1` | 9 | float | MVA | MVA base for winding 3-1 impedance (used when CZ=1) | +| `VMSTAR` | 10 | float | pu | Star bus voltage magnitude initial value | +| `ANSTAR` | 11 | float | degrees | Star bus voltage angle initial value | + +### Data Line 3 -- Winding 1 Parameters + +Data Line 3 contains 17 fields specifying winding 1 (primary/HV) tap ratio, ratings, and tap changer control parameters. + +| Field | Position | Type | Unit | Description | +|-------|----------|------|------|-------------| +| `WINDV1` | 1 | float | CW-dependent | Winding 1 tap ratio or voltage | +| `NOMV1` | 2 | float | kV | Winding 1 nominal voltage | +| `ANG1` | 3 | float | degrees | Winding 1 phase shift angle | +| `RATA1` | 4 | float | MVA | Winding 1 rate A (normal) MVA rating | +| `RATB1` | 5 | float | MVA | Winding 1 rate B (emergency) MVA rating | +| `RATC1` | 6 | float | MVA | Winding 1 rate C (short-term) MVA rating | +| `COD1` | 7 | int | -- | Winding 1 tap changer control mode code | +| `CONT1` | 8 | int | -- | Winding 1 controlled bus number | +| `RMA1` | 9 | float | CW-dependent | Winding 1 maximum tap ratio or angle | +| `RMI1` | 10 | float | CW-dependent | Winding 1 minimum tap ratio or angle | +| `VMA1` | 11 | float | pu or MVA | Winding 1 maximum voltage or flow limit | +| `VMI1` | 12 | float | pu or MVA | Winding 1 minimum voltage or flow limit | +| `NTP1` | 13 | int | -- | Winding 1 number of tap positions | +| `TAB1` | 14 | int | -- | Winding 1 impedance correction table number | +| `CR1` | 15 | float | pu | Winding 1 load drop compensation resistance | +| `CX1` | 16 | float | pu | Winding 1 load drop compensation reactance | +| `CNXA1` | 17 | float | degrees | Winding 1 connection angle for wye-delta transformers | + +### Data Line 4 -- Winding 2 Parameters + +Data Line 4 contains 17 fields specifying winding 2 (secondary/MV) tap ratio, ratings, and tap changer control parameters. + +| Field | Position | Type | Unit | Description | +|-------|----------|------|------|-------------| +| `WINDV2` | 1 | float | CW-dependent | Winding 2 tap ratio or voltage | +| `NOMV2` | 2 | float | kV | Winding 2 nominal voltage | +| `ANG2` | 3 | float | degrees | Winding 2 phase shift angle | +| `RATA2` | 4 | float | MVA | Winding 2 rate A (normal) MVA rating | +| `RATB2` | 5 | float | MVA | Winding 2 rate B (emergency) MVA rating | +| `RATC2` | 6 | float | MVA | Winding 2 rate C (short-term) MVA rating | +| `COD2` | 7 | int | -- | Winding 2 tap changer control mode code | +| `CONT2` | 8 | int | -- | Winding 2 controlled bus number | +| `RMA2` | 9 | float | CW-dependent | Winding 2 maximum tap ratio or angle | +| `RMI2` | 10 | float | CW-dependent | Winding 2 minimum tap ratio or angle | +| `VMA2` | 11 | float | pu or MVA | Winding 2 maximum voltage or flow limit | +| `VMI2` | 12 | float | pu or MVA | Winding 2 minimum voltage or flow limit | +| `NTP2` | 13 | int | -- | Winding 2 number of tap positions | +| `TAB2` | 14 | int | -- | Winding 2 impedance correction table number | +| `CR2` | 15 | float | pu | Winding 2 load drop compensation resistance | +| `CX2` | 16 | float | pu | Winding 2 load drop compensation reactance | +| `CNXA2` | 17 | float | degrees | Winding 2 connection angle for wye-delta transformers | + +### Data Line 5 -- Winding 3 Parameters + +Data Line 5 contains 17 fields specifying winding 3 (tertiary/LV) tap ratio, ratings, and tap changer control parameters. + +| Field | Position | Type | Unit | Description | +|-------|----------|------|------|-------------| +| `WINDV3` | 1 | float | CW-dependent | Winding 3 tap ratio or voltage | +| `NOMV3` | 2 | float | kV | Winding 3 nominal voltage | +| `ANG3` | 3 | float | degrees | Winding 3 phase shift angle | +| `RATA3` | 4 | float | MVA | Winding 3 rate A (normal) MVA rating | +| `RATB3` | 5 | float | MVA | Winding 3 rate B (emergency) MVA rating | +| `RATC3` | 6 | float | MVA | Winding 3 rate C (short-term) MVA rating | +| `COD3` | 7 | int | -- | Winding 3 tap changer control mode code | +| `CONT3` | 8 | int | -- | Winding 3 controlled bus number | +| `RMA3` | 9 | float | CW-dependent | Winding 3 maximum tap ratio or angle | +| `RMI3` | 10 | float | CW-dependent | Winding 3 minimum tap ratio or angle | +| `VMA3` | 11 | float | pu or MVA | Winding 3 maximum voltage or flow limit | +| `VMI3` | 12 | float | pu or MVA | Winding 3 minimum voltage or flow limit | +| `NTP3` | 13 | int | -- | Winding 3 number of tap positions | +| `TAB3` | 14 | int | -- | Winding 3 impedance correction table number | +| `CR3` | 15 | float | pu | Winding 3 load drop compensation resistance | +| `CX3` | 16 | float | pu | Winding 3 load drop compensation reactance | +| `CNXA3` | 17 | float | degrees | Winding 3 connection angle for wye-delta transformers | + +### Field Count Summary + +| Data Line | Field Count | Purpose | +|-----------|-------------|---------| +| 1 | 21 | Bus identifiers, control codes, admittance, status, ownership | +| 2 | 11 | Pairwise winding impedances and star bus voltage | +| 3 | 17 | Winding 1 (primary/HV) parameters | +| 4 | 17 | Winding 2 (secondary/MV) parameters | +| 5 | 17 | Winding 3 (tertiary/LV) parameters | +| **Total** | **83 fields** | All 3-winding transformer parameters | + +## Star-Bus Equivalent Topology + +### Concept + +A 3-winding transformer with winding buses I, J, and K is electrically equivalent to three 2-winding transformers connected at a synthetic star bus S. The star bus is a zero-injection bus (no generation or load) located at the electrical center of the transformer. Each of the three equivalent 2-winding transformers connects one physical winding bus to the star bus, carrying its respective winding impedance, tap ratio, and phase angle. This decomposition is essential for tools that lack a native 3-winding transformer object and is the standard computational approach used during power flow solution even in tools that preserve the 3-winding parameterization in their data model. + +### Textual Topology Diagram + +``` + Bus I (Winding 1 / Primary / HV) + | + | Z1, WINDV1, ANG1 + | + Star Bus S (synthetic, VMSTAR/ANSTAR) + / \ + / \ + / \ + / \ + Z2, / \ Z3, + WINDV2/ \WINDV3 + ANG2 / \ANG3 + / \ + Bus J Bus K + (Winding 2 / (Winding 3 / + Secondary / Tertiary / + MV) LV) + + [MAG1, MAG2 placed on winding 1 branch (primary side)] +``` + +The diagram shows: +- **Bus I** (winding 1, primary, high voltage): connected to star bus via impedance Z1 +- **Bus J** (winding 2, secondary, medium voltage): connected to star bus via impedance Z2 +- **Bus K** (winding 3, tertiary, low voltage): connected to star bus via impedance Z3 +- **Star Bus S** (synthetic, zero-injection): voltage initialized to `VMSTAR`/`ANSTAR` +- Three equivalent branches with per-winding impedances Z1, Z2, Z3, tap ratios WINDV1/WINDV2/WINDV3, and phase angles ANG1/ANG2/ANG3 +- Magnetizing admittance (`MAG1`, `MAG2`) placed on the winding 1 (primary) branch + +### Pairwise to Star-Leg Impedance Conversion + +PSS/E stores pairwise impedances (Z1-2, Z2-3, Z3-1) measured between two windings with the third open-circuited. These must be converted to per-winding star-leg impedances (Z1, Z2, Z3) for the star-bus equivalent: + +**All pairwise impedances must be on a common per-unit base before applying these formulas.** If `SBASE1-2`, `SBASE2-3`, and `SBASE3-1` differ, convert each to the system base first (see [Per-Unit Convention Reference](per-unit-conventions.md#three-winding-transformer-per-unit-bases)). + +``` +Z_1 = (Z_12 + Z_31 - Z_23) / 2 +Z_2 = (Z_12 + Z_23 - Z_31) / 2 +Z_3 = (Z_23 + Z_31 - Z_12) / 2 +``` + +**Inverse (star to pairwise):** + +``` +Z_12 = Z_1 + Z_2 +Z_23 = Z_2 + Z_3 +Z_31 = Z_3 + Z_1 +``` + +**Note on negative impedance:** In autotransformers, it is common for one of the star-leg impedances (typically Z3, the tertiary) to be negative. This is physically meaningful and indicates a magnetizing effect for that winding. A negative star-leg impedance does not indicate an error -- it is a mathematically valid result of the decomposition and must be preserved in the equivalent circuit. + +### Star Bus Properties + +- **Voltage magnitude:** `VMSTAR` (per-unit), typically initialized to 1.0 +- **Voltage angle:** `ANSTAR` (degrees), typically initialized to 0.0 +- **Bus type:** PQ (type 1) -- no generation or load at the star bus; it is a zero-injection node +- **Base voltage:** Determined by the transformer winding voltages; typically set to the winding 1 nominal voltage (NOMV1) or the bus I base voltage (BASKV of bus I) +- **Bus number convention:** Tools that create star buses assign bus numbers sequentially starting from `max_bus_number + 1` in the network + +### Magnetizing Admittance Placement + +The magnetizing admittance fields `MAG1` and `MAG2` represent the transformer core losses and magnetizing current. In PSS/E, these are referenced to the winding 1 (primary) bus voltage base. + +In the star-bus equivalent circuit, the magnetizing admittance is placed on the winding 1 branch (between bus I and star bus S) as a shunt element at the from-bus (bus I) side. This is the convention used by MATPOWER's `psse2mpc` converter and most other tools that perform star-bus decomposition. Some tools (e.g., GridCal) may place the admittance at the star bus itself or distribute it differently, but the winding 1 branch placement is the most common convention. + +The interpretation of `MAG1` and `MAG2` depends on the `CM` code: +- **CM=1:** `MAG1` = per-unit conductance (G) on system base, `MAG2` = per-unit susceptance (B) on system base +- **CM=2:** `MAG1` = exciting current as percentage of nominal current, `MAG2` = core losses in watts + +## Winding Parameters + +### Impedance Parameters + +Each winding pair has its own resistance (R) and reactance (X) specified in Data Line 2: + +- **`R1-2`, `X1-2`, `SBASE1-2`:** Impedance between winding 1 and winding 2, on `SBASE1-2` MVA base (when CZ=1) +- **`R2-3`, `X2-3`, `SBASE2-3`:** Impedance between winding 2 and winding 3, on `SBASE2-3` MVA base (when CZ=1) +- **`R3-1`, `X3-1`, `SBASE3-1`:** Impedance between winding 3 and winding 1, on `SBASE3-1` MVA base (when CZ=1) + +The impedance interpretation depends on the `CZ` code: + +| CZ Mode | R Interpretation | X Interpretation | Base | +|---------|-----------------|-----------------|------| +| CZ=1 | Per-unit resistance on winding MVA base | Per-unit reactance on winding MVA base | `SBASEn-m` | +| CZ=2 | Per-unit resistance on system base | Per-unit reactance on system base | `SBASE` (system) | +| CZ=3 | Load loss in watts | Per-unit reactance on winding base | Mixed | + +**Base conversion (CZ=1 to system base):** Z_pu,system = Z_pu,winding \* (SBASE / SBASEn-m) + +See [Per-Unit Convention Reference](per-unit-conventions.md#three-winding-transformer-per-unit-bases) for detailed conversion formulas and worked examples. + +### Tap Ratios (WINDV1, WINDV2, WINDV3) + +Each winding has its own tap ratio specified in the respective data line (3, 4, or 5). The tap ratio interpretation depends on the `CW` code: + +| CW Mode | Interpretation | Conversion to per-unit of BASKV | +|---------|---------------|-------------------------------| +| CW=1 | Per-unit of bus BASKV | Already in target form | +| CW=2 | Actual kV | WINDV_pu = WINDV_kV / BASKV | +| CW=3 | Per-unit of NOMV | WINDV_pu = WINDV \* (NOMV / BASKV) | + +The `CW` code applies uniformly to all three windings -- PSS/E does not allow different CW modes per winding within the same transformer record. + +See [Per-Unit Convention Reference](per-unit-conventions.md#two-winding-transformer-tap-ratios) for conversion formulas. + +### Phase Angles (ANG1, ANG2, ANG3) + +Each winding has a phase shift angle specified in degrees: + +- **`ANG1`:** Phase shift angle for winding 1, in degrees +- **`ANG2`:** Phase shift angle for winding 2, in degrees +- **`ANG3`:** Phase shift angle for winding 3, in degrees + +Convention: a positive angle means the winding bus voltage leads the star bus voltage by that angle. For standard power transformers without phase shifting, ANG = 0.0 for all windings. Phase-shifting transformers typically have angles in the range -60 to +60 degrees. + +### MVA Ratings (RATA/B/C per winding) + +Each winding has three MVA rating levels: + +| Rating | Winding 1 | Winding 2 | Winding 3 | Description | +|--------|-----------|-----------|-----------|-------------| +| Rate A | `RATA1` | `RATA2` | `RATA3` | Normal operating MVA rating | +| Rate B | `RATB1` | `RATB2` | `RATC2` | Emergency MVA rating | +| Rate C | `RATC1` | `RATC2` | `RATC3` | Short-term (extreme) MVA rating | + +Ratings are per-winding, not per-transformer. Each winding may have a different MVA rating reflecting its physical capacity. The winding 1 (primary) typically has the highest rating. + +`RATA` also serves as the per-winding MVA base for impedance normalization when `SBASE1-2` is zero (defaults to `RATA1` for winding 1-2 pair). + +### Tap Changer Control Fields + +Each winding has a full set of tap changer control fields: + +| Field | Winding 1 | Winding 2 | Winding 3 | Description | +|-------|-----------|-----------|-----------|-------------| +| COD | `COD1` | `COD2` | `COD3` | Control mode code | +| CONT | `CONT1` | `CONT2` | `CONT3` | Controlled bus number | +| RMA | `RMA1` | `RMA2` | `RMA3` | Maximum tap ratio or angle | +| RMI | `RMI1` | `RMI2` | `RMI3` | Minimum tap ratio or angle | +| VMA | `VMA1` | `VMA2` | `VMA3` | Maximum voltage or flow limit | +| VMI | `VMI1` | `VMI2` | `VMI3` | Minimum voltage or flow limit | +| NTP | `NTP1` | `NTP2` | `NTP3` | Number of tap positions | +| TAB | `TAB1` | `TAB2` | `TAB3` | Impedance correction table number | +| CR | `CR1` | `CR2` | `CR3` | Load drop compensation resistance | +| CX | `CX1` | `CX2` | `CX3` | Load drop compensation reactance | +| CNXA | `CNXA1` | `CNXA2` | `CNXA3` | Connection angle for wye-delta transformers | + +**COD mode values:** + +| Code | Description | +|------|-------------| +| 0 | No tap changer control | +| 1 | Voltage control -- adjust tap to regulate `CONT` bus voltage | +| 2 | Reactive power flow control | +| 3 | Active power flow control | +| 4 | Control for HVDC converter transformer | +| -1 to -4 | Same as 1-4 but with step-up/step-down direction constraint | + +### Magnetizing Admittance (MAG1, MAG2) + +The magnetizing admittance is specified once per transformer (not per winding) in Data Line 1. Its interpretation depends on the `CM` code: + +| CM Mode | MAG1 | MAG2 | +|---------|------|------| +| CM=1 | Per-unit conductance (G) on system MVA base | Per-unit susceptance (B) on system MVA base | +| CM=2 | Exciting current as % of nominal current | Core losses in watts | + +The magnetizing admittance is referenced to winding 1 (primary) bus. In the star-bus equivalent, it is placed on the winding 1 branch as a shunt element. + +## Intermediate Format Representation + +### Single-Record Representation + +If the canonical parser preserves the 3-winding transformer as a single record, the intermediate format represents each 3-winding transformer as one row in the `transformer` table with all 83 fields as columns. Fields are organized by data line origin: + +- **Columns from Data Line 1:** `I`, `J`, `K`, `CKT`, `CW`, `CZ`, `CM`, `MAG1`, `MAG2`, `NMETR`, `NAME`, `STAT`, `O1`-`O4`, `F1`-`F4`, `VECGRP` +- **Columns from Data Line 2:** `R1-2`, `X1-2`, `SBASE1-2`, `R2-3`, `X2-3`, `SBASE2-3`, `R3-1`, `X3-1`, `SBASE3-1`, `VMSTAR`, `ANSTAR` +- **Columns from Data Lines 3-5:** Per-winding fields for windings 1, 2, and 3 (WINDV, NOMV, ANG, RATA/B/C, COD, CONT, RMA, RMI, VMA, VMI, NTP, TAB, CR, CX, CNXA) + +This representation preserves maximum fidelity -- all 83 fields are directly accessible and no information is lost. Tools with native 3-winding objects (pandapower, GridCal) can map directly from this representation. + +### Star-Bus Decomposition Representation + +If the canonical parser decomposes to star-bus equivalents, the intermediate format represents each 3-winding transformer as: + +- **Three 2-winding transformer records** in the `transformer` (or `branch`) table, one per winding, each connecting a physical winding bus (I, J, or K) to the synthetic star bus S +- **One synthetic star bus record** in the `bus` table + +**Star bus numbering:** The star bus number is assigned sequentially starting from `max_bus_number + 1`. For example, if the network has buses numbered up to 50000, the first 3-winding transformer's star bus is 50001, the second is 50002, and so on. + +**Branch records:** Each of the three 2-winding transformer branch records contains: +- From bus: physical winding bus (I, J, or K) +- To bus: star bus S +- Series impedance: star-leg impedance (Z1, Z2, or Z3) computed from pairwise impedances +- Tap ratio: per-winding tap ratio (WINDV1, WINDV2, or WINDV3) +- Phase angle: per-winding phase shift (ANG1, ANG2, or ANG3) +- MVA ratings: per-winding ratings (RATA/B/C for the respective winding) + +**Fields preserved in decomposition:** +- Winding impedances (converted to star-leg values) +- Tap ratios (WINDV1, WINDV2, WINDV3) +- Phase angles (ANG1, ANG2, ANG3) +- MVA ratings (RATA/B/C per winding) +- Status (STAT, applied per-winding) + +**Fields lost in decomposition:** +- Unified 3-winding record structure (K field, original pairwise impedances) +- Tap changer control modes and parameters (COD, CONT, NTP, TAB, CR, CX per winding) +- Impedance correction table references (TAB1, TAB2, TAB3) +- Ownership fields (O1-O4, F1-F4) +- Transformer name (NAME) +- Vector group (VECGRP) +- CW/CZ/CM control codes (implicitly consumed during conversion) +- Discrete tap step information (NTP, RMA, RMI) + +**Pairwise to star-leg impedance conversion:** Before decomposition, all pairwise impedances must be converted to a common per-unit base. Then the star-leg formulas are applied (see "Pairwise to Star-Leg Impedance Conversion" above). + +### Parser-Specific Behavior + +**MATPOWER (Phase 1 D4):** The `psse2mpc` converter automatically decomposes all 3-winding transformers into three `mpc.branch` rows at phantom star buses. Star buses are numbered starting from `max_bus_number + 1`. Pairwise impedances are converted to star-leg impedances on the system base. Tap ratios and phase angles are preserved in the `TAP` and `SHIFT` columns. Control modes, ownership, name, and discrete tap data are lost. This is the default MATPOWER representation and cannot be overridden. + +**GridCal (Phase 1 D5):** The GridCal PSS/E parser preserves the original 3-winding record structure as a `Transformer3W` object. All 83 fields are accessible through the object's attributes. Internally, GridCal computes the star-bus equivalent for power flow solution, but the user-facing data model retains the unified 3-winding parameterization. This provides maximum fidelity for round-trip data preservation. + +## Tool Handling + +### Summary Matrix + +| Tool | Native 3W | Representation | Key Limitations | +|------|-----------|---------------|-----------------| +| PyPSA | No | Star-bus decomposition into three `Transformer` components | Loses unified 3W parameterization; no tap changer control fields | +| pandapower | Yes | `create_transformer3w()` with internal star bus | Full 3W support; parameter names differ from PSS/E | +| GridCal | Yes | `Transformer3W` object | Closest to PSS/E native representation; preserves all fields | +| PowerModels.jl | No | Star-bus decomposition into `branch` dict entries | Loses tap changer control modes and discrete tap data | +| PowerSimulations.jl | No | Per-winding `TapTransformer` or `Transformer2W` via PowerSystems.jl | No native `Transformer3W` type; requires decomposition | +| MATPOWER | No | `mpc.branch` rows at phantom star bus | Loses COD, CONT, NTP, TAB, CR, CX, ownership, NAME | + +### PyPSA + +PyPSA does not have a native 3-winding transformer object. The `Transformer` component is strictly 2-winding, defined by a from-bus (`bus0`) and to-bus (`bus1`) with tap ratio (`tap_ratio`) and phase shift (`phase_shift`). + +To represent a 3-winding transformer in PyPSA: +1. Create a synthetic star `Bus` component with appropriate voltage level +2. Create three `Transformer` components connecting bus I to star bus, bus J to star bus, and bus K to star bus +3. Set `tap_ratio` and `phase_shift` on each component from the per-winding values +4. Compute star-leg impedances from pairwise impedances and assign to each component's `r` and `x` parameters + +**Fields preserved:** Winding impedances (as star-leg values), tap ratios, phase angles, MVA ratings. + +**Fields lost:** Unified 3-winding parameterization, tap changer control modes (COD), controlled bus (CONT), discrete tap positions (NTP), impedance correction tables (TAB), load drop compensation (CR, CX), ownership (O1-O4, F1-F4), name (NAME), vector group (VECGRP). + +### pandapower + +pandapower provides native 3-winding transformer support through `create_transformer3w()`. This function creates a single 3-winding transformer element with parameters mapped from the PSS/E fields: + +| pandapower Parameter | PSS/E Source | +|---------------------|-------------| +| `hv_bus` | `I` (winding 1 bus) | +| `mv_bus` | `J` (winding 2 bus) | +| `lv_bus` | `K` (winding 3 bus) | +| `sn_hv_mva` | `RATA1` or `SBASE1-2` | +| `sn_mv_mva` | `RATA2` or `SBASE2-3` | +| `sn_lv_mva` | `RATA3` or `SBASE3-1` | +| `vn_hv_kv` | Bus I `BASKV` or `NOMV1` | +| `vn_mv_kv` | Bus J `BASKV` or `NOMV2` | +| `vn_lv_kv` | Bus K `BASKV` or `NOMV3` | +| `vk_hv_percent` | Derived from `X1-2` (% short-circuit voltage, HV-MV) | +| `vk_mv_percent` | Derived from `X2-3` (% short-circuit voltage, MV-LV) | +| `vk_lv_percent` | Derived from `X3-1` (% short-circuit voltage, LV-HV) | +| `vkr_hv_percent` | Derived from `R1-2` (% resistive component, HV-MV) | +| `vkr_mv_percent` | Derived from `R2-3` (% resistive component, MV-LV) | +| `vkr_lv_percent` | Derived from `R3-1` (% resistive component, LV-HV) | +| `tap_pos` | Derived from `WINDV1`/`WINDV2`/`WINDV3` | +| `tap_side` | Winding with active tap changer (from `COD1`/`COD2`/`COD3`) | +| `tap_step_percent` | Computed from tap range and NTP | + +pandapower internally creates a star-bus decomposition for power flow computation, but this is transparent to the user. The 3-winding transformer data model preserves the unified parameterization. + +**Fields that cannot be expressed:** Per-winding load drop compensation (CR, CX), impedance correction table references (TAB), vector group (VECGRP), detailed ownership fractions (O1-O4, F1-F4). + +### GridCal + +GridCal provides a native `Transformer3W` class that preserves all three windings as a single object. The GridCal PSS/E parser maps PSS/E 3-winding transformer records directly to `Transformer3W` instances, preserving the original record structure. + +Key characteristics: +- All three winding impedances, tap ratios, and phase angles are stored as object attributes +- Internal star-bus computation is performed for power flow solution but is transparent +- The `Transformer3W` object is the closest representation to the PSS/E native format among all evaluated tools +- Tap changer control parameters are preserved at the object level + +**Fields preserved:** All 83 fields are accessible through the object's attributes, including control modes, ownership, and vector group. + +**Fields with limitations:** Some PSS/E-specific fields may be stored as generic metadata rather than typed attributes, depending on the GridCal version. + +### PowerModels.jl + +PowerModels.jl does not have a native 3-winding transformer object. The `parse_psse` function decomposes 3-winding transformers into three `branch` dictionary entries connected at a star bus: + +- A star bus is added to the `bus` dictionary with bus type PQ and voltage initialized from `VMSTAR`/`ANSTAR` +- Three branch entries are created with per-winding impedances (star-leg values), tap ratios, and phase angles +- The `tap` field stores the off-nominal turns ratio (dimensionless) +- The `shift` field stores the phase angle in **radians** (note: PSS/E uses degrees) + +**Fields preserved:** Winding impedances (as star-leg values), tap ratios (`tap`), phase angles (`shift`), MVA ratings (`rate_a`, `rate_b`, `rate_c`). + +**Fields lost:** Tap changer control modes (COD, CONT), discrete tap positions (NTP), impedance correction tables (TAB), load drop compensation (CR, CX), ownership (O1-O4, F1-F4), name (NAME), vector group (VECGRP), original pairwise impedances. + +### PowerSimulations.jl + +PowerSimulations.jl delegates data modeling to PowerSystems.jl. As of the current version, PowerSystems.jl does not include a dedicated `Transformer3W` type. Three-winding transformers must be decomposed into per-winding representations: + +- Each winding is represented as a `TapTransformer` (if it has a tap changer) or a `Transformer2W` equivalent +- A synthetic `ACBus` is created for the star bus +- Per-winding impedances, tap ratios, and phase angles are assigned to each component + +**Fields preserved:** Winding impedances (as star-leg values), tap ratios, phase angles, MVA ratings. + +**Fields lost:** Unified 3-winding parameterization, tap changer control modes beyond simple voltage regulation, discrete tap steps, impedance correction tables, load drop compensation, ownership, name, vector group. + +### MATPOWER + +MATPOWER's `psse2mpc` converter automatically decomposes 3-winding transformers into three `mpc.branch` matrix rows connected at a phantom star bus: + +- **Phantom bus numbering:** Star buses are numbered starting from `max_bus_number + 1`. For example, if the largest bus number in the network is 50000, the first 3-winding transformer's star bus is 50001. +- **Branch columns:** Each of the three branch rows contains: + - `F_BUS` / `T_BUS`: physical winding bus and star bus + - `BR_R` / `BR_X`: star-leg resistance and reactance on system MVA base + - `TAP`: per-winding tap ratio (off-nominal turns ratio) + - `SHIFT`: per-winding phase shift angle in degrees + - `RATE_A` / `RATE_B` / `RATE_C`: per-winding MVA ratings + - `BR_STATUS`: branch status + +**Fields lost in MATPOWER decomposition:** +- Tap changer control modes (`COD1`, `COD2`, `COD3`) +- Controlled bus references (`CONT1`, `CONT2`, `CONT3`) +- Number of tap positions (`NTP1`, `NTP2`, `NTP3`) +- Impedance correction table references (`TAB1`, `TAB2`, `TAB3`) +- Load drop compensation (`CR1`/`CX1`, `CR2`/`CX2`, `CR3`/`CX3`) +- Ownership fields (`O1`-`O4`, `F1`-`F4`) +- Transformer name (`NAME`) +- Vector group (`VECGRP`) +- Connection angles (`CNXA1`, `CNXA2`, `CNXA3`) +- Original pairwise impedances (replaced by star-leg values) + +## Worked Example + +### Source Parameters + +A synthetic 500/230/115 kV autotransformer representative of bulk transmission infrastructure in the ISO transmission system: + +- **Winding 1 (HV):** 500 kV nominal, 600 MVA rating +- **Winding 2 (MV):** 230 kV nominal, 300 MVA rating +- **Winding 3 (LV):** 115 kV nominal, 100 MVA rating +- **System base:** SBASE = 100 MVA + +Pairwise impedances (CZ=1, on per-winding MVA base): +- Z1-2: R1-2 = 0.0012, X1-2 = 0.0856 pu on SBASE1-2 = 600 MVA +- Z2-3: R2-3 = 0.0028, X2-3 = 0.1245 pu on SBASE2-3 = 300 MVA +- Z3-1: R3-1 = 0.0019, X3-1 = 0.0934 pu on SBASE3-1 = 600 MVA + +Tap ratios (CW=1): WINDV1 = 1.025, WINDV2 = 1.000, WINDV3 = 1.000 + +Control: COD1 = 1 (voltage control), CONT1 = 99999 (regulated bus) + +Star bus initial voltage: VMSTAR = 1.0, ANSTAR = 0.0 + +### PSS/E Record Representation + +**Data Line 1:** + +``` +99001, 99002, 99003, '1 ', 1, 1, 1, 0.00000, 0.00000, 2, '3W-EXAMPLE ', 1, 1, 1.0, 0, 1.0, 0, 1.0, 0, 1.0, 'YNyn0' +``` + +Fields: I=99001, J=99002, K=99003, CKT='1 ', CW=1, CZ=1, CM=1, MAG1=0.0, MAG2=0.0, NMETR=2, NAME='3W-EXAMPLE', STAT=1, O1=1, F1=1.0, O2=0, F2=1.0, O3=0, F3=1.0, O4=0, F4=1.0, VECGRP='YNyn0' + +**Data Line 2:** + +``` +0.00120, 0.08560, 600.0, 0.00280, 0.12450, 300.0, 0.00190, 0.09340, 600.0, 1.00000, 0.00000 +``` + +Fields: R1-2=0.0012, X1-2=0.0856, SBASE1-2=600.0, R2-3=0.0028, X2-3=0.1245, SBASE2-3=300.0, R3-1=0.0019, X3-1=0.0934, SBASE3-1=600.0, VMSTAR=1.0, ANSTAR=0.0 + +**Data Line 3 (Winding 1):** + +``` +1.02500, 500.000, 0.000, 600.00, 600.00, 600.00, 1, 99999, 1.10000, 0.90000, 1.10000, 0.90000, 33, 0, 0.00000, 0.00000, 0.00000 +``` + +Fields: WINDV1=1.025, NOMV1=500.0, ANG1=0.0, RATA1=600.0, RATB1=600.0, RATC1=600.0, COD1=1, CONT1=99999, RMA1=1.1, RMI1=0.9, VMA1=1.1, VMI1=0.9, NTP1=33, TAB1=0, CR1=0.0, CX1=0.0, CNXA1=0.0 + +**Data Line 4 (Winding 2):** + +``` +1.00000, 230.000, 0.000, 300.00, 300.00, 300.00, 0, 0, 1.10000, 0.90000, 1.10000, 0.90000, 33, 0, 0.00000, 0.00000, 0.00000 +``` + +Fields: WINDV2=1.0, NOMV2=230.0, ANG2=0.0, RATA2=300.0, RATB2=300.0, RATC2=300.0, COD2=0, CONT2=0, RMA2=1.1, RMI2=0.9, VMA2=1.1, VMI2=0.9, NTP2=33, TAB2=0, CR2=0.0, CX2=0.0, CNXA2=0.0 + +**Data Line 5 (Winding 3):** + +``` +1.00000, 115.000, 0.000, 100.00, 100.00, 100.00, 0, 0, 1.10000, 0.90000, 1.10000, 0.90000, 33, 0, 0.00000, 0.00000, 0.00000 +``` + +Fields: WINDV3=1.0, NOMV3=115.0, ANG3=0.0, RATA3=100.0, RATB3=100.0, RATC3=100.0, COD3=0, CONT3=0, RMA3=1.1, RMI3=0.9, VMA3=1.1, VMI3=0.9, NTP3=33, TAB3=0, CR3=0.0, CX3=0.0, CNXA3=0.0 + +### Star-Bus Decomposition + +Step 1 -- Convert pairwise impedances to system base (100 MVA): + +Each pairwise impedance is on a different winding MVA base. Convert to system base using: Z_pu,system = Z_pu,winding \* (SBASE / SBASEn-m) + +X1-2 on system base: X12_sys = 0.0856 \* (100 / 600) = 0.01427 pu + +X2-3 on system base: X23_sys = 0.1245 \* (100 / 300) = 0.04150 pu + +X3-1 on system base: X31_sys = 0.0934 \* (100 / 600) = 0.01557 pu + +R1-2 on system base: R12_sys = 0.0012 \* (100 / 600) = 0.0002000 pu + +R2-3 on system base: R23_sys = 0.0028 \* (100 / 300) = 0.0009333 pu + +R3-1 on system base: R31_sys = 0.0019 \* (100 / 600) = 0.0003167 pu + +Step 2 -- Apply star-leg impedance formulas: + +X1 = (X12_sys + X31_sys - X23_sys) / 2 = (0.01427 + 0.01557 - 0.04150) / 2 = -0.005833 pu + +X2 = (X12_sys + X23_sys - X31_sys) / 2 = (0.01427 + 0.04150 - 0.01557) / 2 = 0.02010 pu + +X3 = (X23_sys + X31_sys - X12_sys) / 2 = (0.04150 + 0.01557 - 0.01427) / 2 = 0.02140 pu + +R1 = (R12_sys + R31_sys - R23_sys) / 2 = (0.0002000 + 0.0003167 - 0.0009333) / 2 = -0.0002083 pu + +R2 = (R12_sys + R23_sys - R31_sys) / 2 = (0.0002000 + 0.0009333 - 0.0003167) / 2 = 0.0004083 pu + +R3 = (R23_sys + R31_sys - R12_sys) / 2 = (0.0009333 + 0.0003167 - 0.0002000) / 2 = 0.0005250 pu + +Note: X1 and R1 are negative, which is typical for autotransformers where the primary and secondary windings are electrically connected (not magnetically isolated). + +Step 3 -- Resulting 2-winding transformer records: + +| Branch | From Bus | To Bus | R (pu) | X (pu) | TAP | SHIFT | RATA (MVA) | +|--------|----------|--------|--------|--------|-----|-------|------------| +| Winding 1 | 99001 (I) | 50001 (Star) | -0.0002083 | -0.005833 | 1.025 | 0.0 | 600.0 | +| Winding 2 | 99002 (J) | 50001 (Star) | 0.0004083 | 0.02010 | 1.000 | 0.0 | 300.0 | +| Winding 3 | 99003 (K) | 50001 (Star) | 0.0005250 | 0.02140 | 1.000 | 0.0 | 100.0 | + +Step 4 -- Star bus parameters: + +Star bus 50001: VM = 1.0 pu (VMSTAR), VA = 0.0 degrees (ANSTAR), type = PQ (1) + +### Numeric Verification + +Verify that pairwise impedances can be reconstructed from star-leg impedances (all on system base): + +**Z1-2 verification:** + +X1 + X2 = -0.005833 + 0.02010 = 0.01427 pu (matches X12_sys = 0.01427 pu) + +R1 + R2 = -0.0002083 + 0.0004083 = 0.0002000 pu (matches R12_sys = 0.0002000 pu) + +**Z2-3 verification:** + +X2 + X3 = 0.02010 + 0.02140 = 0.04150 pu (matches X23_sys = 0.04150 pu) + +R2 + R3 = 0.0004083 + 0.0005250 = 0.0009333 pu (matches R23_sys = 0.0009333 pu) + +**Z3-1 verification:** + +X3 + X1 = 0.02140 + (-0.005833) = 0.01557 pu (matches X31_sys = 0.01557 pu) + +R3 + R1 = 0.0005250 + (-0.0002083) = 0.0003167 pu (matches R31_sys = 0.0003167 pu) + +All six reconstructed values match the converted pairwise impedances to 4 significant digits, confirming the star-bus decomposition is consistent. + +## Common Pitfalls + +1. **Per-unit base mismatch:** Each winding pair has its own MVA base (SBASE1-2, SBASE2-3, SBASE3-1). All three pairwise impedances must be converted to a common base (typically system MVA base) before applying the star-leg decomposition formulas. Failing to do this produces incorrect star-leg impedances and wrong power flow results. + +2. **Star bus voltage initialization:** Forgetting to set VMSTAR and ANSTAR initial values for the star bus can cause power flow convergence problems. The star bus voltage should be initialized to reasonable values (typically 1.0 pu and 0.0 degrees). + +3. **Negative star-leg impedance:** One or more star-leg impedances (Z1, Z2, or Z3) can be negative, particularly for autotransformers. This is physically meaningful and must be preserved in the equivalent circuit. Tools that reject negative impedance values will fail to model these transformers correctly. + +4. **Tap changer loss in decomposition:** Tools that decompose 3-winding transformers into star-bus equivalents lose the tap changer control mode fields (COD, CONT, NTP, TAB, CR, CX). This means automatic voltage regulation and power flow control by tap changers cannot be modeled at the individual winding level after decomposition. + +5. **Rating interpretation:** RATA, RATB, and RATC are per-winding MVA ratings, not per-transformer totals. Each winding has its own thermal capacity. Using a single transformer-level rating would understate the capacity of smaller windings and overstate the capacity of the tertiary. + +6. **Magnetizing admittance placement:** In the star-bus equivalent, the magnetizing admittance (MAG1, MAG2) must be placed on the correct branch (winding 1 / primary side). Placing it on the wrong branch or distributing it equally across all three branches produces incorrect no-load loss and magnetizing current representation. + +7. **CW/CZ/CM mode inconsistency:** PSS/E enforces a single CW, CZ, and CM code per transformer record (applying to all windings uniformly). However, when constructing transformer data manually or converting between formats, it is possible to inadvertently apply different interpretation modes to different windings. The CW, CZ, and CM codes must be checked and applied consistently to all three windings. + +## Cross-References + +- **Phase 1 D6 -- Parser Fidelity Comparison:** Documents parser-specific 3-winding transformer handling, including star-bus decomposition details and field preservation differences between MATPOWER and GridCal parsers. +- **Phase 2 PRD 01 -- Intermediate Format Schema Reference:** Provides field definitions, data types, and nullable status for all transformer table columns. See `intermediate-schema.md`. +- **Phase 2 PRD 03 -- Per-Unit Convention Reference, Section 6:** Covers 3-winding transformer per-unit bases, per-winding MVA bases, and impedance base conversion formulas. See [per-unit-conventions.md](per-unit-conventions.md#three-winding-transformer-per-unit-bases). +- **Phase 2 PRD 05 -- Field Criticality Matrix:** Classifies each of the 83 3-winding transformer fields by criticality tier (DCPF-critical, ACPF-critical, informational, discardable). See `field-criticality-matrix.md`. diff --git a/data/fnm/intermediate/README.md b/data/fnm/intermediate/README.md new file mode 100644 index 00000000..1e76aa26 --- /dev/null +++ b/data/fnm/intermediate/README.md @@ -0,0 +1,16 @@ +# FNM Intermediate Directory + +This directory stores parser output and intermediate data formats produced during FNM +ingestion. Files here are generated by the parsing scripts in `../scripts/` and consumed +by downstream validation and analysis tools. + +## Contents (populated by later phases) + +- Parsed PSS/E RAW data in Parquet format +- Supplemental CSV data merged and normalized +- Intermediate network model representations + +## NDA Notice + +All files in this directory are derived from NDA-restricted FNM source data and must not +be committed to version control. The parent `.gitignore` blocks this entire directory. diff --git a/data/fnm/intermediate/schemas/area.schema.json b/data/fnm/intermediate/schemas/area.schema.json new file mode 100644 index 00000000..14de7c35 --- /dev/null +++ b/data/fnm/intermediate/schemas/area.schema.json @@ -0,0 +1,67 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "schemas/area.schema.json", + "title": "Area", + "description": "PSS/E v31 Area record type. Each row defines an area for interchange control.", + "type": "object", + "properties": { + "I": { + "type": "integer", + "description": "Area number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ISW": { + "type": "integer", + "description": "Area slack bus number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "PDES": { + "type": "number", + "description": "Desired net interchange", + "x-psse-unit": "MW", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "PTOL": { + "type": "number", + "description": "Interchange tolerance", + "x-psse-unit": "MW", + "x-psse-per-unit-base": "none", + "x-psse-default": 10.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ARNAME": { + "type": "string", + "description": "Area name (12 chars)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": " ", + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + } + }, + "required": [ + "I", + "ISW", + "PDES", + "PTOL", + "ARNAME" + ], + "additionalProperties": false +} diff --git a/data/fnm/intermediate/schemas/branch.schema.json b/data/fnm/intermediate/schemas/branch.schema.json new file mode 100644 index 00000000..08cf3b33 --- /dev/null +++ b/data/fnm/intermediate/schemas/branch.schema.json @@ -0,0 +1,280 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "schemas/branch.schema.json", + "title": "Branch", + "description": "PSS/E v31 Branch record type. Each row represents a transmission line or cable.", + "type": "object", + "properties": { + "I": { + "type": "integer", + "description": "From bus number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "J": { + "type": "integer", + "description": "To bus number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "CKT": { + "type": "string", + "description": "Circuit identifier (2 chars)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": "1 ", + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "R": { + "type": "number", + "description": "Branch resistance", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "system_mva", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "X": { + "type": "number", + "description": "Branch reactance", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "system_mva", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "B": { + "type": "number", + "description": "Branch charging susceptance", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "system_mva", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RATEA": { + "type": "number", + "description": "Rating A (normal)", + "x-psse-unit": "MVA", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RATEB": { + "type": "number", + "description": "Rating B (emergency)", + "x-psse-unit": "MVA", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RATEC": { + "type": "number", + "description": "Rating C (long-term)", + "x-psse-unit": "MVA", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "GI": { + "type": "number", + "description": "Shunt conductance at I", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "system_mva", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "BI": { + "type": "number", + "description": "Shunt susceptance at I", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "system_mva", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "GJ": { + "type": "number", + "description": "Shunt conductance at J", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "system_mva", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "BJ": { + "type": "number", + "description": "Shunt susceptance at J", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "system_mva", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ST": { + "type": "integer", + "description": "Status (1=in, 0=out)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0, + 1 + ] + }, + "MET": { + "type": "integer", + "description": "Metered end (1=I, 2=J)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 1, + 2 + ] + }, + "LEN": { + "type": "number", + "description": "Line length", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "O1": { + "type": "integer", + "description": "Owner 1", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F1": { + "type": "number", + "description": "Fraction owned by owner 1", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0.0, + 1.0 + ] + }, + "O2": { + "type": "integer", + "description": "Owner 2", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F2": { + "type": "number", + "description": "Fraction owned by owner 2", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0.0, + 1.0 + ] + }, + "O3": { + "type": "integer", + "description": "Owner 3", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F3": { + "type": "number", + "description": "Fraction owned by owner 3", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0.0, + 1.0 + ] + }, + "O4": { + "type": "integer", + "description": "Owner 4", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F4": { + "type": "number", + "description": "Fraction owned by owner 4", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0.0, + 1.0 + ] + } + }, + "required": [ + "I", + "J", + "CKT", + "R", + "X", + "B", + "RATEA", + "RATEB", + "RATEC", + "ST" + ], + "additionalProperties": false +} diff --git a/data/fnm/intermediate/schemas/bus.schema.json b/data/fnm/intermediate/schemas/bus.schema.json new file mode 100644 index 00000000..adf1ee7a --- /dev/null +++ b/data/fnm/intermediate/schemas/bus.schema.json @@ -0,0 +1,184 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "schemas/bus.schema.json", + "title": "Bus", + "description": "PSS/E v31 Bus record type. Each row represents one bus in the network model.", + "type": "object", + "properties": { + "I": { + "type": "integer", + "description": "Bus number (1-999997)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 1, + 999997 + ] + }, + "NAME": { + "type": "string", + "description": "Bus name (up to 12 chars)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": " ", + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "BASKV": { + "type": "number", + "description": "Bus base voltage", + "x-psse-unit": "kV", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0.0, + null + ] + }, + "IDE": { + "type": "integer", + "description": "Bus type code (1-4)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 1, + 4 + ] + }, + "AREA": { + "type": "integer", + "description": "Area number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 1, + null + ] + }, + "ZONE": { + "type": "integer", + "description": "Zone number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 1, + null + ] + }, + "OWNER": { + "type": "integer", + "description": "Owner number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 1, + null + ] + }, + "VM": { + "type": "number", + "description": "Bus voltage magnitude", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "bus_kv", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0.0, + 2.0 + ] + }, + "VA": { + "type": "number", + "description": "Bus voltage angle", + "x-psse-unit": "deg", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "NVHI": { + "type": "number", + "description": "Normal voltage high limit", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "bus_kv", + "x-psse-default": 1.1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0.0, + 2.0 + ] + }, + "NVLO": { + "type": "number", + "description": "Normal voltage low limit", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "bus_kv", + "x-psse-default": 0.9, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0.0, + 2.0 + ] + }, + "EVHI": { + "type": "number", + "description": "Emergency voltage high limit", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "bus_kv", + "x-psse-default": 1.1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0.0, + 2.0 + ] + }, + "EVLO": { + "type": "number", + "description": "Emergency voltage low limit", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "bus_kv", + "x-psse-default": 0.9, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0.0, + 2.0 + ] + } + }, + "required": [ + "I", + "NAME", + "BASKV", + "IDE", + "AREA", + "ZONE", + "OWNER", + "VM", + "VA" + ], + "additionalProperties": false +} diff --git a/data/fnm/intermediate/schemas/facts.schema.json b/data/fnm/intermediate/schemas/facts.schema.json new file mode 100644 index 00000000..0c8273b3 --- /dev/null +++ b/data/fnm/intermediate/schemas/facts.schema.json @@ -0,0 +1,156 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "schemas/facts.schema.json", + "title": "FACTS", + "description": "PSS/E v31 FACTS device record type.", + "type": "object", + "properties": { + "NAME": { + "type": "string", + "description": "FACTS device name", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "I": { + "type": "integer", + "description": "Sending end bus", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "J": { + "type": "integer", + "description": "Terminal bus (0=shunt)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "MODE": { + "type": "integer", + "description": "FACTS control mode", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "SET1": { + "type": "number", + "description": "Control setpoint 1", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "SET2": { + "type": "number", + "description": "Control setpoint 2", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "VSREF": { + "type": "number", + "description": "Series voltage reference", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "bus_kv", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "REMOT": { + "type": "integer", + "description": "Remote bus for V control", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "MESSION": { + "type": "number", + "description": "Sending end impedance", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "LINX": { + "type": "number", + "description": "Series reactance", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "system_mva", + "x-psse-default": 0.05, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RMPCT": { + "type": "number", + "description": "MVAR pct for remote reg", + "x-psse-unit": "%", + "x-psse-per-unit-base": "none", + "x-psse-default": 100.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "OWNER": { + "type": "integer", + "description": "Owner number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "SET3": { + "type": "number", + "description": "Control setpoint 3", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "SET4": { + "type": "number", + "description": "Control setpoint 4", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + } + }, + "required": [ + "NAME", + "I", + "J", + "MODE" + ], + "additionalProperties": false +} diff --git a/data/fnm/intermediate/schemas/fixed_shunt.schema.json b/data/fnm/intermediate/schemas/fixed_shunt.schema.json new file mode 100644 index 00000000..f8f4cd6d --- /dev/null +++ b/data/fnm/intermediate/schemas/fixed_shunt.schema.json @@ -0,0 +1,70 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "schemas/fixed_shunt.schema.json", + "title": "Fixed Shunt", + "description": "PSS/E v31 Fixed Shunt record type. Each row represents a fixed shunt element.", + "type": "object", + "properties": { + "I": { + "type": "integer", + "description": "Bus number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ID": { + "type": "string", + "description": "Shunt identifier (2 chars)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": "1 ", + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "STATUS": { + "type": "integer", + "description": "Status (1=in, 0=out)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0, + 1 + ] + }, + "GL": { + "type": "number", + "description": "Shunt conductance", + "x-psse-unit": "MW", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "BL": { + "type": "number", + "description": "Shunt susceptance (+cap)", + "x-psse-unit": "MVAR", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + } + }, + "required": [ + "I", + "ID", + "STATUS", + "GL", + "BL" + ], + "additionalProperties": false +} diff --git a/data/fnm/intermediate/schemas/generator.schema.json b/data/fnm/intermediate/schemas/generator.schema.json new file mode 100644 index 00000000..c6d54455 --- /dev/null +++ b/data/fnm/intermediate/schemas/generator.schema.json @@ -0,0 +1,320 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "schemas/generator.schema.json", + "title": "Generator", + "description": "PSS/E v31 Generator record type. Each row represents one generating unit.", + "type": "object", + "properties": { + "I": { + "type": "integer", + "description": "Bus number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ID": { + "type": "string", + "description": "Machine identifier (2 chars)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": "1 ", + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "PG": { + "type": "number", + "description": "Active power output", + "x-psse-unit": "MW", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "QG": { + "type": "number", + "description": "Reactive power output", + "x-psse-unit": "MVAR", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "QT": { + "type": "number", + "description": "Max reactive power", + "x-psse-unit": "MVAR", + "x-psse-per-unit-base": "none", + "x-psse-default": 9999.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "QB": { + "type": "number", + "description": "Min reactive power", + "x-psse-unit": "MVAR", + "x-psse-per-unit-base": "none", + "x-psse-default": -9999.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "VS": { + "type": "number", + "description": "Regulated voltage setpoint", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "bus_kv", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "IREG": { + "type": "integer", + "description": "Remote regulated bus (0=local)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "MBASE": { + "type": "number", + "description": "Machine MVA base", + "x-psse-unit": "MVA", + "x-psse-per-unit-base": "none", + "x-psse-default": 100.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ZR": { + "type": "number", + "description": "Machine resistance (on MBASE)", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "system_mva", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ZX": { + "type": "number", + "description": "Machine reactance (on MBASE)", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "system_mva", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RT": { + "type": "number", + "description": "Step-up xfmr resistance", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "system_mva", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "XT": { + "type": "number", + "description": "Step-up xfmr reactance", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "system_mva", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "GTAP": { + "type": "number", + "description": "Step-up xfmr tap ratio", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "bus_kv", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "STAT": { + "type": "integer", + "description": "Status (1=in, 0=out)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0, + 1 + ] + }, + "RMPCT": { + "type": "number", + "description": "MVAR range pct for remote reg", + "x-psse-unit": "%", + "x-psse-per-unit-base": "none", + "x-psse-default": 100.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0.0, + 100.0 + ] + }, + "PT": { + "type": "number", + "description": "Max active power", + "x-psse-unit": "MW", + "x-psse-per-unit-base": "none", + "x-psse-default": 9999.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "PB": { + "type": "number", + "description": "Min active power", + "x-psse-unit": "MW", + "x-psse-per-unit-base": "none", + "x-psse-default": -9999.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "O1": { + "type": "integer", + "description": "Owner 1", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F1": { + "type": "number", + "description": "Fraction owned by owner 1", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0.0, + 1.0 + ] + }, + "O2": { + "type": "integer", + "description": "Owner 2", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F2": { + "type": "number", + "description": "Fraction owned by owner 2", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0.0, + 1.0 + ] + }, + "O3": { + "type": "integer", + "description": "Owner 3", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F3": { + "type": "number", + "description": "Fraction owned by owner 3", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0.0, + 1.0 + ] + }, + "O4": { + "type": "integer", + "description": "Owner 4", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F4": { + "type": "number", + "description": "Fraction owned by owner 4", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0.0, + 1.0 + ] + }, + "WMOD": { + "type": "integer", + "description": "Wind machine Q control mode", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "WPF": { + "type": "number", + "description": "Wind machine power factor", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + } + }, + "required": [ + "I", + "ID", + "PG", + "QG", + "QT", + "QB", + "VS", + "IREG", + "MBASE", + "STAT" + ], + "additionalProperties": false +} diff --git a/data/fnm/intermediate/schemas/impedance_correction.schema.json b/data/fnm/intermediate/schemas/impedance_correction.schema.json new file mode 100644 index 00000000..2d0ff3f2 --- /dev/null +++ b/data/fnm/intermediate/schemas/impedance_correction.schema.json @@ -0,0 +1,243 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "schemas/impedance_correction.schema.json", + "title": "Impedance Correction", + "description": "PSS/E v31 Impedance Correction table record type.", + "type": "object", + "properties": { + "T": { + "type": "integer", + "description": "Correction table number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "T1": { + "type": "number", + "description": "Tap ratio/angle pair 1", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F1": { + "type": "number", + "description": "Correction factor pair 1", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "T2": { + "type": "number", + "description": "Tap ratio/angle pair 2", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F2": { + "type": "number", + "description": "Correction factor pair 2", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "T3": { + "type": "number", + "description": "Tap ratio/angle pair 3", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F3": { + "type": "number", + "description": "Correction factor pair 3", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "T4": { + "type": "number", + "description": "Tap ratio/angle pair 4", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F4": { + "type": "number", + "description": "Correction factor pair 4", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "T5": { + "type": "number", + "description": "Tap ratio/angle pair 5", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F5": { + "type": "number", + "description": "Correction factor pair 5", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "T6": { + "type": "number", + "description": "Tap ratio/angle pair 6", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F6": { + "type": "number", + "description": "Correction factor pair 6", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "T7": { + "type": "number", + "description": "Tap ratio/angle pair 7", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F7": { + "type": "number", + "description": "Correction factor pair 7", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "T8": { + "type": "number", + "description": "Tap ratio/angle pair 8", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F8": { + "type": "number", + "description": "Correction factor pair 8", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "T9": { + "type": "number", + "description": "Tap ratio/angle pair 9", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F9": { + "type": "number", + "description": "Correction factor pair 9", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "T10": { + "type": "number", + "description": "Tap ratio/angle pair 10", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F10": { + "type": "number", + "description": "Correction factor pair 10", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "T11": { + "type": "number", + "description": "Tap ratio/angle pair 11", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F11": { + "type": "number", + "description": "Correction factor pair 11", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + } + }, + "required": [ + "T" + ], + "additionalProperties": false +} diff --git a/data/fnm/intermediate/schemas/interarea_transfer.schema.json b/data/fnm/intermediate/schemas/interarea_transfer.schema.json new file mode 100644 index 00000000..f66ee68f --- /dev/null +++ b/data/fnm/intermediate/schemas/interarea_transfer.schema.json @@ -0,0 +1,56 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "schemas/interarea_transfer.schema.json", + "title": "Interarea Transfer", + "description": "PSS/E v31 Interarea Transfer record type.", + "type": "object", + "properties": { + "ARFROM": { + "type": "integer", + "description": "From area number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ARTO": { + "type": "integer", + "description": "To area number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "TRID": { + "type": "string", + "description": "Transfer ID (2 chars)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": "1 ", + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "PTRAN": { + "type": "number", + "description": "Transfer amount", + "x-psse-unit": "MW", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + } + }, + "required": [ + "ARFROM", + "ARTO", + "TRID", + "PTRAN" + ], + "additionalProperties": false +} diff --git a/data/fnm/intermediate/schemas/load.schema.json b/data/fnm/intermediate/schemas/load.schema.json new file mode 100644 index 00000000..8c096653 --- /dev/null +++ b/data/fnm/intermediate/schemas/load.schema.json @@ -0,0 +1,155 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "schemas/load.schema.json", + "title": "Load", + "description": "PSS/E v31 Load record type. Each row represents one load at a bus.", + "type": "object", + "properties": { + "I": { + "type": "integer", + "description": "Bus number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ID": { + "type": "string", + "description": "Load identifier (2 chars)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": "1 ", + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "STATUS": { + "type": "integer", + "description": "Load status (1=in, 0=out)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0, + 1 + ] + }, + "AREA": { + "type": "integer", + "description": "Area number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ZONE": { + "type": "integer", + "description": "Zone number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "PL": { + "type": "number", + "description": "Constant power load (P)", + "x-psse-unit": "MW", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "QL": { + "type": "number", + "description": "Constant power load (Q)", + "x-psse-unit": "MVAR", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "IP": { + "type": "number", + "description": "Constant current load (P)", + "x-psse-unit": "MW", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "IQ": { + "type": "number", + "description": "Constant current load (Q)", + "x-psse-unit": "MVAR", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "YP": { + "type": "number", + "description": "Constant admittance load (P)", + "x-psse-unit": "MW", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "YQ": { + "type": "number", + "description": "Constant admittance load (Q)", + "x-psse-unit": "MVAR", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "OWNER": { + "type": "integer", + "description": "Owner number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "SCALE": { + "type": "integer", + "description": "Scaling flag (1=yes, 0=no)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0, + 1 + ] + } + }, + "required": [ + "I", + "ID", + "STATUS", + "AREA", + "ZONE", + "PL", + "QL" + ], + "additionalProperties": false +} diff --git a/data/fnm/intermediate/schemas/manifest.schema.json b/data/fnm/intermediate/schemas/manifest.schema.json new file mode 100644 index 00000000..06d5dd67 --- /dev/null +++ b/data/fnm/intermediate/schemas/manifest.schema.json @@ -0,0 +1,104 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "schemas/manifest.schema.json", + "title": "Intermediate Format Manifest", + "description": "Top-level manifest listing all tables in the intermediate format with metadata.", + "type": "object", + "properties": { + "sbase": { + "type": "number", + "description": "System MVA base" + }, + "basfrq": { + "type": "number", + "description": "System base frequency (Hz)" + }, + "rev": { + "type": "number", + "description": "PSS/E revision number" + }, + "case_id": { + "type": "string", + "description": "Case identification string" + }, + "canonical_parser": { + "type": "string", + "enum": [ + "matpower", + "gridcal" + ], + "description": "Canonical parser from D6" + }, + "tables": { + "type": "array", + "items": { + "type": "object", + "properties": { + "table_name": { + "type": "string" + }, + "record_type": { + "type": "string" + }, + "file_name": { + "type": "string" + }, + "record_count": { + "type": "integer", + "minimum": 0 + }, + "column_count": { + "type": "integer", + "minimum": 1 + }, + "schema_file": { + "type": "string" + } + }, + "required": [ + "table_name", + "record_type", + "file_name", + "record_count", + "column_count", + "schema_file" + ] + } + }, + "total_records": { + "type": "integer", + "minimum": 0 + }, + "total_tables": { + "type": "integer", + "minimum": 1 + }, + "non_empty_record_types": { + "type": "array", + "items": { + "type": "string" + } + }, + "schema_version": { + "type": "string" + }, + "generated_timestamp": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "sbase", + "basfrq", + "rev", + "case_id", + "canonical_parser", + "tables", + "total_records", + "total_tables", + "non_empty_record_types", + "schema_version", + "generated_timestamp" + ], + "additionalProperties": false +} diff --git a/data/fnm/intermediate/schemas/multi_section_line.schema.json b/data/fnm/intermediate/schemas/multi_section_line.schema.json new file mode 100644 index 00000000..122b7b50 --- /dev/null +++ b/data/fnm/intermediate/schemas/multi_section_line.schema.json @@ -0,0 +1,157 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "schemas/multi_section_line.schema.json", + "title": "Multi-Section Line", + "description": "PSS/E v31 Multi-Section Line grouping record type. DUM fields define intermediate buses.", + "type": "object", + "properties": { + "I": { + "type": "integer", + "description": "From bus number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "J": { + "type": "integer", + "description": "To bus number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ID": { + "type": "string", + "description": "Line identifier", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": "1 ", + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "MET": { + "type": "integer", + "description": "Metered end flag", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 1, + 2 + ] + }, + "DUM1": { + "type": "integer", + "description": "Intermediate bus 1", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "DUM2": { + "type": "integer", + "description": "Intermediate bus 2", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "DUM3": { + "type": "integer", + "description": "Intermediate bus 3", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "DUM4": { + "type": "integer", + "description": "Intermediate bus 4", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "DUM5": { + "type": "integer", + "description": "Intermediate bus 5", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "DUM6": { + "type": "integer", + "description": "Intermediate bus 6", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "DUM7": { + "type": "integer", + "description": "Intermediate bus 7", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "DUM8": { + "type": "integer", + "description": "Intermediate bus 8", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "DUM9": { + "type": "integer", + "description": "Intermediate bus 9", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + } + }, + "required": [ + "I", + "J", + "ID", + "DUM1", + "DUM2", + "DUM3", + "DUM4", + "DUM5", + "DUM6", + "DUM7", + "DUM8", + "DUM9" + ], + "additionalProperties": false +} diff --git a/data/fnm/intermediate/schemas/multi_terminal_dc.schema.json b/data/fnm/intermediate/schemas/multi_terminal_dc.schema.json new file mode 100644 index 00000000..3747cff0 --- /dev/null +++ b/data/fnm/intermediate/schemas/multi_terminal_dc.schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "schemas/multi_terminal_dc.schema.json", + "title": "Multi-Terminal DC", + "description": "PSS/E v31 Multi-Terminal DC line header record type.", + "type": "object", + "properties": { + "NAME": { + "type": "string", + "description": "MT DC line name", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "NCONV": { + "type": "integer", + "description": "Number of AC converters", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "NDCBS": { + "type": "integer", + "description": "Number of DC buses", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "NDCLN": { + "type": "integer", + "description": "Number of DC links", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "MDC": { + "type": "integer", + "description": "Control mode", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "VCONV": { + "type": "integer", + "description": "DC voltage ctrl converter", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "VCMOD": { + "type": "number", + "description": "Mode switch DC voltage", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "VCONVN": { + "type": "integer", + "description": "New voltage ctrl converter", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + } + }, + "required": [ + "NAME", + "NCONV", + "NDCBS", + "NDCLN" + ], + "additionalProperties": false +} diff --git a/data/fnm/intermediate/schemas/owner.schema.json b/data/fnm/intermediate/schemas/owner.schema.json new file mode 100644 index 00000000..416770c3 --- /dev/null +++ b/data/fnm/intermediate/schemas/owner.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "schemas/owner.schema.json", + "title": "Owner", + "description": "PSS/E v31 Owner record type.", + "type": "object", + "properties": { + "I": { + "type": "integer", + "description": "Owner number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "OWNAME": { + "type": "string", + "description": "Owner name (12 chars)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": " ", + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + } + }, + "required": [ + "I", + "OWNAME" + ], + "additionalProperties": false +} diff --git a/data/fnm/intermediate/schemas/switched_shunt.schema.json b/data/fnm/intermediate/schemas/switched_shunt.schema.json new file mode 100644 index 00000000..6858c792 --- /dev/null +++ b/data/fnm/intermediate/schemas/switched_shunt.schema.json @@ -0,0 +1,304 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "schemas/switched_shunt.schema.json", + "title": "Switched Shunt", + "description": "PSS/E v31 Switched Shunt record type. N1-N8 and B1-B8 define discrete switching step blocks.", + "type": "object", + "properties": { + "I": { + "type": "integer", + "description": "Bus number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "MODSW": { + "type": "integer", + "description": "Control mode (0-2)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0, + 2 + ] + }, + "ADJM": { + "type": "integer", + "description": "Adj method (0-1)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0, + 1 + ] + }, + "STAT": { + "type": "integer", + "description": "Status (1=in, 0=out)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0, + 1 + ] + }, + "VSWHI": { + "type": "number", + "description": "Ctrl voltage upper limit", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "bus_kv", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "VSWLO": { + "type": "number", + "description": "Ctrl voltage lower limit", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "bus_kv", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "SWREM": { + "type": "integer", + "description": "Remote bus (0=local)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RMPCT": { + "type": "number", + "description": "MVAR pct for remote reg", + "x-psse-unit": "%", + "x-psse-per-unit-base": "none", + "x-psse-default": 100.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RMIDNT": { + "type": "string", + "description": "Shunt name", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": "", + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "BINIT": { + "type": "number", + "description": "Initial susceptance", + "x-psse-unit": "MVAR", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "N1": { + "type": "integer", + "description": "Steps in block 1", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "B1": { + "type": "number", + "description": "Susceptance/step blk 1", + "x-psse-unit": "MVAR", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "N2": { + "type": "integer", + "description": "Steps in block 2", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "B2": { + "type": "number", + "description": "Susceptance/step blk 2", + "x-psse-unit": "MVAR", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "N3": { + "type": "integer", + "description": "Steps in block 3", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "B3": { + "type": "number", + "description": "Susceptance/step blk 3", + "x-psse-unit": "MVAR", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "N4": { + "type": "integer", + "description": "Steps in block 4", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "B4": { + "type": "number", + "description": "Susceptance/step blk 4", + "x-psse-unit": "MVAR", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "N5": { + "type": "integer", + "description": "Steps in block 5", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "B5": { + "type": "number", + "description": "Susceptance/step blk 5", + "x-psse-unit": "MVAR", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "N6": { + "type": "integer", + "description": "Steps in block 6", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "B6": { + "type": "number", + "description": "Susceptance/step blk 6", + "x-psse-unit": "MVAR", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "N7": { + "type": "integer", + "description": "Steps in block 7", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "B7": { + "type": "number", + "description": "Susceptance/step blk 7", + "x-psse-unit": "MVAR", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "N8": { + "type": "integer", + "description": "Steps in block 8", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "B8": { + "type": "number", + "description": "Susceptance/step blk 8", + "x-psse-unit": "MVAR", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + } + }, + "required": [ + "I", + "MODSW", + "STAT", + "VSWHI", + "VSWLO", + "SWREM", + "BINIT", + "N1", + "B1", + "N2", + "B2", + "N3", + "B3", + "N4", + "B4", + "N5", + "B5", + "N6", + "B6", + "N7", + "B7", + "N8", + "B8" + ], + "additionalProperties": false +} diff --git a/data/fnm/intermediate/schemas/transformer.schema.json b/data/fnm/intermediate/schemas/transformer.schema.json new file mode 100644 index 00000000..ab94a83f --- /dev/null +++ b/data/fnm/intermediate/schemas/transformer.schema.json @@ -0,0 +1,883 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "schemas/transformer.schema.json", + "title": "Transformer", + "description": "PSS/E v31 Transformer record type. Multi-line records flattened into one row. 2-winding (K=0) and 3-winding (K!=0) share the same schema.", + "type": "object", + "properties": { + "I": { + "type": "integer", + "description": "Winding 1 bus", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "J": { + "type": "integer", + "description": "Winding 2 bus", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "K": { + "type": "integer", + "description": "Winding 3 bus (0=2W)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "CKT": { + "type": "string", + "description": "Circuit identifier", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": "1 ", + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "CW": { + "type": "integer", + "description": "Winding data I/O code", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 1, + 3 + ] + }, + "CZ": { + "type": "integer", + "description": "Impedance data I/O code", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 1, + 3 + ] + }, + "CM": { + "type": "integer", + "description": "Mag admittance I/O code", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 1, + 2 + ] + }, + "MAG1": { + "type": "number", + "description": "Magnetizing conductance", + "x-psse-unit": "", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "MAG2": { + "type": "number", + "description": "Magnetizing susceptance", + "x-psse-unit": "", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "NMETR": { + "type": "integer", + "description": "Non-metered end code", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 2, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "NAME": { + "type": "string", + "description": "Transformer name", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": " ", + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "STAT": { + "type": "integer", + "description": "Status (0-4)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0, + 4 + ] + }, + "O1": { + "type": "integer", + "description": "Owner 1", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F1": { + "type": "number", + "description": "Fraction by owner 1", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0.0, + 1.0 + ] + }, + "O2": { + "type": "integer", + "description": "Owner 2", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F2": { + "type": "number", + "description": "Fraction by owner 2", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0.0, + 1.0 + ] + }, + "O3": { + "type": "integer", + "description": "Owner 3", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F3": { + "type": "number", + "description": "Fraction by owner 3", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0.0, + 1.0 + ] + }, + "O4": { + "type": "integer", + "description": "Owner 4", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F4": { + "type": "number", + "description": "Fraction by owner 4", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0.0, + 1.0 + ] + }, + "VECGRP": { + "type": "string", + "description": "Vector group (12 chars)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": " ", + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "R1_2": { + "type": "number", + "description": "R winding 1-2 (CZ dep)", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "X1_2": { + "type": "number", + "description": "X winding 1-2 (CZ dep)", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "mixed", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "SBASE1_2": { + "type": "number", + "description": "MVA base winding 1-2", + "x-psse-unit": "MVA", + "x-psse-per-unit-base": "none", + "x-psse-default": 100.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "R2_3": { + "type": "number", + "description": "R winding 2-3 (3W only)", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "X2_3": { + "type": "number", + "description": "X winding 2-3 (3W only)", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "SBASE2_3": { + "type": "number", + "description": "MVA base winding 2-3", + "x-psse-unit": "MVA", + "x-psse-per-unit-base": "none", + "x-psse-default": 100.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "R3_1": { + "type": "number", + "description": "R winding 3-1 (3W only)", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "X3_1": { + "type": "number", + "description": "X winding 3-1 (3W only)", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "SBASE3_1": { + "type": "number", + "description": "MVA base winding 3-1", + "x-psse-unit": "MVA", + "x-psse-per-unit-base": "none", + "x-psse-default": 100.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "VMSTAR": { + "type": "number", + "description": "Star bus voltage mag", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "bus_kv", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ANSTAR": { + "type": "number", + "description": "Star bus voltage angle", + "x-psse-unit": "deg", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "WINDV1": { + "type": "number", + "description": "Winding 1 turns ratio", + "x-psse-unit": "", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 1.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "NOMV1": { + "type": "number", + "description": "Winding 1 nominal kV", + "x-psse-unit": "kV", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ANG1": { + "type": "number", + "description": "Winding 1 phase shift", + "x-psse-unit": "deg", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RATA1": { + "type": "number", + "description": "Winding 1 rating A", + "x-psse-unit": "MVA", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RATB1": { + "type": "number", + "description": "Winding 1 rating B", + "x-psse-unit": "MVA", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RATC1": { + "type": "number", + "description": "Winding 1 rating C", + "x-psse-unit": "MVA", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "COD1": { + "type": "integer", + "description": "Winding 1 tap control", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "CONT1": { + "type": "integer", + "description": "Winding 1 ctrl bus", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RMA1": { + "type": "number", + "description": "Winding 1 upper tap limit", + "x-psse-unit": "", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 1.1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RMI1": { + "type": "number", + "description": "Winding 1 lower tap limit", + "x-psse-unit": "", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 0.9, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "VMA1": { + "type": "number", + "description": "Winding 1 upper V limit", + "x-psse-unit": "", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 1.1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "VMI1": { + "type": "number", + "description": "Winding 1 lower V limit", + "x-psse-unit": "", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 0.9, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "NTP1": { + "type": "integer", + "description": "Winding 1 tap positions", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 33, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "TAB1": { + "type": "integer", + "description": "Winding 1 impcor table", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "CR1": { + "type": "number", + "description": "Winding 1 LDC resistance", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "system_mva", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "CX1": { + "type": "number", + "description": "Winding 1 LDC reactance", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "system_mva", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "CNXA1": { + "type": "integer", + "description": "Winding 1 conn angle", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "WINDV2": { + "type": "number", + "description": "Winding 2 turns ratio", + "x-psse-unit": "", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 1.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "NOMV2": { + "type": "number", + "description": "Winding 2 nominal kV", + "x-psse-unit": "kV", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ANG2": { + "type": "number", + "description": "Winding 2 phase shift", + "x-psse-unit": "deg", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RATA2": { + "type": "number", + "description": "Winding 2 rating A", + "x-psse-unit": "MVA", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RATB2": { + "type": "number", + "description": "Winding 2 rating B", + "x-psse-unit": "MVA", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RATC2": { + "type": "number", + "description": "Winding 2 rating C", + "x-psse-unit": "MVA", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "COD2": { + "type": "integer", + "description": "Winding 2 tap control", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "CONT2": { + "type": "integer", + "description": "Winding 2 ctrl bus", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RMA2": { + "type": "number", + "description": "Winding 2 upper tap limit", + "x-psse-unit": "", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 1.1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RMI2": { + "type": "number", + "description": "Winding 2 lower tap limit", + "x-psse-unit": "", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 0.9, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "VMA2": { + "type": "number", + "description": "Winding 2 upper V limit", + "x-psse-unit": "", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 1.1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "VMI2": { + "type": "number", + "description": "Winding 2 lower V limit", + "x-psse-unit": "", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 0.9, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "NTP2": { + "type": "integer", + "description": "Winding 2 tap positions", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 33, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "TAB2": { + "type": "integer", + "description": "Winding 2 impcor table", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "CR2": { + "type": "number", + "description": "Winding 2 LDC resistance", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "system_mva", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "CX2": { + "type": "number", + "description": "Winding 2 LDC reactance", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "system_mva", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "CNXA2": { + "type": "integer", + "description": "Winding 2 conn angle", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "WINDV3": { + "type": "number", + "description": "Winding 3 turns ratio", + "x-psse-unit": "", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 1.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "NOMV3": { + "type": "number", + "description": "Winding 3 nominal kV", + "x-psse-unit": "kV", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ANG3": { + "type": "number", + "description": "Winding 3 phase shift", + "x-psse-unit": "deg", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RATA3": { + "type": "number", + "description": "Winding 3 rating A", + "x-psse-unit": "MVA", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": true, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RATB3": { + "type": "number", + "description": "Winding 3 rating B", + "x-psse-unit": "MVA", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RATC3": { + "type": "number", + "description": "Winding 3 rating C", + "x-psse-unit": "MVA", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "COD3": { + "type": "integer", + "description": "Winding 3 tap control", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "CONT3": { + "type": "integer", + "description": "Winding 3 ctrl bus", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RMA3": { + "type": "number", + "description": "Winding 3 upper tap limit", + "x-psse-unit": "", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 1.1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RMI3": { + "type": "number", + "description": "Winding 3 lower tap limit", + "x-psse-unit": "", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 0.9, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "VMA3": { + "type": "number", + "description": "Winding 3 upper V limit", + "x-psse-unit": "", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 1.1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "VMI3": { + "type": "number", + "description": "Winding 3 lower V limit", + "x-psse-unit": "", + "x-psse-per-unit-base": "mixed", + "x-psse-default": 0.9, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "NTP3": { + "type": "integer", + "description": "Winding 3 tap positions", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 33, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "TAB3": { + "type": "integer", + "description": "Winding 3 impcor table", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "CR3": { + "type": "number", + "description": "Winding 3 LDC resistance", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "system_mva", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "CX3": { + "type": "number", + "description": "Winding 3 LDC reactance", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "system_mva", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "CNXA3": { + "type": "integer", + "description": "Winding 3 conn angle", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + } + }, + "required": [ + "I", + "J", + "K", + "CKT", + "CW", + "CZ", + "CM", + "STAT", + "R1_2", + "X1_2", + "SBASE1_2", + "WINDV1", + "NOMV1", + "ANG1", + "RATA1", + "WINDV2", + "NOMV2" + ], + "additionalProperties": false +} diff --git a/data/fnm/intermediate/schemas/two_terminal_dc.schema.json b/data/fnm/intermediate/schemas/two_terminal_dc.schema.json new file mode 100644 index 00000000..c0551055 --- /dev/null +++ b/data/fnm/intermediate/schemas/two_terminal_dc.schema.json @@ -0,0 +1,484 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "schemas/two_terminal_dc.schema.json", + "title": "Two-Terminal DC", + "description": "PSS/E v31 Two-Terminal DC line record type.", + "type": "object", + "properties": { + "NAME": { + "type": "string", + "description": "DC line name", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "MDC": { + "type": "integer", + "description": "Control mode (0-2)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": [ + 0, + 2 + ] + }, + "RDC": { + "type": "number", + "description": "DC line resistance", + "x-psse-unit": "ohm", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "SETVL": { + "type": "number", + "description": "Current or power demand", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "VSCHD": { + "type": "number", + "description": "Scheduled DC voltage", + "x-psse-unit": "kV", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "VCMOD": { + "type": "number", + "description": "Mode switch DC voltage", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RCOMP": { + "type": "number", + "description": "Compounding resistance", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "DELTI": { + "type": "number", + "description": "Inverter firing angle margin", + "x-psse-unit": "deg", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "METER": { + "type": "string", + "description": "Metered end (R or I)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": "I", + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "DCVMIN": { + "type": "number", + "description": "Min DC voltage", + "x-psse-unit": "pu", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "CCCITMX": { + "type": "integer", + "description": "Max converter ctrl iters", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 20, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "CCCACC": { + "type": "number", + "description": "Converter ctrl accel factor", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "IPR": { + "type": "integer", + "description": "Rectifier bus", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "NBR": { + "type": "integer", + "description": "Rectifier bridges", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ANMXR": { + "type": "number", + "description": "Max rect firing angle", + "x-psse-unit": "deg", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ANMNR": { + "type": "number", + "description": "Min rect firing angle", + "x-psse-unit": "deg", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RCR": { + "type": "number", + "description": "Rect commutating R", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "XCR": { + "type": "number", + "description": "Rect commutating X", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "EBASR": { + "type": "number", + "description": "Rect primary base kV", + "x-psse-unit": "kV", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "TRR": { + "type": "number", + "description": "Rect xfmr ratio", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "TAPR": { + "type": "number", + "description": "Rect tap setting", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "TMXR": { + "type": "number", + "description": "Max rect tap", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1.5, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "TMNR": { + "type": "number", + "description": "Min rect tap", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.51, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "STPR": { + "type": "number", + "description": "Rect tap step", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.00625, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ICR": { + "type": "integer", + "description": "Rect firing angle ctrl bus", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "IFR": { + "type": "integer", + "description": "Rect commutating bus (from)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ITR": { + "type": "integer", + "description": "Rect commutating bus (to)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "IDR": { + "type": "string", + "description": "Rect circuit ID", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": "1 ", + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "XCAPR": { + "type": "number", + "description": "Rect capacitor reactance", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "IPI": { + "type": "integer", + "description": "Inverter bus", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "NBI": { + "type": "integer", + "description": "Inverter bridges", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ANMXI": { + "type": "number", + "description": "Max inv firing angle", + "x-psse-unit": "deg", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ANMNI": { + "type": "number", + "description": "Min inv firing angle", + "x-psse-unit": "deg", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RCI": { + "type": "number", + "description": "Inv commutating R", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "XCI": { + "type": "number", + "description": "Inv commutating X", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "EBASI": { + "type": "number", + "description": "Inv primary base kV", + "x-psse-unit": "kV", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "TRI": { + "type": "number", + "description": "Inv xfmr ratio", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "TAPI": { + "type": "number", + "description": "Inv tap setting", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "TMXI": { + "type": "number", + "description": "Max inv tap", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1.5, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "TMNI": { + "type": "number", + "description": "Min inv tap", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.51, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "STPI": { + "type": "number", + "description": "Inv tap step", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.00625, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ICI": { + "type": "integer", + "description": "Inv firing angle ctrl bus", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "IFI": { + "type": "integer", + "description": "Inv commutating bus (from)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ITI": { + "type": "integer", + "description": "Inv commutating bus (to)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "IDI": { + "type": "string", + "description": "Inv circuit ID", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": "1 ", + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "XCAPI": { + "type": "number", + "description": "Inv capacitor reactance", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + } + }, + "required": [ + "NAME", + "MDC", + "RDC", + "SETVL", + "VSCHD", + "IPR", + "NBR", + "IPI", + "NBI" + ], + "additionalProperties": false +} diff --git a/data/fnm/intermediate/schemas/vsc_dc.schema.json b/data/fnm/intermediate/schemas/vsc_dc.schema.json new file mode 100644 index 00000000..1662f878 --- /dev/null +++ b/data/fnm/intermediate/schemas/vsc_dc.schema.json @@ -0,0 +1,427 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "schemas/vsc_dc.schema.json", + "title": "VSC DC", + "description": "PSS/E v31 VSC DC line record type.", + "type": "object", + "properties": { + "NAME": { + "type": "string", + "description": "VSC DC line name", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "MDC": { + "type": "integer", + "description": "Control mode", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RDC": { + "type": "number", + "description": "DC line resistance", + "x-psse-unit": "ohm", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "O1": { + "type": "integer", + "description": "Owner 1", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F1": { + "type": "number", + "description": "Fraction by owner 1", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "O2": { + "type": "integer", + "description": "Owner 2", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F2": { + "type": "number", + "description": "Fraction by owner 2", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "O3": { + "type": "integer", + "description": "Owner 3", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F3": { + "type": "number", + "description": "Fraction by owner 3", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "O4": { + "type": "integer", + "description": "Owner 4", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "F4": { + "type": "number", + "description": "Fraction by owner 4", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "IBUS1": { + "type": "integer", + "description": "Converter 1 AC bus", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "TYPE1": { + "type": "integer", + "description": "Converter 1 type", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "MODE1": { + "type": "integer", + "description": "Converter 1 mode", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "DCSET1": { + "type": "number", + "description": "Converter 1 DC setpoint", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ACSET1": { + "type": "number", + "description": "Converter 1 AC setpoint", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ALOSS1": { + "type": "number", + "description": "Converter 1 loss A", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "BLOSS1": { + "type": "number", + "description": "Converter 1 loss B", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "MINLOSS1": { + "type": "number", + "description": "Converter 1 min loss", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "SMAX1": { + "type": "number", + "description": "Converter 1 MVA rating", + "x-psse-unit": "MVA", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "IMAX1": { + "type": "number", + "description": "Converter 1 current rating", + "x-psse-unit": "A", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "PWF1": { + "type": "number", + "description": "Converter 1 power weight", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "MAXQ1": { + "type": "number", + "description": "Converter 1 max Q", + "x-psse-unit": "MVAR", + "x-psse-per-unit-base": "none", + "x-psse-default": 9999.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "MINQ1": { + "type": "number", + "description": "Converter 1 min Q", + "x-psse-unit": "MVAR", + "x-psse-per-unit-base": "none", + "x-psse-default": -9999.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "REMOT1": { + "type": "integer", + "description": "Converter 1 remote bus", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RMPCT1": { + "type": "number", + "description": "Converter 1 MVAR pct", + "x-psse-unit": "%", + "x-psse-per-unit-base": "none", + "x-psse-default": 100.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "IBUS2": { + "type": "integer", + "description": "Converter 2 AC bus", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "TYPE2": { + "type": "integer", + "description": "Converter 2 type", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "MODE2": { + "type": "integer", + "description": "Converter 2 mode", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "DCSET2": { + "type": "number", + "description": "Converter 2 DC setpoint", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ACSET2": { + "type": "number", + "description": "Converter 2 AC setpoint", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ALOSS2": { + "type": "number", + "description": "Converter 2 loss A", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "BLOSS2": { + "type": "number", + "description": "Converter 2 loss B", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "MINLOSS2": { + "type": "number", + "description": "Converter 2 min loss", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "SMAX2": { + "type": "number", + "description": "Converter 2 MVA rating", + "x-psse-unit": "MVA", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "IMAX2": { + "type": "number", + "description": "Converter 2 current rating", + "x-psse-unit": "A", + "x-psse-per-unit-base": "none", + "x-psse-default": 0.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "PWF2": { + "type": "number", + "description": "Converter 2 power weight", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 1.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "MAXQ2": { + "type": "number", + "description": "Converter 2 max Q", + "x-psse-unit": "MVAR", + "x-psse-per-unit-base": "none", + "x-psse-default": 9999.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "MINQ2": { + "type": "number", + "description": "Converter 2 min Q", + "x-psse-unit": "MVAR", + "x-psse-per-unit-base": "none", + "x-psse-default": -9999.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "REMOT2": { + "type": "integer", + "description": "Converter 2 remote bus", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": 0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "RMPCT2": { + "type": "number", + "description": "Converter 2 MVAR pct", + "x-psse-unit": "%", + "x-psse-per-unit-base": "none", + "x-psse-default": 100.0, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + } + }, + "required": [ + "NAME", + "MDC", + "RDC", + "IBUS1", + "IBUS2" + ], + "additionalProperties": false +} diff --git a/data/fnm/intermediate/schemas/zone.schema.json b/data/fnm/intermediate/schemas/zone.schema.json new file mode 100644 index 00000000..0964b0b8 --- /dev/null +++ b/data/fnm/intermediate/schemas/zone.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "schemas/zone.schema.json", + "title": "Zone", + "description": "PSS/E v31 Zone record type.", + "type": "object", + "properties": { + "I": { + "type": "integer", + "description": "Zone number", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": null, + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + }, + "ZONAME": { + "type": "string", + "description": "Zone name (12 chars)", + "x-psse-unit": "", + "x-psse-per-unit-base": "none", + "x-psse-default": " ", + "x-psse-preservation-critical": false, + "x-psse-present-but-inactive": false, + "x-psse-valid-range": null + } + }, + "required": [ + "I", + "ZONAME" + ], + "additionalProperties": false +} diff --git a/data/fnm/manifest.json b/data/fnm/manifest.json new file mode 100644 index 00000000..810358a2 --- /dev/null +++ b/data/fnm/manifest.json @@ -0,0 +1,63 @@ +{ + "version": "1.0", + "variant": "FNM_ANNUAL_S01", + "source_files": [ + { + "file_name": "FNM_ANNUAL_S01.raw", + "file_type": "psse_raw", + "description": "PSS/E v31 RAW file containing the full network model", + "sha256": null, + "required": true + }, + { + "file_name": "bus_names.csv", + "file_type": "supplemental_csv", + "description": "Bus name mapping with station and voltage level metadata", + "sha256": null, + "required": true + }, + { + "file_name": "branch_ratings.csv", + "file_type": "supplemental_csv", + "description": "Branch thermal rating overrides and seasonal limits", + "sha256": null, + "required": true + }, + { + "file_name": "generator_costs.csv", + "file_type": "supplemental_csv", + "description": "Generator cost curves and fuel type classifications", + "sha256": null, + "required": true + }, + { + "file_name": "load_distribution.csv", + "file_type": "supplemental_csv", + "description": "Load distribution factors by weather zone and bus", + "sha256": null, + "required": true + }, + { + "file_name": "transformer_taps.csv", + "file_type": "supplemental_csv", + "description": "Transformer tap position settings and regulation bands", + "sha256": null, + "required": true + }, + { + "file_name": "shunt_switching.csv", + "file_type": "supplemental_csv", + "description": "Switched shunt device status and control parameters", + "sha256": null, + "required": true + }, + { + "file_name": "contingency_definitions.csv", + "file_type": "supplemental_csv", + "description": "Contingency definitions for N-1 and N-2 analysis", + "sha256": null, + "required": true + } + ], + "notes": "Default manifest for FNM ingestion. SHA-256 checksums are populated after first successful parse via update_manifest_checksums()." +} diff --git a/data/fnm/reference/README.md b/data/fnm/reference/README.md new file mode 100644 index 00000000..4d94dfc8 --- /dev/null +++ b/data/fnm/reference/README.md @@ -0,0 +1,16 @@ +# FNM Reference Directory + +This directory stores reference solutions and verification datasets used to validate +parser correctness (Phase 3). Reference data is compared against parser output to ensure +fidelity of the ingestion pipeline. + +## Contents (populated by later phases) + +- Known-good power flow solutions for test cases +- Reference bus/branch/generator data for spot-checking +- Verification datasets from independent sources + +## NDA Notice + +All files in this directory are derived from NDA-restricted FNM source data and must not +be committed to version control. The parent `.gitignore` blocks this entire directory. diff --git a/data/fnm/reference/pass_conditions.json b/data/fnm/reference/pass_conditions.json new file mode 100644 index 00000000..e4a331e8 --- /dev/null +++ b/data/fnm/reference/pass_conditions.json @@ -0,0 +1,196 @@ +{ + "$schema_version": "1.0.0", + "$description": "Pass condition definitions for ACPF and DCPF FNM verification. Machine-readable specification consumed by evaluate-tool agents at runtime.", + "bus_exclusion": { + "registry_path": "data/fnm/reference/excluded_buses.json", + "description": "Path to the D1 bus exclusion registry. All buses listed in this file are excluded from metric denominators. Exclusion reasons: ide_4_isolated, vm_zero_deenergized, disconnected_island.", + "usage": "Load excluded_buses[].bus_number to build the exclusion set. Metric denominators = total_buses - len(exclusion_set)." + }, + "acpf": { + "reference_dir": "data/fnm/reference/acpf/", + "reference_files": { + "buses": "buses_acpf.csv", + "branches": "branches_acpf.csv", + "generators": "generators_acpf.csv", + "summary": "summary_acpf.json" + }, + "aggregate": { + "description": "Primary pass gate. A bus passes if BOTH VM and VA deviations are within tolerance. The fraction of passing buses must exceed the minimum threshold.", + "min_passing_fraction": 0.95, + "vm_tolerance_pu": 0.005, + "va_tolerance_deg": 0.5, + "bus_pass_condition": "|VM_tool - VM_ref| < 0.005 AND |VA_tool - VA_ref| < 0.5", + "metric": "count(passing_buses) / count(non_excluded_buses) >= 0.95" + }, + "hard_fail": { + "description": "Any single condition triggers unconditional test failure, regardless of aggregate statistics.", + "conditions": [ + { + "name": "excessive_failing_fraction", + "description": "More than 20% of non-excluded buses fail the aggregate tolerance.", + "condition": "count(failing_buses) / count(non_excluded_buses) > 0.2", + "threshold": 0.2 + }, + { + "name": "extreme_vm_deviation", + "description": "Any single bus has VM deviation exceeding 0.1 p.u. Indicates fundamental voltage error, not solver variation.", + "condition": "max(|VM_tool - VM_ref|) > 0.1", + "threshold_pu": 0.1 + }, + { + "name": "extreme_va_deviation", + "description": "Any single bus has VA deviation exceeding 10.0 degrees. Indicates topology or connectivity error.", + "condition": "max(|VA_tool - VA_ref|) > 10.0", + "threshold_deg": 10.0 + } + ] + }, + "outlier_classification": { + "description": "Buses that fail the aggregate tolerance are classified by probable cause. Classification does not change pass/fail -- it explains why outliers exist and whether they indicate ingestion error vs. expected solver variation.", + "evaluation_order": "Rules are evaluated in the order listed. First matching rule assigns the primary cause. A bus may match multiple rules; only the first (highest priority) is assigned.", + "rules": [ + { + "priority": 1, + "cause": "switched_shunt", + "description": "Bus has a switched shunt device in the intermediate format. Discrete switching step differences between solvers produce VM deviations of 0.002-0.010 p.u. that are legitimate solver-variation artifacts.", + "match_condition": "has_switched_shunt(bus)", + "required_data": [ + "bus", + "switched_shunt" + ], + "applies_to": "acpf" + }, + { + "priority": 2, + "cause": "q_limit", + "description": "Bus has a generator at or near a reactive power limit (|Q - Qmax| < 1.0 MVAr or |Q - Qmin| < 1.0 MVAr in the ACPF reference). Different Q-limit enforcement sequences across solvers produce different voltage setpoints at PV-to-PQ transitioned buses.", + "match_condition": "generator_at_q_limit(bus, tolerance_mvar=1.0)", + "required_data": [ + "bus", + "generator" + ], + "applies_to": "acpf" + }, + { + "priority": 3, + "cause": "slack_distribution", + "description": "Bus is the slack bus (type=3) or within 2 branches of the slack bus in the network graph. Slack bus power absorption differs between solvers, causing VA deviations that propagate to electrically nearby buses.", + "match_condition": "is_slack_or_neighbor(bus, max_hops=2)", + "required_data": [ + "bus", + "branch" + ], + "applies_to": "both" + }, + { + "priority": 4, + "cause": "tap_position", + "description": "Bus is the regulated bus (CONT field) of an in-service tap-changing transformer. Different tap optimization algorithms produce different tap positions, causing VM deviations at the regulated bus.", + "match_condition": "is_tap_regulated_bus(bus)", + "required_data": [ + "bus", + "transformer" + ], + "applies_to": "acpf" + }, + { + "priority": 5, + "cause": "island_boundary", + "description": "Bus is at the boundary of a weakly connected subnetwork (network degree <= 2 and base_kv < 69 kV). Low-voltage radial boundary buses are highly sensitive to upstream modeling differences.", + "match_condition": "is_island_boundary(bus, max_degree=2, max_kv=69.0)", + "required_data": [ + "bus", + "branch" + ], + "applies_to": "both" + } + ], + "warning_thresholds": { + "max_classified_fraction": 0.1, + "max_classified_description": "If classified outliers (all causes except unclassified) exceed 10% of non-excluded buses, emit a warning.", + "max_unclassified_fraction": 0.02, + "max_unclassified_description": "If unclassified outliers exceed 2% of non-excluded buses, emit a warning." + } + }, + "formulation_difference_max_abs": { + "description": "Maximum absolute VM/VA deviation (p.u.) permitted under formulation_difference classification.", + "threshold_pu": null, + "unit": "pu" + } + }, + "dcpf": { + "reference_dir": "data/fnm/reference/dcpf/", + "reference_files": { + "buses": "buses_dcpf.csv", + "branches": "branches_dcpf.csv", + "summary": "summary_dcpf.json" + }, + "aggregate": { + "description": "Primary pass gate. Two independent metrics: bus angles and branch flows.", + "bus_angle": { + "description": "Fraction of non-excluded buses with VA deviation within tolerance.", + "min_passing_fraction": 0.95, + "va_tolerance_deg": 1.0, + "bus_pass_condition": "|VA_tool - VA_ref| < 1.0", + "metric": "count(passing_buses) / count(non_excluded_buses) >= 0.95" + }, + "branch_flow": { + "description": "Fraction of in-service branches with P deviation within tolerance.", + "min_passing_fraction": 0.9, + "p_tolerance_pct": 10.0, + "p_base_floor_mw": 1.0, + "deviation_formula": "|P_tool - P_ref| / max(|P_ref|, 1.0) * 100", + "branch_pass_condition": "deviation_pct < 10.0", + "metric": "count(passing_branches) / count(in_service_branches) >= 0.9" + } + }, + "hard_fail": { + "description": "Any single condition triggers unconditional test failure.", + "conditions": [ + { + "name": "excessive_bus_failing_fraction", + "description": "More than 20% of non-excluded buses fail the VA tolerance.", + "condition": "count(failing_buses) / count(non_excluded_buses) > 0.2", + "threshold": 0.2 + }, + { + "name": "excessive_branch_failing_fraction", + "description": "More than 20% of in-service branches fail the P tolerance.", + "condition": "count(failing_branches) / count(in_service_branches) > 0.2", + "threshold": 0.2 + }, + { + "name": "extreme_branch_flow_deviation", + "description": "Any single branch has P deviation exceeding 50.0%. Indicates topology or impedance error.", + "condition": "max(deviation_pct) > 50.0", + "threshold_pct": 50.0 + } + ] + }, + "formulation_difference_max_abs": { + "description": "Maximum absolute VA deviation (degrees) permitted under formulation_difference classification.", + "threshold_deg": null, + "unit": "degrees" + } + }, + "voltage_level_tiers": { + "description": "Informational voltage-level breakdown in verification results. Not a pass/fail gate -- the primary pass condition uses a single tolerance for all buses. This breakdown helps diagnose systematic voltage-level-correlated errors.", + "tiers": [ + { + "label": "transmission_230kv_plus", + "min_kv": 230.0, + "max_kv_exclusive": null + }, + { + "label": "subtransmission_69_to_229kv", + "min_kv": 69.0, + "max_kv_exclusive": 230.0 + }, + { + "label": "distribution_below_69kv", + "min_kv": 0.0, + "max_kv_exclusive": 69.0 + } + ] + } +} diff --git a/data/fnm/scripts/README.md b/data/fnm/scripts/README.md new file mode 100644 index 00000000..e372b211 --- /dev/null +++ b/data/fnm/scripts/README.md @@ -0,0 +1,15 @@ +# FNM Scripts Directory + +This directory contains Python modules for FNM data parsing, validation, and manifest +management. + +## Contents + +- `manifest_io.py` — Functions for loading, validating, and updating the FNM manifest +- `tests/` — Unit tests for all script modules + +## Later Phases + +- PSS/E RAW parser (Phase 1, D3) +- Supplemental CSV parsers (Phase 1, D4-D5) +- Validation scripts (Phase 1, D6) diff --git a/data/fnm/scripts/__init__.py b/data/fnm/scripts/__init__.py new file mode 100644 index 00000000..c2fb4209 --- /dev/null +++ b/data/fnm/scripts/__init__.py @@ -0,0 +1,605 @@ +from __future__ import annotations + +from fnm.scripts.acpf_reference import ( + ConvergenceInfo, + SolutionSource, + SolverSettings, + SystemSummary, + build_acpf_reference, + compute_system_summary, + determine_solution_source, + extract_branch_results, + extract_bus_results, + extract_generator_results, + read_snapshot_classification, + run_gridcal_acpf, + run_matpower_acpf, + write_branches_csv, + write_buses_csv, + write_generators_csv, + write_shunts_csv, + write_summary_json, + write_taps_csv, +) +from fnm.scripts.acpf_reference import main as acpf_reference_main +from fnm.scripts.bus_exclusion_registry import ( + BusExclusionRegistry, + ExcludedBusRecord, + ExclusionReason, + ExclusionSummary, + IslandSummary, + RegistryMetadata, + build_connectivity_graph, + build_excluded_bus_records, + build_exclusion_summary, + build_island_summaries, + build_registry, + find_connected_components, + find_disconnected_buses, + find_ide4_buses, + find_vm_zero_buses, + identify_main_island, + load_branch_table, + load_transformer_table, + registry_to_csv, + registry_to_dict, + registry_to_json, +) +from fnm.scripts.bus_exclusion_registry import ( + load_bus_table as exclusion_load_bus_table, +) +from fnm.scripts.bus_exclusion_registry import main as bus_exclusion_registry_main +from fnm.scripts.csv_join_keys import ( + CandidateKey, + CsvJoinMapping, + JoinCardinality, + JoinKeyReport, + JoinValidationResult, + KeyColumnPattern, + KeyType, + ReportMetadata, + ReportSummary, + analyze_csv, + build_join_key_report, + discover_candidate_keys, + get_default_key_patterns, + load_intermediate_key_values, + read_csv_header, + read_csv_key_values, + read_csv_sample, + validate_join, +) +from fnm.scripts.csv_join_keys import report_to_dict as csv_join_keys_report_to_dict +from fnm.scripts.csv_join_keys import report_to_markdown as csv_join_keys_report_to_markdown +from fnm.scripts.dcpf_acpf_characterization import ( + AggregateStats, + BranchDeviation, + BusDeviation, + CharacterizationResult, + ComplianceFractions, + DeviationCause, + annotate_branch_causes, + annotate_bus_causes, + build_characterization, + compute_aggregate_stats, + compute_branch_deviations, + compute_bus_deviations, + compute_compliance_fractions, + extract_worst_branches, + extract_worst_buses, + join_branches, + join_buses, + write_characterization_json, + write_characterization_md, +) +from fnm.scripts.dcpf_acpf_characterization import ( + load_acpf_branches as charac_load_acpf_branches, +) +from fnm.scripts.dcpf_acpf_characterization import ( + load_acpf_buses as charac_load_acpf_buses, +) +from fnm.scripts.dcpf_acpf_characterization import ( + load_dcpf_branches as charac_load_dcpf_branches, +) +from fnm.scripts.dcpf_acpf_characterization import ( + load_dcpf_buses as charac_load_dcpf_buses, +) +from fnm.scripts.dcpf_acpf_characterization import ( + load_intermediate_branches as charac_load_intermediate_branches, +) +from fnm.scripts.dcpf_acpf_characterization import ( + load_intermediate_buses as charac_load_intermediate_buses, +) +from fnm.scripts.dcpf_acpf_characterization import load_summary_json as charac_load_summary_json +from fnm.scripts.dcpf_acpf_characterization import main as dcpf_acpf_characterization_main +from fnm.scripts.dcpf_reference import ( + BMatrixResult, + BranchFlow, + DCPFSolution, + DCPFValidation, + build_b_matrix, + compute_branch_flows, + compute_bus_injections, + compute_phase_shift_injections, + filter_active_buses, + identify_slack_bus, + load_excluded_buses, + run_dcpf_reference, + solve_dcpf, + validate_dcpf_solution, +) +from fnm.scripts.dcpf_reference import BranchRecord as DcpfBranchRecord +from fnm.scripts.dcpf_reference import BusRecord as DcpfBusRecord +from fnm.scripts.dcpf_reference import GeneratorRecord as DcpfGeneratorRecord +from fnm.scripts.dcpf_reference import load_branch_table as dcpf_load_branch_table +from fnm.scripts.dcpf_reference import load_bus_table as dcpf_load_bus_table +from fnm.scripts.dcpf_reference import load_generator_table as dcpf_load_generator_table +from fnm.scripts.dcpf_reference import main as dcpf_reference_main +from fnm.scripts.dcpf_reference import write_branches_csv as dcpf_write_branches_csv +from fnm.scripts.dcpf_reference import write_buses_csv as dcpf_write_buses_csv +from fnm.scripts.dcpf_reference import write_summary_json as dcpf_write_summary_json +from fnm.scripts.fnm_gating import ( + FnmFileCheck, + FnmFileStatus, + FnmPathResult, + FnmPathStatus, + find_repo_root, + load_fnm_manifest, + resolve_fnm_path, +) +from fnm.scripts.fnm_gating_cli import cli_validate_fnm_path +from fnm.scripts.fnm_gating_fixtures import require_fnm, require_fnm_csvs, require_fnm_raw +from fnm.scripts.generate_schema_reference import ( + generate_document, +) +from fnm.scripts.generate_schema_reference import main as generate_schema_reference_main +from fnm.scripts.gridcal_parser import ( + GRIDCAL_ELEMENT_COLLECTIONS, + PSSE_TO_GRIDCAL_MAPPING, + GridCalParserSummary, + MultiCircuitCounts, + ParserLog, + ParserLogEntry, + PsseIntermediateCounts, + RecordTypeMapping, + build_record_type_mapping, +) +from fnm.scripts.intermediate_schema import ( + ConformanceFinding, + ConformanceLevel, + ConformanceReport, + FieldSpec, + IntermediateFormatManifest, + ManifestEntry, + PerUnitBase, + TableSchema, + detect_inactive_fields, + generate_reference_markdown, + get_table_schemas, + load_schema, + manifest_to_json_schema, + table_schema_to_json_schema, + validate_tables, + write_schemas, +) +from fnm.scripts.intermediate_schema import report_to_dict as intermediate_report_to_dict +from fnm.scripts.matpower_parser import ( + KnownLimitation, + MatpowerParserLog, + MatpowerParserSummary, + ParserWarning, + SectionCountMap, + build_known_limitations, + build_octave_command, + find_matpower_path, + log_to_dict, + parse_octave_stdout, + parse_octave_warnings, + read_csv_field_counts, + run_psse2mpc, +) +from fnm.scripts.matpower_parser import summary_to_dict as matpower_summary_to_dict +from fnm.scripts.parser_comparison import ( + CanonicalParserSelection, + ComparisonMetadata, + DataLossEntry, + DiscrepancyType, + FidelityScore, + FieldCoverageEntry, + ParserComparisonReport, + ParserName, + RecordCountComparison, + SelectionRationale, + build_comparison_report, + compare_field_coverage, + compare_record_counts, + compute_fidelity_score, + report_to_dict, + report_to_markdown, + select_canonical_parser, +) +from fnm.scripts.pass_conditions import ( + DEFAULT_OUTLIER_RULES, + DEFAULT_VOLTAGE_TIERS, + OUTLIER_PRIORITY, + ACPFAggregateThresholds, + ACPFHardFailThresholds, + DCPFAggregateThresholds, + DCPFHardFailThresholds, + HardFailResult, + MetricResult, + OutlierCause, + OutlierClassificationConfig, + OutlierRule, + OutlierSummary, + PassConditionSpec, + VerificationVerdict, + VoltageLevelBreakdown, + VoltageLevelTier, + build_pass_condition_spec, + classify_outlier_bus, + evaluate_acpf, + evaluate_dcpf, + generate_pass_conditions, + load_spec, + spec_to_dict, + write_json, + write_markdown, +) +from fnm.scripts.pass_conditions import main as pass_conditions_main +from fnm.scripts.raw_record_counter import ( + PSSE_V31_SECTION_NAMES, + HeaderInfo, + RecordCountSummary, + count_raw_records, + count_section_records, + parse_header, + summary_to_dict, +) +from fnm.scripts.solved_snapshot import ( + ConfirmationMetadata, + DistributionStats, + GeneratorQgStats, + IndicatorResult, + IndicatorSignal, + SnapshotClassification, + SnapshotConfirmation, + build_confirmation, + classify_overall, + classify_qg, + classify_va, + classify_vm, + compute_distribution_stats, + compute_qg_stats, + confirmation_to_dict, + confirmation_to_markdown, + derive_phase3_implications, + load_bus_data, + load_generator_data, +) +from fnm.scripts.validation_report import ( + CheckResult, + CheckStatus, + ValidationReport, + build_validation_report, + check_acpf_generator_limits, + check_acpf_kcl, + check_acpf_power_balance, + check_acpf_vm_plausibility, + check_dcpf_flow_angle_consistency, + check_dcpf_power_balance, + check_dcpf_slack_angle, + load_acpf_generators, + load_acpf_summary, + load_dcpf_summary, + load_intermediate_generators, + run_validation, + write_report_json, + write_report_markdown, +) +from fnm.scripts.validation_report import ( + ReportSummary as ValidationReportSummary, +) +from fnm.scripts.validation_report import ( + load_acpf_branches as validation_load_acpf_branches, +) +from fnm.scripts.validation_report import ( + load_acpf_buses as validation_load_acpf_buses, +) +from fnm.scripts.validation_report import ( + load_dcpf_branches as validation_load_dcpf_branches, +) +from fnm.scripts.validation_report import ( + load_dcpf_buses as validation_load_dcpf_buses, +) +from fnm.scripts.validation_report import ( + load_excluded_buses as validation_load_excluded_buses, +) +from fnm.scripts.validation_report import ( + load_intermediate_branches as validation_load_intermediate_branches, +) +from fnm.scripts.validation_report import ( + load_intermediate_buses as validation_load_intermediate_buses, +) +from fnm.scripts.validation_report import main as validation_report_main + +__all__ = [ + "FnmFileCheck", + "FnmFileStatus", + "FnmPathResult", + "FnmPathStatus", + "cli_validate_fnm_path", + "find_repo_root", + "load_fnm_manifest", + "require_fnm", + "require_fnm_csvs", + "require_fnm_raw", + "resolve_fnm_path", + # matpower_parser + "KnownLimitation", + "MatpowerParserLog", + "MatpowerParserSummary", + "ParserWarning", + "SectionCountMap", + "build_known_limitations", + "build_octave_command", + "find_matpower_path", + "log_to_dict", + "matpower_summary_to_dict", + "parse_octave_stdout", + "parse_octave_warnings", + "read_csv_field_counts", + "run_psse2mpc", + # gridcal_parser + "GRIDCAL_ELEMENT_COLLECTIONS", + "GridCalParserSummary", + "MultiCircuitCounts", + "PSSE_TO_GRIDCAL_MAPPING", + "ParserLog", + "ParserLogEntry", + "PsseIntermediateCounts", + "RecordTypeMapping", + "build_record_type_mapping", + # parser_comparison + "CanonicalParserSelection", + "ComparisonMetadata", + "DataLossEntry", + "DiscrepancyType", + "FieldCoverageEntry", + "FidelityScore", + "ParserComparisonReport", + "ParserName", + "RecordCountComparison", + "SelectionRationale", + "build_comparison_report", + "compare_field_coverage", + "compare_record_counts", + "compute_fidelity_score", + "report_to_dict", + "report_to_markdown", + "select_canonical_parser", + # intermediate_schema + "ConformanceFinding", + "ConformanceLevel", + "ConformanceReport", + "FieldSpec", + "IntermediateFormatManifest", + "ManifestEntry", + "PerUnitBase", + "TableSchema", + "detect_inactive_fields", + "generate_reference_markdown", + "get_table_schemas", + "intermediate_report_to_dict", + "load_schema", + "manifest_to_json_schema", + "table_schema_to_json_schema", + "validate_tables", + "write_schemas", + # solved_snapshot + "ConfirmationMetadata", + "DistributionStats", + "GeneratorQgStats", + "IndicatorResult", + "IndicatorSignal", + "SnapshotClassification", + "SnapshotConfirmation", + "build_confirmation", + "classify_overall", + "classify_qg", + "classify_va", + "classify_vm", + "compute_distribution_stats", + "compute_qg_stats", + "confirmation_to_dict", + "confirmation_to_markdown", + "derive_phase3_implications", + "load_bus_data", + "load_generator_data", + # csv_join_keys + "CandidateKey", + "CsvJoinMapping", + "JoinCardinality", + "JoinKeyReport", + "JoinValidationResult", + "KeyColumnPattern", + "KeyType", + "ReportMetadata", + "ReportSummary", + "analyze_csv", + "build_join_key_report", + "csv_join_keys_report_to_dict", + "csv_join_keys_report_to_markdown", + "discover_candidate_keys", + "get_default_key_patterns", + "load_intermediate_key_values", + "read_csv_header", + "read_csv_key_values", + "read_csv_sample", + "validate_join", + # raw_record_counter + "HeaderInfo", + "PSSE_V31_SECTION_NAMES", + "RecordCountSummary", + "count_raw_records", + "count_section_records", + "parse_header", + "summary_to_dict", + # generate_schema_reference + "generate_document", + "generate_schema_reference_main", + # bus_exclusion_registry + "BusExclusionRegistry", + "ExcludedBusRecord", + "ExclusionReason", + "ExclusionSummary", + "IslandSummary", + "RegistryMetadata", + "build_connectivity_graph", + "build_excluded_bus_records", + "build_exclusion_summary", + "build_island_summaries", + "build_registry", + "bus_exclusion_registry_main", + "exclusion_load_bus_table", + "find_connected_components", + "find_disconnected_buses", + "find_ide4_buses", + "find_vm_zero_buses", + "identify_main_island", + "load_branch_table", + "load_transformer_table", + "registry_to_csv", + "registry_to_dict", + "registry_to_json", + # acpf_reference + "ConvergenceInfo", + "SolutionSource", + "SolverSettings", + "SystemSummary", + "acpf_reference_main", + "build_acpf_reference", + "compute_system_summary", + "determine_solution_source", + "extract_branch_results", + "extract_bus_results", + "extract_generator_results", + "read_snapshot_classification", + "run_gridcal_acpf", + "run_matpower_acpf", + "write_branches_csv", + "write_buses_csv", + "write_generators_csv", + "write_shunts_csv", + "write_summary_json", + "write_taps_csv", + # dcpf_reference + "BMatrixResult", + "BranchFlow", + "DCPFSolution", + "DCPFValidation", + "DcpfBranchRecord", + "DcpfBusRecord", + "DcpfGeneratorRecord", + "build_b_matrix", + "compute_branch_flows", + "compute_bus_injections", + "compute_phase_shift_injections", + "dcpf_load_branch_table", + "dcpf_load_bus_table", + "dcpf_load_generator_table", + "dcpf_reference_main", + "dcpf_write_branches_csv", + "dcpf_write_buses_csv", + "dcpf_write_summary_json", + "filter_active_buses", + "identify_slack_bus", + "load_excluded_buses", + "run_dcpf_reference", + "solve_dcpf", + "validate_dcpf_solution", + # dcpf_acpf_characterization + "AggregateStats", + "BranchDeviation", + "BusDeviation", + "CharacterizationResult", + "ComplianceFractions", + "DeviationCause", + "annotate_branch_causes", + "annotate_bus_causes", + "build_characterization", + "charac_load_acpf_branches", + "charac_load_acpf_buses", + "charac_load_dcpf_branches", + "charac_load_dcpf_buses", + "charac_load_intermediate_branches", + "charac_load_intermediate_buses", + "charac_load_summary_json", + "compute_aggregate_stats", + "compute_branch_deviations", + "compute_bus_deviations", + "compute_compliance_fractions", + "dcpf_acpf_characterization_main", + "extract_worst_branches", + "extract_worst_buses", + "join_branches", + "join_buses", + "write_characterization_json", + "write_characterization_md", + # validation_report + "CheckResult", + "CheckStatus", + "ValidationReport", + "ValidationReportSummary", + "build_validation_report", + "check_acpf_generator_limits", + "check_acpf_kcl", + "check_acpf_power_balance", + "check_acpf_vm_plausibility", + "check_dcpf_flow_angle_consistency", + "check_dcpf_power_balance", + "check_dcpf_slack_angle", + "load_acpf_generators", + "load_acpf_summary", + "load_dcpf_summary", + "load_intermediate_generators", + "run_validation", + "validation_load_acpf_branches", + "validation_load_acpf_buses", + "validation_load_dcpf_branches", + "validation_load_dcpf_buses", + "validation_load_excluded_buses", + "validation_load_intermediate_branches", + "validation_load_intermediate_buses", + "validation_report_main", + "write_report_json", + "write_report_markdown", + # pass_conditions + "ACPFAggregateThresholds", + "ACPFHardFailThresholds", + "DCPFAggregateThresholds", + "DCPFHardFailThresholds", + "DEFAULT_OUTLIER_RULES", + "DEFAULT_VOLTAGE_TIERS", + "HardFailResult", + "MetricResult", + "OUTLIER_PRIORITY", + "OutlierCause", + "OutlierClassificationConfig", + "OutlierRule", + "OutlierSummary", + "PassConditionSpec", + "VerificationVerdict", + "VoltageLevelBreakdown", + "VoltageLevelTier", + "build_pass_condition_spec", + "classify_outlier_bus", + "evaluate_acpf", + "evaluate_dcpf", + "generate_pass_conditions", + "load_spec", + "pass_conditions_main", + "spec_to_dict", + "write_json", + "write_markdown", +] diff --git a/data/fnm/scripts/acpf_reference.py b/data/fnm/scripts/acpf_reference.py new file mode 100644 index 00000000..995f036e --- /dev/null +++ b/data/fnm/scripts/acpf_reference.py @@ -0,0 +1,1366 @@ +"""ACPF Reference Solution Extraction for FNM Annual S01. + +Produces the AC Power Flow (ACPF) reference solution dataset, handling two +mutually exclusive paths based on the D8 snapshot classification: + +- **Solved-case path (Path A):** Extracts VM, VA, P/Q flows, and generator P/Q + directly from the canonical parser's intermediate format tables. +- **Flat-start path (Path B):** Runs a verified solver (MATPOWER ``runpf`` via + Octave or GridCal Newton-Raphson) on the intermediate format data to produce + a converged ACPF solution. + +Both paths produce identical output CSV schemas and a metadata JSON documenting +the solution source, solver settings (if applicable), convergence status, and +system-level summary statistics. + +Output directory: ``data/fnm/reference/acpf/`` +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import Enum +from pathlib import Path + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +# MATPOWER bus matrix column indices (standard 13-column format, no header) +_MPC_BUS_COL_BUS_I: int = 0 +_MPC_BUS_COL_TYPE: int = 1 +_MPC_BUS_COL_PD: int = 2 +_MPC_BUS_COL_QD: int = 3 +_MPC_BUS_COL_VM: int = 7 +_MPC_BUS_COL_VA: int = 8 + +# MATPOWER gen matrix column indices (no header) +_MPC_GEN_COL_BUS: int = 0 +_MPC_GEN_COL_PG: int = 1 +_MPC_GEN_COL_QG: int = 2 +_MPC_GEN_COL_QMAX: int = 3 +_MPC_GEN_COL_QMIN: int = 4 +_MPC_GEN_COL_STATUS: int = 7 +_MPC_GEN_COL_PMAX: int = 8 + +# MATPOWER branch matrix column indices (no header) +_MPC_BRANCH_COL_FBUS: int = 0 +_MPC_BRANCH_COL_TBUS: int = 1 +_MPC_BRANCH_COL_STATUS: int = 10 +# Flow columns (present only after runpf) +_MPC_BRANCH_COL_PF: int = 13 +_MPC_BRANCH_COL_QF: int = 14 +_MPC_BRANCH_COL_PT: int = 15 +_MPC_BRANCH_COL_QT: int = 16 + +_ISOLATED_BUS_TYPE: int = 4 +_SLACK_BUS_TYPE: int = 3 + +_POWER_BALANCE_WARN_THRESHOLD_MW: float = 1.0 + + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + + +class SolutionSource(Enum): + """How the ACPF reference solution was obtained.""" + + EXTRACTED = "extracted" + """Extracted directly from a converged solved case in the intermediate format.""" + + COMPUTED = "computed" + """Computed by running a solver on the intermediate format data.""" + + +@dataclass(frozen=True) +class SolverSettings: + """Solver configuration for the flat-start path. + + All fields are None when solution_source is EXTRACTED. + """ + + name: str | None = None + """Solver identifier: 'runpf' for MATPOWER, 'gridcal_nr' for GridCal.""" + + version: str | None = None + """Solver version string.""" + + tolerance: float | None = None + """Newton-Raphson convergence tolerance (p.u. mismatch).""" + + max_iterations: int | None = None + """Maximum NR iterations.""" + + q_limits_enforced: bool | None = None + """Whether Q-limits were enforced in the final solution.""" + + q_limit_strategy: str | None = None + """Description of Q-limit enforcement approach.""" + + enforce_area_interchange: bool | None = None + """Whether area interchange control was active.""" + + +@dataclass(frozen=True) +class ConvergenceInfo: + """Solver convergence details for the flat-start path. + + All fields are None when solution_source is EXTRACTED. + """ + + converged: bool | None = None + """Whether the solver reached the convergence tolerance.""" + + iterations: int | None = None + """Number of NR iterations performed.""" + + final_mismatch_mw: float | None = None + """Largest active power mismatch at convergence (MW).""" + + final_mismatch_mvar: float | None = None + """Largest reactive power mismatch at convergence (MVAr).""" + + +@dataclass(frozen=True) +class SystemSummary: + """System-level aggregate quantities from the ACPF solution.""" + + total_gen_mw: float + """Total active power generation (MW).""" + + total_gen_mvar: float + """Total reactive power generation (MVAr).""" + + total_load_mw: float + """Total active power load (MW).""" + + total_load_mvar: float + """Total reactive power load (MVAr).""" + + total_loss_mw: float + """Total active power losses (MW). Equals total_gen_mw - total_load_mw.""" + + total_loss_mvar: float + """Total reactive power losses (MVAr).""" + + slack_bus: int + """Slack bus number (bus type 3).""" + + power_balance_residual_mw: float + """Residual: total_gen_mw - total_load_mw - total_loss_mw. + Should be ~0 for a consistent solution.""" + + +# --------------------------------------------------------------------------- +# CSV detection helpers (reuses logic patterns from solved_snapshot.py) +# --------------------------------------------------------------------------- + + +def _is_header_row(row: list[str]) -> bool: + """Determine if a CSV row is a header (non-numeric first field).""" + if not row: + return False + try: + float(row[0]) + return False + except ValueError: + return True + + +def _detect_column_index(headers: list[str], candidates: list[str]) -> int | None: + """Find the index of the first matching header from a list of candidates.""" + lower_headers = [h.strip().lower() for h in headers] + for candidate in candidates: + if candidate.lower() in lower_headers: + return lower_headers.index(candidate.lower()) + return None + + +# --------------------------------------------------------------------------- +# Path selection +# --------------------------------------------------------------------------- + + +def read_snapshot_classification(snapshot_json_path: Path) -> str: + """Read the overall classification from the D8 snapshot confirmation JSON. + + Args: + snapshot_json_path: Path to ``snapshot_confirmation.json`` from D8. + + Returns: + The classification string: ``'solved'``, ``'flat_start'``, or + ``'indeterminate'``. + + Raises: + FileNotFoundError: If the JSON file does not exist. + KeyError: If the ``classification`` field is missing. + ValueError: If the classification value is not one of the three + expected strings. + """ + if not snapshot_json_path.exists(): + raise FileNotFoundError(f"Snapshot JSON not found: {snapshot_json_path}") + + with open(snapshot_json_path, encoding="utf-8") as f: + data = json.load(f) + + if "classification" not in data: + raise KeyError( + f"Snapshot JSON missing 'classification' field. Keys found: {list(data.keys())}" + ) + + classification = data["classification"] + valid_values = {"solved", "flat_start", "indeterminate"} + if classification not in valid_values: + raise ValueError( + f"Invalid classification value '{classification}'. Expected one of: {valid_values}" + ) + + return classification + + +def determine_solution_source(classification: str) -> SolutionSource: + """Map a D8 classification to the solution source for this deliverable. + + Args: + classification: One of ``'solved'``, ``'flat_start'``, ``'indeterminate'``. + + Returns: + ``SolutionSource.EXTRACTED`` for ``'solved'``, + ``SolutionSource.COMPUTED`` for ``'flat_start'``. + + Raises: + ValueError: If classification is ``'indeterminate'``. The ACPF + reference cannot be produced until the indeterminate case + is manually resolved. + """ + if classification == "solved": + return SolutionSource.EXTRACTED + if classification == "flat_start": + return SolutionSource.COMPUTED + if classification == "indeterminate": + raise ValueError( + "Snapshot classification is 'indeterminate'. " + "The ACPF reference solution cannot be produced until the " + "indeterminate case is manually resolved (OQ-E01)." + ) + raise ValueError(f"Unknown classification: '{classification}'") + + +# --------------------------------------------------------------------------- +# Solved-case extraction (Path A) +# --------------------------------------------------------------------------- + + +def _load_bus_csv_raw( + bus_csv_path: Path, +) -> tuple[list[str] | None, list[list[str]]]: + """Load a bus CSV and return (headers_or_None, data_rows).""" + if not bus_csv_path.exists(): + raise FileNotFoundError(f"Bus CSV not found: {bus_csv_path}") + + with open(bus_csv_path, encoding="utf-8") as f: + reader = csv.reader(f) + rows = list(reader) + + if not rows: + raise ValueError(f"Bus CSV is empty: {bus_csv_path}") + + first_row = rows[0] + if _is_header_row(first_row): + return first_row, rows[1:] + return None, rows + + +def _resolve_bus_col_indices( + headers: list[str] | None, +) -> tuple[int, int, int, int, int, int]: + """Return (bus_idx, type_idx, vm_idx, va_idx, pd_idx, qd_idx) for the bus CSV.""" + if headers is not None: + bus_idx = _detect_column_index(headers, ["bus_i", "i", "bus_id", "bus"]) + type_idx = _detect_column_index(headers, ["type", "ide", "bus_type"]) + vm_idx = _detect_column_index(headers, ["vm", "vm_pu", "Vm"]) + va_idx = _detect_column_index(headers, ["va", "va_deg", "Va"]) + pd_idx = _detect_column_index(headers, ["pd", "Pd", "PD"]) + qd_idx = _detect_column_index(headers, ["qd", "Qd", "QD"]) + if bus_idx is None: + raise ValueError(f"Cannot find bus number column in headers: {headers}") + if vm_idx is None or va_idx is None: + raise ValueError(f"Cannot find VM/VA columns in headers: {headers}") + if type_idx is None: + raise ValueError(f"Cannot find bus type column in headers: {headers}") + if pd_idx is None or qd_idx is None: + raise ValueError(f"Cannot find PD/QD columns in headers: {headers}") + return bus_idx, type_idx, vm_idx, va_idx, pd_idx, qd_idx + else: + return ( + _MPC_BUS_COL_BUS_I, + _MPC_BUS_COL_TYPE, + _MPC_BUS_COL_VM, + _MPC_BUS_COL_VA, + _MPC_BUS_COL_PD, + _MPC_BUS_COL_QD, + ) + + +def extract_bus_results(bus_csv_path: Path) -> list[dict]: + """Extract per-bus VM and VA from the canonical parser's bus CSV. + + Reads the intermediate format bus table. Excludes isolated buses + (IDE/type = 4) and de-energized buses (VM = 0). Auto-detects column + names from MATPOWER and GridCal conventions (same logic as D8's + ``load_bus_data``). + + Args: + bus_csv_path: Path to the intermediate format bus CSV. + + Returns: + List of dicts with keys ``bus`` (int), ``VM`` (float), ``VA`` (float). + Sorted by bus number ascending. + + Raises: + FileNotFoundError: If the CSV does not exist. + ValueError: If required columns (bus number, VM, VA, bus type) + cannot be identified. + """ + headers, data_rows = _load_bus_csv_raw(bus_csv_path) + bus_idx, type_idx, vm_idx, va_idx, _pd_idx, _qd_idx = _resolve_bus_col_indices(headers) + + results: list[dict] = [] + for row in data_rows: + if not row or all(cell.strip() == "" for cell in row): + continue + + try: + bus_type = int(float(row[type_idx].strip())) + except (ValueError, IndexError): + bus_type = 0 + + # Exclude isolated buses (type 4) + if bus_type == _ISOLATED_BUS_TYPE: + continue + + try: + vm = float(row[vm_idx].strip()) + va = float(row[va_idx].strip()) + bus_num = int(float(row[bus_idx].strip())) + except (ValueError, IndexError) as exc: + raise ValueError(f"Cannot parse bus data from row: {row}") from exc + + # Exclude de-energized buses (VM = 0) + if vm == 0.0: + continue + + results.append({"bus": bus_num, "VM": vm, "VA": va}) + + results.sort(key=lambda r: r["bus"]) + return results + + +def extract_branch_results(branch_csv_path: Path) -> list[dict]: + """Extract per-branch P/Q flows from the canonical parser's branch CSV. + + Reads the intermediate format branch table. Includes only in-service + branches (status = 1). Extracts from-end and to-end active and reactive + power flows. + + For the solved-case path, flow values must already be present in the + intermediate format — they were part of the converged PSS/E solution. + If flow columns are absent (indicating the parser did not extract them), + raises ValueError with a diagnostic message. + + Args: + branch_csv_path: Path to the intermediate format branch CSV. + + Returns: + List of dicts with keys ``from_bus`` (int), ``to_bus`` (int), + ``ckt`` (str), ``P_from`` (float), ``Q_from`` (float), + ``P_to`` (float), ``Q_to`` (float). Sorted by (from_bus, to_bus, ckt). + + Raises: + FileNotFoundError: If the CSV does not exist. + ValueError: If required columns cannot be identified or flow + columns are absent. + """ + if not branch_csv_path.exists(): + raise FileNotFoundError(f"Branch CSV not found: {branch_csv_path}") + + with open(branch_csv_path, encoding="utf-8") as f: + reader = csv.reader(f) + rows = list(reader) + + if not rows: + raise ValueError(f"Branch CSV is empty: {branch_csv_path}") + + first_row = rows[0] + if _is_header_row(first_row): + headers = first_row + data_rows = rows[1:] + fbus_idx = _detect_column_index(headers, ["fbus", "f_bus", "from_bus", "i"]) + tbus_idx = _detect_column_index(headers, ["tbus", "t_bus", "to_bus", "j"]) + ckt_idx = _detect_column_index(headers, ["ckt", "circuit", "cid"]) + status_idx = _detect_column_index(headers, ["status", "st", "br_status"]) + pf_idx = _detect_column_index(headers, ["pf", "p_from", "P_from"]) + qf_idx = _detect_column_index(headers, ["qf", "q_from", "Q_from"]) + pt_idx = _detect_column_index(headers, ["pt", "p_to", "P_to"]) + qt_idx = _detect_column_index(headers, ["qt", "q_to", "Q_to"]) + + if fbus_idx is None or tbus_idx is None: + raise ValueError(f"Cannot find from/to bus columns in headers: {headers}") + if pf_idx is None or qf_idx is None or pt_idx is None or qt_idx is None: + raise ValueError( + "Branch CSV is missing flow columns (PF, QF, PT, QT). " + "The solved-case path requires pre-computed flows in the intermediate format. " + "If the parser did not extract flows, consider using the flat-start path or " + "a different parser." + ) + else: + data_rows = rows + fbus_idx = _MPC_BRANCH_COL_FBUS + tbus_idx = _MPC_BRANCH_COL_TBUS + ckt_idx = None # MATPOWER headerless format has no ckt column + status_idx = _MPC_BRANCH_COL_STATUS + pf_idx = _MPC_BRANCH_COL_PF + qf_idx = _MPC_BRANCH_COL_QF + pt_idx = _MPC_BRANCH_COL_PT + qt_idx = _MPC_BRANCH_COL_QT + + # Verify flow columns have data (check width of first data row) + if data_rows: + first_data = data_rows[0] + max_flow_idx = max(pf_idx, qf_idx, pt_idx, qt_idx) + if len(first_data) <= max_flow_idx: + raise ValueError( + f"Branch CSV has {len(first_data)} columns but flow columns require " + f"index {max_flow_idx}. The parser may not have extracted branch flows. " + "Consider using the flat-start path to compute flows from the solved " + "voltage profile." + ) + + results: list[dict] = [] + for row in data_rows: + if not row or all(cell.strip() == "" for cell in row): + continue + + # Check status + if status_idx is not None and status_idx < len(row): + try: + status = int(float(row[status_idx].strip())) + except (ValueError, IndexError): + status = 1 + if status != 1: + continue + + try: + from_bus = int(float(row[fbus_idx].strip())) + to_bus = int(float(row[tbus_idx].strip())) + ckt = row[ckt_idx].strip() if ckt_idx is not None and ckt_idx < len(row) else "1" + p_from = float(row[pf_idx].strip()) + q_from = float(row[qf_idx].strip()) + p_to = float(row[pt_idx].strip()) + q_to = float(row[qt_idx].strip()) + except (ValueError, IndexError) as exc: + raise ValueError(f"Cannot parse branch data from row: {row}") from exc + + results.append( + { + "from_bus": from_bus, + "to_bus": to_bus, + "ckt": ckt, + "P_from": p_from, + "Q_from": q_from, + "P_to": p_to, + "Q_to": q_to, + } + ) + + results.sort(key=lambda r: (r["from_bus"], r["to_bus"], r["ckt"])) + return results + + +def extract_generator_results(gen_csv_path: Path) -> list[dict]: + """Extract per-generator P and Q from the canonical parser's generator CSV. + + Reads the intermediate format generator table. Includes only in-service + generators (status = 1). Extracts active and reactive power output. + + Args: + gen_csv_path: Path to the intermediate format generator CSV. + + Returns: + List of dicts with keys ``bus`` (int), ``machine_id`` (str), + ``P`` (float), ``Q`` (float). Sorted by (bus, machine_id). + + Raises: + FileNotFoundError: If the CSV does not exist. + ValueError: If required columns cannot be identified. + """ + if not gen_csv_path.exists(): + raise FileNotFoundError(f"Generator CSV not found: {gen_csv_path}") + + with open(gen_csv_path, encoding="utf-8") as f: + reader = csv.reader(f) + rows = list(reader) + + if not rows: + raise ValueError(f"Generator CSV is empty: {gen_csv_path}") + + first_row = rows[0] + if _is_header_row(first_row): + headers = first_row + data_rows = rows[1:] + bus_idx = _detect_column_index(headers, ["bus", "bus_i", "i"]) + machine_id_idx = _detect_column_index(headers, ["machine_id", "id", "gen_id", "ID"]) + pg_idx = _detect_column_index(headers, ["pg", "p", "Pg", "PG"]) + qg_idx = _detect_column_index(headers, ["qg", "q", "Qg", "QG"]) + status_idx = _detect_column_index(headers, ["status", "st", "stat"]) + + if bus_idx is None: + raise ValueError(f"Cannot find bus column in generator headers: {headers}") + if pg_idx is None or qg_idx is None: + raise ValueError(f"Cannot find PG/QG columns in generator headers: {headers}") + else: + data_rows = rows + bus_idx = _MPC_GEN_COL_BUS + machine_id_idx = None # MATPOWER headerless format has no machine_id + pg_idx = _MPC_GEN_COL_PG + qg_idx = _MPC_GEN_COL_QG + status_idx = _MPC_GEN_COL_STATUS + + results: list[dict] = [] + gen_counter: dict[int, int] = {} # Track machine_id per bus for headerless + + for row in data_rows: + if not row or all(cell.strip() == "" for cell in row): + continue + + # Check status + if status_idx is not None and status_idx < len(row): + try: + status = int(float(row[status_idx].strip())) + except (ValueError, IndexError): + status = 1 + if status != 1: + continue + + try: + bus_num = int(float(row[bus_idx].strip())) + p = float(row[pg_idx].strip()) + q = float(row[qg_idx].strip()) + except (ValueError, IndexError) as exc: + raise ValueError(f"Cannot parse generator data from row: {row}") from exc + + if machine_id_idx is not None and machine_id_idx < len(row): + machine_id = row[machine_id_idx].strip() + else: + # Assign sequential IDs per bus for headerless CSVs + gen_counter[bus_num] = gen_counter.get(bus_num, 0) + 1 + machine_id = str(gen_counter[bus_num]) + + results.append( + { + "bus": bus_num, + "machine_id": machine_id, + "P": p, + "Q": q, + } + ) + + results.sort(key=lambda r: (r["bus"], r["machine_id"])) + return results + + +# --------------------------------------------------------------------------- +# Flat-start solver execution (Path B) +# --------------------------------------------------------------------------- + + +def run_matpower_acpf( + intermediate_dir: Path, + output_dir: Path, + settings: SolverSettings, +) -> tuple[list[dict], list[dict], list[dict], ConvergenceInfo]: + """Run MATPOWER runpf via Octave on the intermediate format data. + + Generates a temporary Octave script that: + 1. Loads the intermediate format CSVs into an mpc struct. + 2. Configures MATPOWER options (tolerance, max iterations, Q-limit enforcement). + 3. Runs ``runpf`` with flat-start initial conditions. + 4. If two-stage Q-limit enforcement: runs a second ``runpf`` with + ``enforce_q_lims`` enabled, using the first solution as the starting point. + 5. Exports bus results (VM, VA), branch results (P/Q flows), and + generator results (P, Q) as CSV files. + + The function invokes Octave via ``subprocess``, captures stdout/stderr, + parses the exported CSV files, and returns structured results. + + Args: + intermediate_dir: Directory containing the intermediate format CSVs. + output_dir: Temporary directory for Octave script and CSV exports. + settings: Solver configuration. + + Returns: + A tuple of (bus_results, branch_results, gen_results, convergence_info). + Each results list has the same dict schema as the extract_* functions. + + Raises: + RuntimeError: If Octave exits with a non-zero status. + FileNotFoundError: If intermediate format CSVs are missing. + """ + import subprocess + + if not intermediate_dir.is_dir(): + raise FileNotFoundError(f"Intermediate format directory not found: {intermediate_dir}") + + output_dir.mkdir(parents=True, exist_ok=True) + tol = settings.tolerance or 1e-8 + max_iter = settings.max_iterations or 100 + + # Build Octave script + script = _build_matpower_octave_script( + intermediate_dir=intermediate_dir, + output_dir=output_dir, + tolerance=tol, + max_iterations=max_iter, + enforce_q_limits=(settings.q_limits_enforced is True), + ) + + script_path = output_dir / "run_acpf.m" + script_path.write_text(script, encoding="utf-8") + + result = subprocess.run( + ["octave", "--no-gui", "--no-window-system", str(script_path)], + capture_output=True, + text=True, + timeout=600, + ) + + if result.returncode != 0: + raise RuntimeError( + f"Octave runpf failed (exit code {result.returncode}).\n" + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + + # Parse convergence info from stdout + convergence = _parse_matpower_convergence(result.stdout) + + # Read exported CSVs + bus_results = extract_bus_results(output_dir / "bus_results.csv") + branch_results = extract_branch_results(output_dir / "branch_results.csv") + gen_results = extract_generator_results(output_dir / "gen_results.csv") + + return bus_results, branch_results, gen_results, convergence + + +def _build_matpower_octave_script( + intermediate_dir: Path, + output_dir: Path, + tolerance: float, + max_iterations: int, + enforce_q_limits: bool, +) -> str: + """Build an Octave script to run MATPOWER runpf.""" + return f"""\ +% Auto-generated ACPF reference extraction script +addpath(genpath(getenv('MATPOWER_PATH'))); + +% Load intermediate format CSVs into mpc struct +bus = csvread('{intermediate_dir}/bus.csv'); +gen = csvread('{intermediate_dir}/gen.csv'); +branch = csvread('{intermediate_dir}/branch.csv'); + +mpc = struct(); +mpc.version = '2'; +mpc.baseMVA = 100; +mpc.bus = bus; +mpc.gen = gen; +mpc.branch = branch; + +% Flat start: set all VM=1.0, VA=0.0 +mpc.bus(:, 8) = 1.0; +mpc.bus(:, 9) = 0.0; + +% Configure options +mpopt = mpoption('verbose', 2, 'out.all', 0); +mpopt = mpoption(mpopt, 'pf.tol', {tolerance}); +mpopt = mpoption(mpopt, 'pf.nr.max_it', {max_iterations}); + +% Stage 1: converge without Q-limits +mpopt = mpoption(mpopt, 'pf.enforce_q_lims', 0); +results = runpf(mpc, mpopt); + +if results.success + fprintf('CONVERGED_STAGE1\\n'); + fprintf('ITERATIONS: %d\\n', results.iterations); +else + error('Stage 1 (relaxed Q-limits) failed to converge'); +end + +{"% Stage 2: re-converge with Q-limits" if enforce_q_limits else "% Q-limit enforcement skipped"} +{ + ''' +mpopt2 = mpoption(mpopt, 'pf.enforce_q_lims', 1); +results2 = runpf(results, mpopt2); +if results2.success + results = results2; + fprintf('CONVERGED_STAGE2\\n'); + fprintf('Q_LIMITS_ENFORCED: true\\n'); +else + fprintf('STAGE2_FAILED\\n'); + fprintf('Q_LIMITS_ENFORCED: false\\n'); +end +''' + if enforce_q_limits + else "fprintf('Q_LIMITS_ENFORCED: false\\n');" + } + +% Export results +csvwrite('{output_dir}/bus_results.csv', results.bus); +csvwrite('{output_dir}/branch_results.csv', results.branch); +csvwrite('{output_dir}/gen_results.csv', results.gen); + +fprintf('DONE\\n'); +""" + + +def _parse_matpower_convergence(stdout: str) -> ConvergenceInfo: + """Parse convergence info from MATPOWER Octave stdout.""" + converged = "CONVERGED_STAGE1" in stdout or "CONVERGED_STAGE2" in stdout + iterations: int | None = None + for line in stdout.splitlines(): + if line.startswith("ITERATIONS:"): + try: + iterations = int(line.split(":")[1].strip()) + except (ValueError, IndexError): + pass + + return ConvergenceInfo( + converged=converged, + iterations=iterations, + final_mismatch_mw=None, + final_mismatch_mvar=None, + ) + + +def run_gridcal_acpf( + intermediate_dir: Path, + settings: SolverSettings, +) -> tuple[list[dict], list[dict], list[dict], ConvergenceInfo]: + """Run GridCal Newton-Raphson on the intermediate format data. + + Loads the intermediate format into a GridCal MultiCircuit object, + configures the NR solver (tolerance, max iterations), runs power flow, + and extracts results. + + For two-stage Q-limit enforcement: first run with control_q disabled, + then re-run with control_q enabled using the converged voltages as + the starting point. + + Args: + intermediate_dir: Directory containing the intermediate format CSVs. + settings: Solver configuration. + + Returns: + A tuple of (bus_results, branch_results, gen_results, convergence_info). + + Raises: + RuntimeError: If the solver fails to converge on both stages. + ImportError: If GridCal is not installed in the current environment. + """ + try: + import GridCal # noqa: F401 + except ImportError as exc: + raise ImportError( + "GridCal is not installed. Install it to use the GridCal solver path." + ) from exc + + raise NotImplementedError( + "GridCal ACPF solver path is not yet implemented. " + "Use MATPOWER (--canonical-parser matpower) for flat-start path." + ) + + +# --------------------------------------------------------------------------- +# Bus exclusion (D1 registry consumption) +# --------------------------------------------------------------------------- + + +def _load_exclusion_registry( + registry_path: Path, +) -> tuple[set[int], dict[str, int]]: + """Load the D1 bus exclusion registry. + + Args: + registry_path: Path to ``excluded_buses.json``. + + Returns: + A tuple of (excluded_bus_set, reason_counts) where reason_counts + maps exclusion reason strings to counts. + """ + if not registry_path.exists(): + return set(), {} + + with open(registry_path, encoding="utf-8") as f: + data = json.load(f) + + excluded: set[int] = set() + reason_counts: dict[str, int] = {} + + buses = data.get("excluded_buses", data.get("buses", [])) + for entry in buses: + if isinstance(entry, dict): + raw_bus = entry.get("bus", entry.get("bus_i", 0)) + bus_num = int(raw_bus) if raw_bus is not None else 0 + reason = entry.get("reason", "unknown") + else: + bus_num = int(entry) + reason = "unknown" + excluded.add(bus_num) + reason_counts[reason] = reason_counts.get(reason, 0) + 1 + + return excluded, reason_counts + + +def _apply_exclusions( + bus_results: list[dict], + gen_results: list[dict], + excluded_buses: set[int], +) -> tuple[list[dict], list[dict]]: + """Remove excluded buses from bus and generator results.""" + filtered_buses = [r for r in bus_results if r["bus"] not in excluded_buses] + filtered_gens = [r for r in gen_results if r["bus"] not in excluded_buses] + return filtered_buses, filtered_gens + + +# --------------------------------------------------------------------------- +# System summary computation +# --------------------------------------------------------------------------- + + +def compute_system_summary( + bus_results: list[dict], + branch_results: list[dict], + gen_results: list[dict], + bus_csv_path: Path, +) -> SystemSummary: + """Compute system-level aggregate quantities from the ACPF solution. + + Total generation is the sum of all generator P and Q values. + Total load is read from the intermediate format bus table (PD, QD columns). + Total losses are computed as: sum of (P_from + P_to) across all branches. + The slack bus is identified as the bus with type = 3. + Power balance residual = total_gen_mw - total_load_mw - total_loss_mw. + + Args: + bus_results: Per-bus results from extraction or solver. + branch_results: Per-branch results from extraction or solver. + gen_results: Per-generator results from extraction or solver. + bus_csv_path: Path to the intermediate format bus CSV (for load data + and slack bus identification). + + Returns: + A SystemSummary with all fields populated. + + Raises: + ValueError: If no slack bus (type=3) is found. + """ + # Total generation + total_gen_mw = sum(g["P"] for g in gen_results) + total_gen_mvar = sum(g["Q"] for g in gen_results) + + # Total load and slack bus from bus CSV + headers, data_rows = _load_bus_csv_raw(bus_csv_path) + bus_idx, type_idx, _vm_idx, _va_idx, pd_idx, qd_idx = _resolve_bus_col_indices(headers) + + total_load_mw = 0.0 + total_load_mvar = 0.0 + slack_candidates: list[tuple[int, float]] = [] # (bus_num, max_gen_capacity) + + # Build set of output bus numbers for load filtering + output_bus_nums = {r["bus"] for r in bus_results} + + for row in data_rows: + if not row or all(cell.strip() == "" for cell in row): + continue + try: + bus_num = int(float(row[bus_idx].strip())) + bus_type = int(float(row[type_idx].strip())) + pd = float(row[pd_idx].strip()) + qd = float(row[qd_idx].strip()) + except (ValueError, IndexError): + continue + + # Only count load from non-excluded, non-isolated buses + if bus_num in output_bus_nums: + total_load_mw += pd + total_load_mvar += qd + + if bus_type == _SLACK_BUS_TYPE: + # Find max gen capacity at this bus + gen_cap = sum(g["P"] for g in gen_results if g["bus"] == bus_num) + slack_candidates.append((bus_num, gen_cap)) + + if not slack_candidates: + raise ValueError("No slack bus (type=3) found in bus CSV.") + + # Primary slack = type-3 bus with largest generator MW capacity + slack_candidates.sort(key=lambda x: x[1], reverse=True) + slack_bus = slack_candidates[0][0] + + # Total losses from branch flows + total_loss_mw = sum(b["P_from"] + b["P_to"] for b in branch_results) + total_loss_mvar = sum(b["Q_from"] + b["Q_to"] for b in branch_results) + + # Power balance residual + residual = total_gen_mw - total_load_mw - total_loss_mw + + return SystemSummary( + total_gen_mw=total_gen_mw, + total_gen_mvar=total_gen_mvar, + total_load_mw=total_load_mw, + total_load_mvar=total_load_mvar, + total_loss_mw=total_loss_mw, + total_loss_mvar=total_loss_mvar, + slack_bus=slack_bus, + power_balance_residual_mw=residual, + ) + + +# --------------------------------------------------------------------------- +# Output writing +# --------------------------------------------------------------------------- + + +def write_buses_csv(bus_results: list[dict], output_path: Path) -> None: + """Write buses_acpf.csv. + + Args: + bus_results: Per-bus results. Each dict has keys: bus, VM, VA. + output_path: Full path to the output CSV file. + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["bus", "VM", "VA"]) + for r in bus_results: + writer.writerow( + [ + r["bus"], + f"{r['VM']:.8f}", + f"{r['VA']:.6f}", + ] + ) + + +def write_branches_csv(branch_results: list[dict], output_path: Path) -> None: + """Write branches_acpf.csv. + + Args: + branch_results: Per-branch results. Each dict has keys: + from_bus, to_bus, ckt, P_from, Q_from, P_to, Q_to. + output_path: Full path to the output CSV file. + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["from_bus", "to_bus", "ckt", "P_from", "Q_from", "P_to", "Q_to"]) + for r in branch_results: + writer.writerow( + [ + r["from_bus"], + r["to_bus"], + r["ckt"], + f"{r['P_from']:.4f}", + f"{r['Q_from']:.4f}", + f"{r['P_to']:.4f}", + f"{r['Q_to']:.4f}", + ] + ) + + +def write_generators_csv(gen_results: list[dict], output_path: Path) -> None: + """Write generators_acpf.csv. + + Args: + gen_results: Per-generator results. Each dict has keys: + bus, machine_id, P, Q. + output_path: Full path to the output CSV file. + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["bus", "machine_id", "P", "Q"]) + for r in gen_results: + writer.writerow( + [ + r["bus"], + r["machine_id"], + f"{r['P']:.4f}", + f"{r['Q']:.4f}", + ] + ) + + +def write_summary_json( + source: SolutionSource, + classification: str, + canonical_parser: str, + settings: SolverSettings, + convergence: ConvergenceInfo | None, + summary: SystemSummary, + counts: dict[str, int], + warnings: list[str], + output_path: Path, +) -> None: + """Write summary_acpf.json. + + Serializes all metadata into the JSON schema defined in the PRD. + + Args: + source: EXTRACTED or COMPUTED. + classification: D8 snapshot classification string. + canonical_parser: Name of the canonical parser. + settings: Solver settings (fields are None for EXTRACTED). + convergence: Solver convergence info (None for EXTRACTED). + summary: System-level summary. + counts: Dict with keys: buses_total, buses_excluded_isolated, + buses_excluded_deenergized, buses_in_output, + branches_in_output, generators_in_output. + warnings: List of warning messages (e.g., power balance residual). + output_path: Full path to the output JSON file. + """ + conv = convergence or ConvergenceInfo() + + data = { + "solution_source": source.value, + "snapshot_classification": classification, + "canonical_parser": canonical_parser, + "solver": { + "name": settings.name, + "version": settings.version, + "settings": { + "initial_conditions": ("flat_start" if source == SolutionSource.COMPUTED else None), + "tolerance": settings.tolerance, + "max_iterations": settings.max_iterations, + "q_limits_enforced": settings.q_limits_enforced, + "q_limit_strategy": settings.q_limit_strategy, + "enforce_area_interchange": settings.enforce_area_interchange, + }, + "convergence": { + "converged": conv.converged, + "iterations": conv.iterations, + "final_mismatch_mw": conv.final_mismatch_mw, + "final_mismatch_mvar": conv.final_mismatch_mvar, + }, + }, + "system_summary": { + "total_gen_mw": summary.total_gen_mw, + "total_gen_mvar": summary.total_gen_mvar, + "total_load_mw": summary.total_load_mw, + "total_load_mvar": summary.total_load_mvar, + "total_loss_mw": summary.total_loss_mw, + "total_loss_mvar": summary.total_loss_mvar, + "slack_bus": summary.slack_bus, + "power_balance_residual_mw": summary.power_balance_residual_mw, + }, + "counts": counts, + "warnings": warnings, + "timestamp": datetime.now(timezone.utc).isoformat(), + } + + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + + +# --------------------------------------------------------------------------- +# Supplemental reference files (OQ-P3-05) +# --------------------------------------------------------------------------- + + +def write_taps_csv(bus_csv_path: Path, output_path: Path) -> None: + """Write taps_acpf.csv — transformer tap positions. + + Placeholder for supplemental reference; reads transformer data from + intermediate format and writes bus/tap pairs. + + Args: + bus_csv_path: Path to intermediate format bus or transformer CSV. + output_path: Full path to the output CSV file. + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["bus", "tap_pu"]) + # Transformer tap data extraction requires transformer-specific CSV + # which may or may not exist in the intermediate format. Write header + # only if no transformer data is available. + + +def write_shunts_csv(bus_csv_path: Path, output_path: Path) -> None: + """Write shunts_acpf.csv — switched shunt admittance values. + + Placeholder for supplemental reference; reads shunt data from + intermediate format and writes bus/admittance pairs. + + Args: + bus_csv_path: Path to intermediate format bus or shunt CSV. + output_path: Full path to the output CSV file. + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["bus", "b_mvar"]) + # Switched shunt data extraction requires shunt-specific CSV + # which may or may not exist in the intermediate format. Write header + # only if no shunt data is available. + + +# --------------------------------------------------------------------------- +# Orchestrator +# --------------------------------------------------------------------------- + + +def build_acpf_reference( + intermediate_dir: Path, + snapshot_json_path: Path, + canonical_parser: str, + output_dir: Path, +) -> Path: + """Top-level orchestrator for ACPF reference solution extraction. + + Steps: + 1. Read D8 snapshot classification. + 2. Determine solution source (extracted vs. computed). + 3. If extracted: call extract_bus_results, extract_branch_results, + extract_generator_results. + 4. If computed: configure SolverSettings, call the appropriate solver + function (run_matpower_acpf or run_gridcal_acpf). + 5. Apply bus exclusions from D1 registry if available. + 6. Compute system summary. + 7. Check power balance residual -- warn if > 1 MW. + 8. Write buses_acpf.csv, branches_acpf.csv, generators_acpf.csv, + summary_acpf.json to output_dir. + + Args: + intermediate_dir: Directory containing intermediate format CSVs + (bus.csv, branch.csv, gen.csv, etc.). + snapshot_json_path: Path to D8's snapshot_confirmation.json. + canonical_parser: ``'matpower'`` or ``'gridcal'``. + output_dir: Output directory (created if it does not exist). + + Returns: + Path to the output directory containing all four output files. + + Raises: + ValueError: If snapshot classification is 'indeterminate'. + RuntimeError: If solver fails to converge (flat-start path). + FileNotFoundError: If required input files are missing. + """ + warnings_list: list[str] = [] + + # Step 1-2: Determine path + classification = read_snapshot_classification(snapshot_json_path) + source = determine_solution_source(classification) + + # Locate intermediate format CSVs + bus_csv = intermediate_dir / "bus.csv" + branch_csv = intermediate_dir / "branch.csv" + gen_csv = intermediate_dir / "gen.csv" + + solver_settings = SolverSettings() + convergence: ConvergenceInfo | None = None + + if source == SolutionSource.EXTRACTED: + # Path A: extract directly + bus_results = extract_bus_results(bus_csv) + branch_results = extract_branch_results(branch_csv) + gen_results = extract_generator_results(gen_csv) + else: + # Path B: flat-start solver + solver_settings = SolverSettings( + name="runpf" if canonical_parser == "matpower" else "gridcal_nr", + tolerance=1e-8, + max_iterations=100, + q_limits_enforced=True, + q_limit_strategy="two_stage_relaxed_then_enforced", + enforce_area_interchange=False, + ) + if canonical_parser == "matpower": + bus_results, branch_results, gen_results, convergence = run_matpower_acpf( + intermediate_dir, output_dir / "_solver_tmp", solver_settings + ) + else: + bus_results, branch_results, gen_results, convergence = run_gridcal_acpf( + intermediate_dir, solver_settings + ) + + # Load D1 bus exclusion registry if available + repo_root = intermediate_dir.parent.parent # data/fnm/intermediate -> data/fnm + exclusion_path = repo_root / "reference" / "excluded_buses.json" + excluded_buses, exclusion_reason_counts = _load_exclusion_registry(exclusion_path) + + if excluded_buses: + bus_results, gen_results = _apply_exclusions(bus_results, gen_results, excluded_buses) + + if not bus_results: + raise ValueError( + "No buses remain after exclusions. This indicates corrupted intermediate format data." + ) + + # Compute counts + headers, all_data_rows = _load_bus_csv_raw(bus_csv) + total_bus_count = 0 + isolated_count = 0 + deenergized_count = 0 + _, type_idx_c, vm_idx_c, _, _, _ = _resolve_bus_col_indices(headers) + + for row in all_data_rows: + if not row or all(cell.strip() == "" for cell in row): + continue + total_bus_count += 1 + try: + btype = int(float(row[type_idx_c].strip())) + except (ValueError, IndexError): + btype = 0 + if btype == _ISOLATED_BUS_TYPE: + isolated_count += 1 + continue + try: + vm_val = float(row[vm_idx_c].strip()) + except (ValueError, IndexError): + vm_val = 1.0 + if vm_val == 0.0: + deenergized_count += 1 + + counts = { + "buses_total": total_bus_count, + "buses_excluded_isolated": isolated_count, + "buses_excluded_deenergized": deenergized_count, + "buses_in_output": len(bus_results), + "branches_in_output": len(branch_results), + "generators_in_output": len(gen_results), + } + + # System summary + system_summary = compute_system_summary(bus_results, branch_results, gen_results, bus_csv) + + # Power balance check + residual_abs = abs(system_summary.power_balance_residual_mw) + if residual_abs > _POWER_BALANCE_WARN_THRESHOLD_MW: + warnings_list.append( + f"Power balance residual is {system_summary.power_balance_residual_mw:.4f} MW " + f"(exceeds {_POWER_BALANCE_WARN_THRESHOLD_MW} MW threshold)." + ) + + # Write outputs + output_dir.mkdir(parents=True, exist_ok=True) + write_buses_csv(bus_results, output_dir / "buses_acpf.csv") + write_branches_csv(branch_results, output_dir / "branches_acpf.csv") + write_generators_csv(gen_results, output_dir / "generators_acpf.csv") + write_summary_json( + source=source, + classification=classification, + canonical_parser=canonical_parser, + settings=solver_settings, + convergence=convergence, + summary=system_summary, + counts=counts, + warnings=warnings_list, + output_path=output_dir / "summary_acpf.json", + ) + + # Supplemental files + write_taps_csv(bus_csv, output_dir / "taps_acpf.csv") + write_shunts_csv(bus_csv, output_dir / "shunts_acpf.csv") + + return output_dir + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> None: + """CLI entry point for ACPF reference solution extraction. + + Usage:: + + python -m data.fnm.scripts.acpf_reference \\ + --intermediate-dir data/fnm/intermediate/canonical/ \\ + --snapshot-json data/fnm/intermediate/snapshot/snapshot_confirmation.json \\ + --canonical-parser gridcal \\ + [-o data/fnm/reference/acpf/] + + Exit codes: + - 0: Reference solution produced successfully. + - 1: Snapshot classification is 'indeterminate' -- manual resolution required. + - 2: Solver failed to converge (flat-start path). + - 3: Input error (missing files, malformed data). + + Args: + argv: Command-line arguments. If None, reads from sys.argv[1:]. + """ + parser = argparse.ArgumentParser( + description="Extract or compute ACPF reference solution from FNM intermediate format." + ) + parser.add_argument( + "--intermediate-dir", + type=Path, + required=True, + help="Directory containing intermediate format CSVs (bus.csv, branch.csv, gen.csv).", + ) + parser.add_argument( + "--snapshot-json", + type=Path, + required=True, + help="Path to D8's snapshot_confirmation.json.", + ) + parser.add_argument( + "--canonical-parser", + type=str, + required=True, + choices=["matpower", "gridcal"], + help="Name of the canonical parser.", + ) + parser.add_argument( + "-o", + "--output-dir", + type=Path, + default=Path("data/fnm/reference/acpf"), + help="Output directory (default: data/fnm/reference/acpf/).", + ) + + args = parser.parse_args(argv if argv is not None else sys.argv[1:]) + + try: + result_dir = build_acpf_reference( + intermediate_dir=args.intermediate_dir, + snapshot_json_path=args.snapshot_json, + canonical_parser=args.canonical_parser, + output_dir=args.output_dir, + ) + print(f"ACPF reference solution written to: {result_dir}") + sys.exit(0) + except ValueError as exc: + if "indeterminate" in str(exc).lower(): + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1) + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(3) + except RuntimeError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(2) + except FileNotFoundError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(3) + + +if __name__ == "__main__": + main() diff --git a/data/fnm/scripts/bus_exclusion_registry.py b/data/fnm/scripts/bus_exclusion_registry.py new file mode 100644 index 00000000..66afafeb --- /dev/null +++ b/data/fnm/scripts/bus_exclusion_registry.py @@ -0,0 +1,1188 @@ +"""Bus Exclusion Registry for FNM verification. + +Analyzes intermediate format bus, branch, and transformer tables to identify +all buses that must be excluded from ACPF and DCPF verification metrics. +Three exclusion categories: PSS/E-declared isolated buses (IDE=4), +de-energized buses (VM=0), and topologically disconnected buses. + +For each excluded bus, the registry records bus number, name, area, zone, +base kV, and a machine-readable exclusion reason. Output formats: CSV and JSON. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from collections import deque +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import Enum +from pathlib import Path + +# --------------------------------------------------------------------------- +# Exclusion reason classification +# --------------------------------------------------------------------------- + + +class ExclusionReason(Enum): + """Why a bus is excluded from verification metrics.""" + + IDE_4_ISOLATED = "ide_4_isolated" + """Bus type code IDE=4 in the PSS/E bus record.""" + + VM_ZERO_DEENERGIZED = "vm_zero_deenergized" + """Bus voltage magnitude VM=0.0 in the solved case.""" + + DISCONNECTED_ISLAND = "disconnected_island" + """Bus belongs to a connected component that does not contain the slack bus.""" + + +# Priority order for primary reason assignment when multiple apply. +# Lower index = higher priority. +EXCLUSION_PRIORITY: list[ExclusionReason] = [ + ExclusionReason.IDE_4_ISOLATED, + ExclusionReason.VM_ZERO_DEENERGIZED, + ExclusionReason.DISCONNECTED_ISLAND, +] + + +# --------------------------------------------------------------------------- +# Per-bus exclusion record +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ExcludedBusRecord: + """A single bus excluded from verification metrics.""" + + bus_number: int + """PSS/E bus number (I field).""" + + bus_name: str + """PSS/E bus name (NAME field). May be empty or whitespace-padded.""" + + area: int + """Area number (AREA field).""" + + zone: int + """Zone number (ZONE field).""" + + base_kv: float + """Bus base voltage in kV (BASKV field).""" + + primary_reason: ExclusionReason + """The highest-priority exclusion reason that applies to this bus.""" + + all_reasons: list[ExclusionReason] + """All exclusion reasons that apply, ordered by priority.""" + + island_id: int | None + """Connected component ID this bus belongs to. None if IDE=4 and the + bus was excluded before connectivity analysis.""" + + vm: float + """Voltage magnitude from the bus table (for diagnostic reference).""" + + va: float + """Voltage angle from the bus table (for diagnostic reference).""" + + ide: int + """Bus type code from the bus table.""" + + +# --------------------------------------------------------------------------- +# Island summary +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class IslandSummary: + """Summary of one connected component in the network.""" + + island_id: int + """Sequential integer ID. 0 = main island (contains the slack bus).""" + + bus_count: int + """Number of buses in this island.""" + + is_main: bool + """True if this island contains the slack bus.""" + + slack_bus: int | None + """Slack bus number if this is the main island, else None.""" + + sample_buses: list[int] + """Up to 5 bus numbers from this island (for diagnostic display).""" + + voltage_levels: list[float] + """Distinct base kV values present in this island, sorted descending.""" + + +# --------------------------------------------------------------------------- +# Summary statistics +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ExclusionSummary: + """Aggregate statistics for the bus exclusion registry.""" + + total_buses: int + """Total number of buses in the intermediate format bus table.""" + + excluded_total: int + """Total number of unique excluded buses.""" + + excluded_by_reason: dict[str, int] + """Count of buses where each reason is the primary reason.""" + + remaining_for_verification: int + """total_buses - excluded_total.""" + + connected_components: int + """Total number of connected components found.""" + + main_island_size: int + """Number of buses in the main island.""" + + disconnected_island_count: int + """Number of connected components that are NOT the main island.""" + + disconnected_island_sizes: list[int] + """Size of each disconnected island, sorted descending.""" + + islands: list[IslandSummary] + """Summary of every connected component, main island first.""" + + ide4_count: int + """Number of buses with IDE=4.""" + + vm_zero_count: int + """Number of buses with VM=0.0. May overlap with IDE=4 count.""" + + disconnected_count: int + """Number of buses whose primary reason is DISCONNECTED_ISLAND.""" + + +# --------------------------------------------------------------------------- +# Metadata +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class RegistryMetadata: + """Provenance metadata for the bus exclusion registry.""" + + bus_csv_path: str + """Path to the bus table CSV that was analyzed.""" + + branch_csv_path: str + """Path to the branch table CSV used for connectivity.""" + + transformer_csv_path: str + """Path to the transformer table CSV used for connectivity.""" + + generated_timestamp: str + """ISO 8601 timestamp of when the registry was generated.""" + + slack_bus_number: int + """The slack bus (IDE=3) used as the main island anchor.""" + + vm_zero_threshold: float + """Threshold for VM=0 detection.""" + + graph_node_count: int + """Number of nodes in the connectivity graph (total buses minus IDE=4).""" + + graph_edge_count: int + """Number of edges in the connectivity graph.""" + + +# --------------------------------------------------------------------------- +# Top-level registry +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class BusExclusionRegistry: + """Complete bus exclusion registry output.""" + + excluded_buses: list[ExcludedBusRecord] + """All excluded buses, sorted by bus number.""" + + summary: ExclusionSummary + """Aggregate statistics.""" + + metadata: RegistryMetadata + """Provenance and configuration metadata.""" + + +# --------------------------------------------------------------------------- +# Column name mapping helpers +# --------------------------------------------------------------------------- + +_BUS_COLUMN_MAP: dict[str, list[str]] = { + "I": ["i", "bus_i", "number", "bus_number"], + "IDE": ["ide", "type", "bus_type"], + "VM": ["vm", "vm_pu"], + "VA": ["va", "va_deg"], + "BASKV": ["baskv", "basekv", "vnom", "base_kv"], + "NAME": ["name", "bus_name"], + "AREA": ["area"], + "ZONE": ["zone"], + "OWNER": ["owner"], +} + +_BRANCH_COLUMN_MAP: dict[str, list[str]] = { + "I": ["i", "fbus", "from_bus"], + "J": ["j", "tbus", "to_bus"], + "CKT": ["ckt", "circuit"], + "ST": ["st", "status", "br_status"], +} + +_TRANSFORMER_COLUMN_MAP: dict[str, list[str]] = { + "I": ["i", "fbus", "bus1"], + "J": ["j", "tbus", "bus2"], + "K": ["k", "bus3"], + "CKT": ["ckt", "circuit"], + "STAT": ["stat", "status"], +} + + +def _resolve_columns( + headers: list[str], + column_map: dict[str, list[str]], + required: list[str], +) -> dict[str, int]: + """Map normalized column names to CSV column indices. + + Args: + headers: Raw CSV header row. + column_map: Mapping from canonical name to list of variant names. + required: Canonical names that must be found. + + Returns: + Dict mapping canonical name to column index. + + Raises: + ValueError: If a required column cannot be found. + """ + lower_headers = [h.strip().lower() for h in headers] + result: dict[str, int] = {} + + for canonical, variants in column_map.items(): + # Check canonical name first (case-insensitive) + canonical_lower = canonical.lower() + if canonical_lower in lower_headers: + result[canonical] = lower_headers.index(canonical_lower) + continue + # Check variants + for variant in variants: + if variant.lower() in lower_headers: + result[canonical] = lower_headers.index(variant.lower()) + break + + missing = [r for r in required if r not in result] + if missing: + raise ValueError(f"Required columns not found: {missing}. Available headers: {headers}") + return result + + +# --------------------------------------------------------------------------- +# Input loading +# --------------------------------------------------------------------------- + + +def load_bus_table(bus_csv_path: Path) -> list[dict[str, str | int | float]]: + """Load the intermediate format bus table from CSV. + + Reads the bus CSV produced by the canonical parser. Expects columns + matching the Phase 1 D7 intermediate format bus schema. + + Args: + bus_csv_path: Path to the bus table CSV file. + + Returns: + List of dicts with keys normalized to PSS/E field names and values + cast to appropriate types. + + Raises: + FileNotFoundError: If the CSV file does not exist. + ValueError: If required columns cannot be identified. + """ + if not bus_csv_path.exists(): + raise FileNotFoundError(f"Bus CSV not found: {bus_csv_path}") + + with open(bus_csv_path, encoding="utf-8") as f: + reader = csv.reader(f) + rows = list(reader) + + if not rows: + raise ValueError(f"Bus CSV is empty: {bus_csv_path}") + + headers = rows[0] + col_map = _resolve_columns(headers, _BUS_COLUMN_MAP, required=["I", "IDE", "VM"]) + data_rows = rows[1:] + + result: list[dict[str, str | int | float]] = [] + for row in data_rows: + if not row or all(cell.strip() == "" for cell in row): + continue + + bus: dict[str, str | int | float] = {} + bus["I"] = int(float(row[col_map["I"]].strip())) + bus["IDE"] = int(float(row[col_map["IDE"]].strip())) + bus["VM"] = float(row[col_map["VM"]].strip()) + bus["VA"] = float(row[col_map.get("VA", col_map["VM"])].strip()) if "VA" in col_map else 0.0 + bus["BASKV"] = float(row[col_map["BASKV"]].strip()) if "BASKV" in col_map else 0.0 + bus["NAME"] = row[col_map["NAME"]].strip() if "NAME" in col_map else "" + bus["AREA"] = int(float(row[col_map["AREA"]].strip())) if "AREA" in col_map else 0 + bus["ZONE"] = int(float(row[col_map["ZONE"]].strip())) if "ZONE" in col_map else 0 + + result.append(bus) + + return result + + +def load_branch_table(branch_csv_path: Path) -> list[dict[str, str | int | float]]: + """Load the intermediate format branch table from CSV. + + Args: + branch_csv_path: Path to the branch table CSV file. + + Returns: + List of dicts with keys normalized to I, J, CKT, ST. + + Raises: + FileNotFoundError: If the CSV file does not exist. + ValueError: If required columns cannot be identified. + """ + if not branch_csv_path.exists(): + raise FileNotFoundError(f"Branch CSV not found: {branch_csv_path}") + + with open(branch_csv_path, encoding="utf-8") as f: + reader = csv.reader(f) + rows = list(reader) + + if not rows: + raise ValueError(f"Branch CSV is empty: {branch_csv_path}") + + headers = rows[0] + col_map = _resolve_columns(headers, _BRANCH_COLUMN_MAP, required=["I", "J", "ST"]) + data_rows = rows[1:] + + result: list[dict[str, str | int | float]] = [] + for row in data_rows: + if not row or all(cell.strip() == "" for cell in row): + continue + + branch: dict[str, str | int | float] = {} + branch["I"] = int(float(row[col_map["I"]].strip())) + branch["J"] = int(float(row[col_map["J"]].strip())) + branch["ST"] = int(float(row[col_map["ST"]].strip())) + branch["CKT"] = row[col_map["CKT"]].strip() if "CKT" in col_map else "" + + result.append(branch) + + return result + + +def load_transformer_table( + transformer_csv_path: Path, +) -> list[dict[str, str | int | float]]: + """Load the intermediate format transformer table from CSV. + + Args: + transformer_csv_path: Path to the transformer table CSV file. + + Returns: + List of dicts with keys normalized to I, J, K, CKT, STAT. + + Raises: + FileNotFoundError: If the CSV file does not exist. + ValueError: If required columns cannot be identified. + """ + if not transformer_csv_path.exists(): + raise FileNotFoundError(f"Transformer CSV not found: {transformer_csv_path}") + + with open(transformer_csv_path, encoding="utf-8") as f: + reader = csv.reader(f) + rows = list(reader) + + if not rows: + raise ValueError(f"Transformer CSV is empty: {transformer_csv_path}") + + headers = rows[0] + col_map = _resolve_columns(headers, _TRANSFORMER_COLUMN_MAP, required=["I", "J", "STAT"]) + data_rows = rows[1:] + + result: list[dict[str, str | int | float]] = [] + for row in data_rows: + if not row or all(cell.strip() == "" for cell in row): + continue + + xfmr: dict[str, str | int | float] = {} + xfmr["I"] = int(float(row[col_map["I"]].strip())) + xfmr["J"] = int(float(row[col_map["J"]].strip())) + xfmr["K"] = int(float(row[col_map["K"]].strip())) if "K" in col_map else 0 + xfmr["STAT"] = int(float(row[col_map["STAT"]].strip())) + xfmr["CKT"] = row[col_map["CKT"]].strip() if "CKT" in col_map else "" + + result.append(xfmr) + + return result + + +# --------------------------------------------------------------------------- +# Exclusion detection -- IDE=4 +# --------------------------------------------------------------------------- + + +def find_ide4_buses(bus_rows: list[dict]) -> set[int]: + """Identify buses with IDE=4 (PSS/E isolated bus type code). + + Args: + bus_rows: Bus table rows from ``load_bus_table()``. + + Returns: + Set of bus numbers (I) where IDE == 4. + """ + return {int(row["I"]) for row in bus_rows if int(row["IDE"]) == 4} + + +# --------------------------------------------------------------------------- +# Exclusion detection -- VM=0 +# --------------------------------------------------------------------------- + + +def find_vm_zero_buses( + bus_rows: list[dict], + threshold: float = 0.0, +) -> set[int]: + """Identify buses with voltage magnitude of exactly zero. + + Args: + bus_rows: Bus table rows from ``load_bus_table()``. + threshold: VM values <= this threshold are classified as zero. + + Returns: + Set of bus numbers (I) where VM <= threshold. + """ + return {int(row["I"]) for row in bus_rows if float(row["VM"]) <= threshold} + + +# --------------------------------------------------------------------------- +# Network connectivity analysis +# --------------------------------------------------------------------------- + + +def build_connectivity_graph( + bus_rows: list[dict], + branch_rows: list[dict], + transformer_rows: list[dict], + excluded_bus_numbers: set[int], +) -> dict[int, set[int]]: + """Build an undirected adjacency list from in-service branches and + transformers, excluding pre-identified buses. + + Args: + bus_rows: All bus table rows (used to enumerate graph nodes). + branch_rows: Branch table rows. + transformer_rows: Transformer table rows. + excluded_bus_numbers: Bus numbers to exclude from the graph. + + Returns: + Adjacency list mapping each bus number to the set of connected buses. + """ + # Initialize nodes: all non-excluded buses + adjacency: dict[int, set[int]] = {} + for row in bus_rows: + bus_num = int(row["I"]) + if bus_num not in excluded_bus_numbers: + adjacency[bus_num] = set() + + def _add_edge(a: int, b: int) -> None: + if a == b: + return + if a in adjacency and b in adjacency: + adjacency[a].add(b) + adjacency[b].add(a) + + # Edges from in-service branches + for row in branch_rows: + if int(row["ST"]) != 1: + continue + i = int(row["I"]) + j = int(row["J"]) + _add_edge(i, j) + + # Edges from in-service transformers + for row in transformer_rows: + if int(row["STAT"]) != 1: + continue + i = int(row["I"]) + j = int(row["J"]) + k = int(row.get("K", 0)) + + # Two-winding: K=0 + _add_edge(i, j) + + # Three-winding: K!=0 -> fully connect I, J, K + if k != 0: + _add_edge(i, k) + _add_edge(j, k) + + return adjacency + + +def find_connected_components( + adjacency: dict[int, set[int]], +) -> list[set[int]]: + """Find all connected components in an undirected graph via BFS. + + Args: + adjacency: Undirected adjacency list. + + Returns: + List of sets sorted by descending size. + """ + visited: set[int] = set() + components: list[set[int]] = [] + + for node in adjacency: + if node in visited: + continue + # BFS + queue: deque[int] = deque([node]) + component: set[int] = set() + while queue: + current = queue.popleft() + if current in visited: + continue + visited.add(current) + component.add(current) + for neighbor in adjacency[current]: + if neighbor not in visited: + queue.append(neighbor) + components.append(component) + + # Sort by descending size + components.sort(key=len, reverse=True) + return components + + +def identify_main_island( + components: list[set[int]], + bus_rows: list[dict], +) -> tuple[int, int]: + """Identify the main island as the component containing the slack bus. + + Args: + components: Connected components from ``find_connected_components()``. + bus_rows: Bus table rows (to find IDE=3 bus). + + Returns: + Tuple of (component_index, slack_bus_number). + + Raises: + ValueError: If no bus with IDE=3 is found. + """ + # Find the first IDE=3 bus + slack_bus: int | None = None + for row in bus_rows: + if int(row["IDE"]) == 3: + slack_bus = int(row["I"]) + break + + if slack_bus is None: + raise ValueError("No slack bus (IDE=3) found in the bus table.") + + for idx, component in enumerate(components): + if slack_bus in component: + return idx, slack_bus + + raise ValueError( + f"Slack bus {slack_bus} not found in any connected component. " + "It may have been excluded as IDE=4." + ) + + +def find_disconnected_buses( + components: list[set[int]], + main_island_index: int, +) -> set[int]: + """Return bus numbers from all components except the main island. + + Args: + components: Connected components. + main_island_index: Index of the main island. + + Returns: + Set of bus numbers belonging to non-main-island components. + """ + result: set[int] = set() + for idx, component in enumerate(components): + if idx != main_island_index: + result |= component + return result + + +# --------------------------------------------------------------------------- +# Registry assembly +# --------------------------------------------------------------------------- + + +def build_excluded_bus_records( + bus_rows: list[dict], + ide4_buses: set[int], + vm_zero_buses: set[int], + disconnected_buses: set[int], +) -> list[ExcludedBusRecord]: + """Assemble ExcludedBusRecord for every excluded bus. + + Args: + bus_rows: All bus table rows. + ide4_buses: Bus numbers with IDE=4. + vm_zero_buses: Bus numbers with VM=0. + disconnected_buses: Bus numbers in disconnected islands. + + Returns: + List of ExcludedBusRecord sorted by bus_number ascending. + """ + all_excluded = ide4_buses | vm_zero_buses | disconnected_buses + if not all_excluded: + return [] + + # Build lookup by bus number + bus_lookup: dict[int, dict] = {} + for row in bus_rows: + bus_lookup[int(row["I"])] = row + + records: list[ExcludedBusRecord] = [] + for bus_num in sorted(all_excluded): + bus_row = bus_lookup.get(bus_num) + if bus_row is None: + continue + + # Determine all applicable reasons in priority order + all_reasons: list[ExclusionReason] = [] + for reason in EXCLUSION_PRIORITY: + if reason == ExclusionReason.IDE_4_ISOLATED and bus_num in ide4_buses: + all_reasons.append(reason) + elif reason == ExclusionReason.VM_ZERO_DEENERGIZED and bus_num in vm_zero_buses: + all_reasons.append(reason) + elif reason == ExclusionReason.DISCONNECTED_ISLAND and bus_num in disconnected_buses: + all_reasons.append(reason) + + primary_reason = all_reasons[0] + + # IDE=4 buses have island_id=None (excluded before connectivity) + island_id: int | None = None + if bus_num in ide4_buses: + island_id = None + # For non-IDE=4 buses, island_id will be set by the caller if needed + # We leave it as None here; build_registry sets it properly + + records.append( + ExcludedBusRecord( + bus_number=bus_num, + bus_name=str(bus_row.get("NAME", "")), + area=int(bus_row.get("AREA", 0)), + zone=int(bus_row.get("ZONE", 0)), + base_kv=float(bus_row.get("BASKV", 0.0)), + primary_reason=primary_reason, + all_reasons=all_reasons, + island_id=island_id, + vm=float(bus_row.get("VM", 0.0)), + va=float(bus_row.get("VA", 0.0)), + ide=int(bus_row.get("IDE", 0)), + ) + ) + + return records + + +def _assign_island_ids( + records: list[ExcludedBusRecord], + components: list[set[int]], + main_island_index: int, +) -> list[ExcludedBusRecord]: + """Assign island_id to excluded bus records based on component membership. + + Main island = island_id 0. Other islands get sequential IDs starting at 1, + ordered by descending size. + + IDE=4 buses retain island_id=None. + """ + # Build bus -> island_id mapping + bus_to_island: dict[int, int] = {} + # Main island gets id=0 + for bus_num in components[main_island_index]: + bus_to_island[bus_num] = 0 + + # Other islands get sequential IDs by descending size + other_indices = [i for i in range(len(components)) if i != main_island_index] + # They are already sorted by descending size from find_connected_components + for isl_id, idx in enumerate(other_indices, start=1): + for bus_num in components[idx]: + bus_to_island[bus_num] = isl_id + + updated: list[ExcludedBusRecord] = [] + for rec in records: + if rec.primary_reason == ExclusionReason.IDE_4_ISOLATED: + updated.append(rec) + else: + rec_island_id: int | None = bus_to_island.get(rec.bus_number) + updated.append( + ExcludedBusRecord( + bus_number=rec.bus_number, + bus_name=rec.bus_name, + area=rec.area, + zone=rec.zone, + base_kv=rec.base_kv, + primary_reason=rec.primary_reason, + all_reasons=rec.all_reasons, + island_id=rec_island_id, + vm=rec.vm, + va=rec.va, + ide=rec.ide, + ) + ) + return updated + + +def build_island_summaries( + components: list[set[int]], + main_island_index: int, + slack_bus_number: int, + bus_rows: list[dict], +) -> list[IslandSummary]: + """Build IslandSummary for every connected component. + + Args: + components: Connected components. + main_island_index: Index of the main island. + slack_bus_number: The slack bus number. + bus_rows: Bus table rows (for base kV lookup). + + Returns: + List of IslandSummary, main island first, then by descending size. + """ + bus_kv: dict[int, float] = {} + for row in bus_rows: + bus_kv[int(row["I"])] = float(row.get("BASKV", 0.0)) + + summaries: list[IslandSummary] = [] + + # Main island first (id=0) + main_component = components[main_island_index] + main_buses_sorted = sorted(main_component) + main_kvs = sorted({bus_kv.get(b, 0.0) for b in main_component}, reverse=True) + summaries.append( + IslandSummary( + island_id=0, + bus_count=len(main_component), + is_main=True, + slack_bus=slack_bus_number, + sample_buses=main_buses_sorted[:5], + voltage_levels=main_kvs, + ) + ) + + # Other islands by descending size + other_indices = [i for i in range(len(components)) if i != main_island_index] + for island_id, idx in enumerate(other_indices, start=1): + comp = components[idx] + comp_sorted = sorted(comp) + kvs = sorted({bus_kv.get(b, 0.0) for b in comp}, reverse=True) + summaries.append( + IslandSummary( + island_id=island_id, + bus_count=len(comp), + is_main=False, + slack_bus=None, + sample_buses=comp_sorted[:5], + voltage_levels=kvs, + ) + ) + + return summaries + + +def build_exclusion_summary( + total_buses: int, + excluded_records: list[ExcludedBusRecord], + island_summaries: list[IslandSummary], + ide4_count: int, + vm_zero_count: int, + disconnected_count: int, +) -> ExclusionSummary: + """Compute aggregate exclusion statistics. + + Args: + total_buses: Total buses in the bus table. + excluded_records: All excluded bus records. + island_summaries: Island summaries. + ide4_count: Total IDE=4 buses. + vm_zero_count: Total VM=0 buses (may overlap with IDE=4). + disconnected_count: Buses excluded primarily for disconnection. + + Returns: + An ExclusionSummary with all fields populated. + """ + excluded_total = len(excluded_records) + + # Count by primary reason + excluded_by_reason: dict[str, int] = {} + for reason in ExclusionReason: + count = sum(1 for r in excluded_records if r.primary_reason == reason) + if count > 0: + excluded_by_reason[reason.value] = count + + main_island_size = 0 + disconnected_islands: list[IslandSummary] = [] + for s in island_summaries: + if s.is_main: + main_island_size = s.bus_count + else: + disconnected_islands.append(s) + + disconnected_island_sizes = sorted([s.bus_count for s in disconnected_islands], reverse=True) + + return ExclusionSummary( + total_buses=total_buses, + excluded_total=excluded_total, + excluded_by_reason=excluded_by_reason, + remaining_for_verification=total_buses - excluded_total, + connected_components=len(island_summaries), + main_island_size=main_island_size, + disconnected_island_count=len(disconnected_islands), + disconnected_island_sizes=disconnected_island_sizes, + islands=island_summaries, + ide4_count=ide4_count, + vm_zero_count=vm_zero_count, + disconnected_count=disconnected_count, + ) + + +# --------------------------------------------------------------------------- +# Registry orchestration +# --------------------------------------------------------------------------- + + +def build_registry( + bus_csv_path: Path, + branch_csv_path: Path, + transformer_csv_path: Path, + vm_zero_threshold: float = 0.0, +) -> BusExclusionRegistry: + """Orchestrate the full bus exclusion analysis. + + Args: + bus_csv_path: Path to the bus table CSV. + branch_csv_path: Path to the branch table CSV. + transformer_csv_path: Path to the transformer table CSV. + vm_zero_threshold: Threshold for VM=0 detection (default 0.0). + + Returns: + A complete BusExclusionRegistry. + + Raises: + FileNotFoundError: If any input CSV does not exist. + ValueError: If required columns are missing or no slack bus found. + """ + # 1. Load tables + bus_rows = load_bus_table(bus_csv_path) + branch_rows = load_branch_table(branch_csv_path) + transformer_rows = load_transformer_table(transformer_csv_path) + + # 2. Identify IDE=4 buses + ide4_buses = find_ide4_buses(bus_rows) + + # 3. Identify VM=0 buses + vm_zero_buses = find_vm_zero_buses(bus_rows, threshold=vm_zero_threshold) + + # 4. Build connectivity graph (excluding IDE=4 buses) + adjacency = build_connectivity_graph( + bus_rows, branch_rows, transformer_rows, excluded_bus_numbers=ide4_buses + ) + + # Count edges + edge_count = sum(len(neighbors) for neighbors in adjacency.values()) // 2 + + # 5. Find connected components + components = find_connected_components(adjacency) + + # 6. Identify main island + main_island_index, slack_bus_number = identify_main_island(components, bus_rows) + + # 7. Find disconnected buses + disconnected_buses = find_disconnected_buses(components, main_island_index) + + # 8. Build records + excluded_records = build_excluded_bus_records( + bus_rows, ide4_buses, vm_zero_buses, disconnected_buses + ) + + # Assign island IDs + excluded_records = _assign_island_ids(excluded_records, components, main_island_index) + + # 9. Build island summaries + island_summaries = build_island_summaries( + components, main_island_index, slack_bus_number, bus_rows + ) + + # Count disconnected_count = buses whose primary reason is DISCONNECTED_ISLAND + disconnected_primary_count = sum( + 1 for r in excluded_records if r.primary_reason == ExclusionReason.DISCONNECTED_ISLAND + ) + + # 10. Build summary + summary = build_exclusion_summary( + total_buses=len(bus_rows), + excluded_records=excluded_records, + island_summaries=island_summaries, + ide4_count=len(ide4_buses), + vm_zero_count=len(vm_zero_buses), + disconnected_count=disconnected_primary_count, + ) + + # Build metadata + metadata = RegistryMetadata( + bus_csv_path=str(bus_csv_path), + branch_csv_path=str(branch_csv_path), + transformer_csv_path=str(transformer_csv_path), + generated_timestamp=datetime.now(timezone.utc).isoformat(), + slack_bus_number=slack_bus_number, + vm_zero_threshold=vm_zero_threshold, + graph_node_count=len(adjacency), + graph_edge_count=edge_count, + ) + + return BusExclusionRegistry( + excluded_buses=excluded_records, + summary=summary, + metadata=metadata, + ) + + +# --------------------------------------------------------------------------- +# Output serialization +# --------------------------------------------------------------------------- + + +def registry_to_dict(registry: BusExclusionRegistry) -> dict: + """Convert a BusExclusionRegistry to a JSON-serializable dict. + + Args: + registry: The registry to serialize. + + Returns: + A dict safe for ``json.dumps()``. + """ + excluded_buses = [] + for rec in registry.excluded_buses: + excluded_buses.append( + { + "bus_number": rec.bus_number, + "bus_name": rec.bus_name, + "area": rec.area, + "zone": rec.zone, + "base_kv": rec.base_kv, + "primary_reason": rec.primary_reason.value, + "all_reasons": [r.value for r in rec.all_reasons], + "island_id": rec.island_id, + "vm": rec.vm, + "va": rec.va, + "ide": rec.ide, + } + ) + + s = registry.summary + summary = { + "total_buses": s.total_buses, + "excluded_total": s.excluded_total, + "excluded_by_reason": s.excluded_by_reason, + "remaining_for_verification": s.remaining_for_verification, + "connected_components": s.connected_components, + "main_island_size": s.main_island_size, + "disconnected_island_count": s.disconnected_island_count, + "disconnected_island_sizes": s.disconnected_island_sizes, + "islands": [ + { + "island_id": isl.island_id, + "bus_count": isl.bus_count, + "is_main": isl.is_main, + "slack_bus": isl.slack_bus, + "sample_buses": isl.sample_buses, + "voltage_levels": isl.voltage_levels, + } + for isl in s.islands + ], + "ide4_count": s.ide4_count, + "vm_zero_count": s.vm_zero_count, + "disconnected_count": s.disconnected_count, + } + + m = registry.metadata + metadata = { + "bus_csv_path": m.bus_csv_path, + "branch_csv_path": m.branch_csv_path, + "transformer_csv_path": m.transformer_csv_path, + "generated_timestamp": m.generated_timestamp, + "slack_bus_number": m.slack_bus_number, + "vm_zero_threshold": m.vm_zero_threshold, + "graph_node_count": m.graph_node_count, + "graph_edge_count": m.graph_edge_count, + } + + return { + "excluded_buses": excluded_buses, + "summary": summary, + "metadata": metadata, + } + + +def registry_to_csv(registry: BusExclusionRegistry, output_path: Path) -> None: + """Write the excluded bus list as a CSV file. + + Args: + registry: The bus exclusion registry. + output_path: Path to write the CSV file. + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + + fieldnames = [ + "bus_number", + "bus_name", + "area", + "zone", + "base_kv", + "primary_reason", + "all_reasons", + "island_id", + "vm", + "va", + "ide", + ] + + with open(output_path, "w", encoding="utf-8", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + for rec in registry.excluded_buses: + writer.writerow( + { + "bus_number": rec.bus_number, + "bus_name": rec.bus_name, + "area": rec.area, + "zone": rec.zone, + "base_kv": rec.base_kv, + "primary_reason": rec.primary_reason.value, + "all_reasons": ";".join(r.value for r in rec.all_reasons), + "island_id": rec.island_id if rec.island_id is not None else "", + "vm": rec.vm, + "va": rec.va, + "ide": rec.ide, + } + ) + + +def registry_to_json(registry: BusExclusionRegistry, output_path: Path) -> None: + """Write the complete registry as a JSON file. + + Args: + registry: The bus exclusion registry. + output_path: Path to write the JSON file. + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + data = registry_to_dict(registry) + output_path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> None: + """CLI entry point for the bus exclusion registry. + + Args: + argv: Command-line arguments. If ``None``, reads from ``sys.argv[1:]``. + """ + parser = argparse.ArgumentParser( + description="Analyze FNM intermediate format to build a bus exclusion registry." + ) + parser.add_argument( + "--bus-csv", + type=Path, + required=True, + help="Path to the bus table CSV.", + ) + parser.add_argument( + "--branch-csv", + type=Path, + required=True, + help="Path to the branch table CSV.", + ) + parser.add_argument( + "--transformer-csv", + type=Path, + required=True, + help="Path to the transformer table CSV.", + ) + parser.add_argument( + "-o", + "--output-dir", + type=Path, + default=None, + help="Output directory (default: data/fnm/reference/).", + ) + parser.add_argument( + "--vm-threshold", + type=float, + default=0.0, + help="VM threshold for de-energized classification (default: 0.0).", + ) + + args = parser.parse_args(argv if argv is not None else sys.argv[1:]) + + output_dir: Path = args.output_dir or Path("data/fnm/reference") + + try: + registry = build_registry( + bus_csv_path=args.bus_csv, + branch_csv_path=args.branch_csv, + transformer_csv_path=args.transformer_csv, + vm_zero_threshold=args.vm_threshold, + ) + except ValueError as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1 if "slack" in str(exc).lower() else 2) + + output_dir.mkdir(parents=True, exist_ok=True) + csv_path = output_dir / "excluded_buses.csv" + json_path = output_dir / "excluded_buses.json" + + registry_to_csv(registry, csv_path) + registry_to_json(registry, json_path) + + s = registry.summary + print(f"Total buses: {s.total_buses}") + print( + f"Excluded: {s.excluded_total} " + f"(IDE=4: {s.ide4_count}, VM=0: {s.vm_zero_count}, " + f"disconnected: {s.disconnected_count})" + ) + print(f"Remaining for verification: {s.remaining_for_verification}") + print( + f"Connected components: {s.connected_components} (main island: {s.main_island_size} buses)" + ) + print(f"CSV: {csv_path}") + print(f"JSON: {json_path}") + + +if __name__ == "__main__": + main() diff --git a/data/fnm/scripts/csv_join_keys.py b/data/fnm/scripts/csv_join_keys.py new file mode 100644 index 00000000..7dbf696e --- /dev/null +++ b/data/fnm/scripts/csv_join_keys.py @@ -0,0 +1,1102 @@ +"""Supplemental CSV Join-Key Mapping. + +Analyzes supplemental CSVs accompanying the FNM Annual S01 variant, +identifies columns that serve as join keys to PSS/E network elements in the +intermediate format tables, validates those joins against actual data, and +produces a structured mapping report (JSON + markdown). + +This resolves OQ-E03 (join keys between supplemental CSVs and PSS/E network +elements). +""" + +from __future__ import annotations + +import argparse +import csv +import json +import os +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from enum import Enum +from pathlib import Path +from typing import Any + +# --------------------------------------------------------------------------- +# Join key pattern registry +# --------------------------------------------------------------------------- + + +class JoinCardinality(Enum): + """Cardinality of a join between a supplemental CSV and an intermediate format table.""" + + ONE_TO_ONE = "1:1" + ONE_TO_MANY = "1:N" + MANY_TO_ONE = "N:1" + MANY_TO_MANY = "M:N" + + +class KeyType(Enum): + """Classification of a join key column's semantic type.""" + + BUS_NUMBER = "bus_number" + GENERATOR_ID = "generator_id" + GENERATOR_NAME = "generator_name" + BRANCH_COMPOSITE = "branch_composite" + TRANSFORMER_COMPOSITE = "transformer_composite" + AREA_NUMBER = "area_number" + ZONE_NUMBER = "zone_number" + ELEMENT_NAME = "element_name" + UNKNOWN = "unknown" + + +# --------------------------------------------------------------------------- +# Column name patterns for key discovery +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class KeyColumnPattern: + """A pattern for discovering candidate join-key columns in supplemental CSVs. + + The discovery engine matches CSV column names against these patterns + (case-insensitive substring or regex match) to identify candidate keys. + """ + + key_type: KeyType + column_patterns: list[str] + target_table: str + target_columns: list[str] + is_composite: bool = False + + +# --------------------------------------------------------------------------- +# Key discovery and validation results +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class CandidateKey: + """A candidate join key discovered in a supplemental CSV.""" + + csv_file: str + csv_columns: list[str] + key_type: KeyType + confidence: float + + +@dataclass(frozen=True) +class JoinValidationResult: + """Result of validating a candidate key against an intermediate format table.""" + + candidate: CandidateKey + target_table: str + target_columns: list[str] + csv_row_count: int + matched_row_count: int + unmatched_row_count: int + match_rate: float + cardinality: JoinCardinality + is_valid: bool + unmatched_sample: list[dict[str, str]] + notes: str = "" + + +# --------------------------------------------------------------------------- +# Per-CSV mapping +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class CsvJoinMapping: + """Complete join-key mapping for a single supplemental CSV.""" + + csv_file: str + csv_columns: list[str] + csv_row_count: int + candidate_keys: list[CandidateKey] + validated_joins: list[JoinValidationResult] + primary_join: JoinValidationResult | None + secondary_joins: list[JoinValidationResult] + sample_rows: list[dict[str, str]] + + +# --------------------------------------------------------------------------- +# Top-level mapping report +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ReportSummary: + """Aggregate statistics for the join-key report.""" + + total_csvs_analyzed: int + total_valid_joins: int + total_csvs_with_valid_join: int + total_csvs_without_valid_join: int + average_match_rate: float + csvs_needing_review: list[str] + + +@dataclass(frozen=True) +class ReportMetadata: + """Provenance metadata for the join-key report.""" + + fnm_path: str = "" + intermediate_dir: str = "" + match_rate_threshold: float = 0.80 + report_timestamp: str = "" + + +@dataclass(frozen=True) +class JoinKeyReport: + """Complete join-key mapping report for all supplemental CSVs.""" + + csv_mappings: list[CsvJoinMapping] + csvs_found: list[str] + csvs_missing: list[str] + intermediate_tables_used: list[str] + overall_summary: ReportSummary + metadata: ReportMetadata + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +FNM_CSV_NAMES: list[str] = [ + "LINE_AND_TRANSFORMER.csv", + "TRADING_HUB.csv", + "GEN_DISTRIBUTION_FACTOR.csv", + "CONTINGENCY.csv", + "INTERFACE.csv", + "INTERFACE_ELEMENT.csv", + "OUTAGE.csv", +] + + +# --------------------------------------------------------------------------- +# Key pattern registry +# --------------------------------------------------------------------------- + + +def get_default_key_patterns() -> list[KeyColumnPattern]: + """Return the default registry of join-key column patterns. + + Returns: + List of KeyColumnPattern entries sorted by specificity (composite + patterns first, then by number of column_patterns descending). + """ + patterns = [ + KeyColumnPattern( + key_type=KeyType.BRANCH_COMPOSITE, + column_patterns=["from_bus", "to_bus", "ckt"], + target_table="branch", + target_columns=["I", "J", "CKT"], + is_composite=True, + ), + KeyColumnPattern( + key_type=KeyType.TRANSFORMER_COMPOSITE, + column_patterns=["from_bus", "to_bus", "ckt"], + target_table="transformer", + target_columns=["I", "J", "K", "CKT"], + is_composite=True, + ), + KeyColumnPattern( + key_type=KeyType.GENERATOR_ID, + column_patterns=["gen_bus", "machine_id", "gen_id", "generator_id"], + target_table="generator", + target_columns=["I", "ID"], + is_composite=False, + ), + KeyColumnPattern( + key_type=KeyType.GENERATOR_NAME, + column_patterns=["gen_name", "generator_name", "unit_name"], + target_table="generator", + target_columns=["NAME"], + is_composite=False, + ), + KeyColumnPattern( + key_type=KeyType.BUS_NUMBER, + column_patterns=[ + "bus_num", + "bus_no", + "busnum", + "bus_number", + "bus_i", + "bus", + ], + target_table="bus", + target_columns=["I"], + is_composite=False, + ), + KeyColumnPattern( + key_type=KeyType.AREA_NUMBER, + column_patterns=["area_num", "area_number", "area_no", "area"], + target_table="area", + target_columns=["I"], + is_composite=False, + ), + KeyColumnPattern( + key_type=KeyType.ZONE_NUMBER, + column_patterns=["zone_num", "zone_number", "zone_no", "zone"], + target_table="zone", + target_columns=["I"], + is_composite=False, + ), + ] + # Sort: composites first, then by number of column_patterns descending + patterns.sort(key=lambda p: (not p.is_composite, -len(p.column_patterns))) + return patterns + + +# --------------------------------------------------------------------------- +# CSV reading +# --------------------------------------------------------------------------- + + +def read_csv_header(csv_path: Path) -> list[str]: + """Read the header row of a CSV file and return column names. + + Strips whitespace from column names. Handles BOM-prefixed files. + + Args: + csv_path: Path to the CSV file. + + Returns: + List of column names in order. + + Raises: + FileNotFoundError: If the file does not exist. + ValueError: If the file is empty (no header row). + """ + if not csv_path.exists(): + raise FileNotFoundError(f"CSV file not found: {csv_path}") + + with open(csv_path, encoding="utf-8-sig", newline="") as f: + reader = csv.reader(f) + try: + header = next(reader) + except StopIteration: + raise ValueError(f"CSV file is empty (no header row): {csv_path}") + + return [col.strip() for col in header] + + +def read_csv_sample(csv_path: Path, n_rows: int = 100) -> list[dict[str, str]]: + """Read up to n_rows of data from a CSV file. + + Returns rows as dicts mapping column name to string value. Does not + attempt type conversion. Strips whitespace from both column names + and values. + + Args: + csv_path: Path to the CSV file. + n_rows: Maximum number of data rows to read. + + Returns: + List of row dicts, up to n_rows. + + Raises: + FileNotFoundError: If the file does not exist. + ValueError: If the file has no data rows (header only). + """ + if not csv_path.exists(): + raise FileNotFoundError(f"CSV file not found: {csv_path}") + + rows: list[dict[str, str]] = [] + with open(csv_path, encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + if reader.fieldnames is None: + raise ValueError(f"CSV file is empty (no header row): {csv_path}") + for i, row in enumerate(reader): + if i >= n_rows: + break + rows.append({k.strip(): v.strip() if v else "" for k, v in row.items()}) + + if not rows: + raise ValueError(f"CSV file has no data rows (header only): {csv_path}") + + return rows + + +def read_csv_key_values( + csv_path: Path, + key_columns: list[str], +) -> list[tuple[str, ...]]: + """Read all values for the specified key columns from a CSV file. + + Returns a list of tuples, one per data row, containing the string values + of the specified columns in order. + + Args: + csv_path: Path to the CSV file. + key_columns: Column names to extract. + + Returns: + List of key value tuples. + + Raises: + FileNotFoundError: If the file does not exist. + KeyError: If any specified column is not in the CSV header. + """ + if not csv_path.exists(): + raise FileNotFoundError(f"CSV file not found: {csv_path}") + + result: list[tuple[str, ...]] = [] + with open(csv_path, encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + if reader.fieldnames is None: + return result + + # Normalize fieldnames for matching + raw_fields = [fn.strip() for fn in reader.fieldnames] + for kc in key_columns: + if kc not in raw_fields: + raise KeyError(f"Column '{kc}' not found in CSV header: {raw_fields}") + + for row in reader: + # Build normalized row + norm_row = {k.strip(): v.strip() if v else "" for k, v in row.items()} + result.append(tuple(norm_row[kc] for kc in key_columns)) + + return result + + +# --------------------------------------------------------------------------- +# Intermediate format table reading +# --------------------------------------------------------------------------- + + +def load_intermediate_key_values( + intermediate_dir: Path, + table_name: str, + key_columns: list[str], +) -> set[tuple[str, ...]]: + """Load the set of valid key values from an intermediate format table. + + Reads the specified columns from the intermediate format CSV table + (file named ``.csv`` in ``intermediate_dir``) and returns + a set of tuples representing the unique key values. + + Args: + intermediate_dir: Path to the directory containing D7 intermediate + format CSV tables. + table_name: Table name (file stem, e.g., 'bus', 'generator', 'branch'). + key_columns: Column names to extract as the key. + + Returns: + Set of key value tuples present in the intermediate table. + + Raises: + FileNotFoundError: If the table file does not exist. + KeyError: If any specified column is not in the table header. + """ + table_path = intermediate_dir / f"{table_name}.csv" + if not table_path.exists(): + raise FileNotFoundError(f"Intermediate table not found: {table_path}") + + result: set[tuple[str, ...]] = set() + with open(table_path, encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + if reader.fieldnames is None: + return result + + raw_fields = [fn.strip() for fn in reader.fieldnames] + for kc in key_columns: + if kc not in raw_fields: + raise KeyError( + f"Column '{kc}' not found in intermediate table '{table_name}': {raw_fields}" + ) + + for row in reader: + norm_row = {k.strip(): v.strip() if v else "" for k, v in row.items()} + result.add(tuple(norm_row[kc] for kc in key_columns)) + + return result + + +# --------------------------------------------------------------------------- +# Key discovery +# --------------------------------------------------------------------------- + + +def _match_column(csv_col: str, pattern: str) -> bool: + """Check if a CSV column name matches a pattern (case-insensitive). + + Uses exact match after normalizing: lowercased, underscores/spaces/hyphens + collapsed. Also matches if the pattern is a substring of the column name. + """ + col_lower = csv_col.lower().replace("-", "_").replace(" ", "_") + pat_lower = pattern.lower().replace("-", "_").replace(" ", "_") + return pat_lower == col_lower or pat_lower in col_lower + + +def _find_matching_columns( + csv_columns: list[str], + pattern: str, +) -> list[str]: + """Find CSV columns matching a pattern.""" + return [c for c in csv_columns if _match_column(c, pattern)] + + +def _values_look_numeric(sample: list[dict[str, str]], column: str) -> bool: + """Check if sample values for a column look like integers.""" + for row in sample: + val = row.get(column, "").strip() + if val and not val.lstrip("-").isdigit(): + return False + return True + + +def discover_candidate_keys( + csv_file: str, + csv_columns: list[str], + csv_sample: list[dict[str, str]], + key_patterns: list[KeyColumnPattern], +) -> list[CandidateKey]: + """Discover candidate join keys in a supplemental CSV. + + For each key pattern in the registry: + 1. Check whether the CSV contains columns matching the pattern's + ``column_patterns`` (case-insensitive substring match). + 2. For composite keys, all component columns must be present. + 3. Optionally inspect sample data values to boost confidence. + 4. Assign a confidence score based on match quality. + + Returns all candidate keys with confidence > 0.0, sorted by + confidence descending. + + Args: + csv_file: Name of the CSV file (for labeling). + csv_columns: Column names from the CSV header. + csv_sample: Sample data rows for value-based heuristics. + key_patterns: Registry of key column patterns to match against. + + Returns: + List of CandidateKey entries, sorted by confidence descending. + """ + candidates: list[CandidateKey] = [] + + for pattern in key_patterns: + if pattern.is_composite: + # All component patterns must match at least one column + matched_cols: list[str] = [] + all_found = True + for col_pat in pattern.column_patterns: + matches = _find_matching_columns(csv_columns, col_pat) + if matches: + matched_cols.append(matches[0]) + else: + all_found = False + break + + if not all_found: + continue + + # Compute confidence + confidence = 0.7 # base for composite match + # Boost if numeric values where expected + for col in matched_cols: + if col != matched_cols[-1]: # skip CKT (may be string) + if _values_look_numeric(csv_sample, col): + confidence = min(1.0, confidence + 0.1) + + candidates.append( + CandidateKey( + csv_file=csv_file, + csv_columns=matched_cols, + key_type=pattern.key_type, + confidence=confidence, + ) + ) + else: + # Simple key: find any column matching any pattern + for col_pat in pattern.column_patterns: + matches = _find_matching_columns(csv_columns, col_pat) + for matched_col in matches: + # Avoid matching a column already part of a composite + confidence = 0.6 # base for simple match + # Exact match gets higher confidence + if matched_col.lower() == col_pat.lower(): + confidence = 0.8 + + # Boost for numeric values if bus/area/zone + if pattern.key_type in ( + KeyType.BUS_NUMBER, + KeyType.AREA_NUMBER, + KeyType.ZONE_NUMBER, + ): + if _values_look_numeric(csv_sample, matched_col): + confidence = min(1.0, confidence + 0.15) + + candidates.append( + CandidateKey( + csv_file=csv_file, + csv_columns=[matched_col], + key_type=pattern.key_type, + confidence=confidence, + ) + ) + break # One match per pattern is enough + + # Deduplicate by csv_columns + key_type + seen: set[tuple[tuple[str, ...], str]] = set() + unique: list[CandidateKey] = [] + for c in candidates: + key = (tuple(c.csv_columns), c.key_type.value) + if key not in seen: + seen.add(key) + unique.append(c) + + unique.sort(key=lambda c: -c.confidence) + return unique + + +# --------------------------------------------------------------------------- +# Join validation +# --------------------------------------------------------------------------- + + +def _determine_cardinality( + csv_key_values: list[tuple[str, ...]], + target_key_set: set[tuple[str, ...]], +) -> JoinCardinality: + """Determine the join cardinality from key value distributions. + + Args: + csv_key_values: All key value tuples from the CSV (with duplicates). + target_key_set: Set of valid key value tuples from the target table. + + Returns: + The inferred JoinCardinality. + """ + csv_distinct = set(csv_key_values) + matched_csv_distinct = csv_distinct & target_key_set + + if not matched_csv_distinct: + return JoinCardinality.ONE_TO_ONE + + total_rows = len(csv_key_values) + distinct_count = len(csv_distinct) + + # Multiple CSV rows per distinct key => many CSV rows map to one target + has_csv_duplicates = total_rows > distinct_count + + # Check if distinct CSV keys map to more target keys than CSV keys + # (not really possible with set intersection, so we focus on CSV side) + matched_target_count = len(matched_csv_distinct) + + if has_csv_duplicates: + if matched_target_count < distinct_count: + return JoinCardinality.MANY_TO_MANY + return JoinCardinality.MANY_TO_ONE + else: + if matched_target_count == distinct_count: + return JoinCardinality.ONE_TO_ONE + return JoinCardinality.ONE_TO_MANY + + +def validate_join( + csv_path: Path, + candidate: CandidateKey, + intermediate_dir: Path, + target_table: str, + target_columns: list[str], + match_threshold: float = 0.80, +) -> JoinValidationResult: + """Validate a candidate join key against an intermediate format table. + + Args: + csv_path: Path to the supplemental CSV file. + candidate: The candidate key to validate. + intermediate_dir: Path to D7 intermediate format tables. + target_table: Intermediate table to join against. + target_columns: Key columns in the target table. + match_threshold: Minimum match_rate for the join to be valid. + + Returns: + A JoinValidationResult with match statistics and cardinality. + """ + csv_key_values = read_csv_key_values(csv_path, candidate.csv_columns) + target_key_set = load_intermediate_key_values(intermediate_dir, target_table, target_columns) + + csv_row_count = len(csv_key_values) + if csv_row_count == 0: + return JoinValidationResult( + candidate=candidate, + target_table=target_table, + target_columns=target_columns, + csv_row_count=0, + matched_row_count=0, + unmatched_row_count=0, + match_rate=0.0, + cardinality=JoinCardinality.ONE_TO_ONE, + is_valid=False, + unmatched_sample=[], + notes="CSV has no data rows.", + ) + + matched = 0 + unmatched_samples: list[dict[str, str]] = [] + + for key_tuple in csv_key_values: + if key_tuple in target_key_set: + matched += 1 + else: + if len(unmatched_samples) < 10: + sample_dict = {col: val for col, val in zip(candidate.csv_columns, key_tuple)} + unmatched_samples.append(sample_dict) + + unmatched = csv_row_count - matched + match_rate = matched / csv_row_count + + cardinality = _determine_cardinality(csv_key_values, target_key_set) + + return JoinValidationResult( + candidate=candidate, + target_table=target_table, + target_columns=target_columns, + csv_row_count=csv_row_count, + matched_row_count=matched, + unmatched_row_count=unmatched, + match_rate=match_rate, + cardinality=cardinality, + is_valid=match_rate >= match_threshold, + unmatched_sample=unmatched_samples, + ) + + +# --------------------------------------------------------------------------- +# Per-CSV analysis +# --------------------------------------------------------------------------- + + +def analyze_csv( + csv_path: Path, + intermediate_dir: Path, + key_patterns: list[KeyColumnPattern] | None = None, + match_threshold: float = 0.80, +) -> CsvJoinMapping: + """Analyze a single supplemental CSV for join keys. + + Orchestrates the full discovery-and-validation pipeline for one CSV: + 1. Read header and sample rows. + 2. Discover candidate keys using the pattern registry. + 3. Validate each candidate against the appropriate intermediate table. + 4. Select the primary join (highest match_rate among valid joins). + 5. Collect secondary valid joins. + 6. Return the complete CsvJoinMapping. + + Args: + csv_path: Path to the supplemental CSV file. + intermediate_dir: Path to D7 intermediate format tables. + key_patterns: Optional custom key pattern registry. + match_threshold: Minimum match_rate for join validity. + + Returns: + A CsvJoinMapping with all discovery and validation results. + """ + if key_patterns is None: + key_patterns = get_default_key_patterns() + + csv_file = csv_path.name + columns = read_csv_header(csv_path) + + try: + sample = read_csv_sample(csv_path, n_rows=100) + except ValueError: + sample = [] + + candidates = discover_candidate_keys(csv_file, columns, sample, key_patterns) + + # Validate each candidate + validated: list[JoinValidationResult] = [] + for candidate in candidates: + # Find the matching pattern to get target table/columns + target_table = None + target_columns = None + for pat in key_patterns: + if pat.key_type == candidate.key_type: + target_table = pat.target_table + target_columns = pat.target_columns + break + + if target_table is None or target_columns is None: + continue + + # Check if the intermediate table exists + table_path = intermediate_dir / f"{target_table}.csv" + if not table_path.exists(): + continue + + try: + result = validate_join( + csv_path, + candidate, + intermediate_dir, + target_table, + target_columns, + match_threshold, + ) + validated.append(result) + except (FileNotFoundError, KeyError): + continue + + # Select primary and secondary joins + valid_joins = [v for v in validated if v.is_valid] + valid_joins.sort(key=lambda v: -v.match_rate) + + primary = valid_joins[0] if valid_joins else None + secondary = valid_joins[1:] if len(valid_joins) > 1 else [] + + # Sample rows for documentation + sample_rows = sample[:5] if sample else [] + + # Count rows + csv_row_count = len(sample) + if sample: + # Read actual row count + try: + all_keys = read_csv_key_values(csv_path, [columns[0]]) + csv_row_count = len(all_keys) + except (KeyError, IndexError): + pass + + return CsvJoinMapping( + csv_file=csv_file, + csv_columns=columns, + csv_row_count=csv_row_count, + candidate_keys=candidates, + validated_joins=validated, + primary_join=primary, + secondary_joins=secondary, + sample_rows=sample_rows, + ) + + +# --------------------------------------------------------------------------- +# Full report generation +# --------------------------------------------------------------------------- + + +def build_join_key_report( + fnm_path: Path, + intermediate_dir: Path, + manifest_csv_names: list[str] | None = None, + key_patterns: list[KeyColumnPattern] | None = None, + match_threshold: float = 0.80, +) -> JoinKeyReport: + """Build the complete join-key mapping report for all supplemental CSVs. + + Args: + fnm_path: Resolved FNM_PATH directory containing supplemental CSVs. + intermediate_dir: Path to D7 intermediate format tables. + manifest_csv_names: Expected CSV file names (from D1 manifest). + key_patterns: Optional custom key pattern registry. + match_threshold: Minimum match_rate for join validity. + + Returns: + A complete JoinKeyReport. + """ + if manifest_csv_names is None: + manifest_csv_names = FNM_CSV_NAMES + + csvs_found: list[str] = [] + csvs_missing: list[str] = [] + + for name in manifest_csv_names: + if (fnm_path / name).exists(): + csvs_found.append(name) + else: + csvs_missing.append(name) + + # Analyze each found CSV + mappings: list[CsvJoinMapping] = [] + tables_used: set[str] = set() + + for csv_name in csvs_found: + csv_path = fnm_path / csv_name + mapping = analyze_csv(csv_path, intermediate_dir, key_patterns, match_threshold) + mappings.append(mapping) + + for vj in mapping.validated_joins: + tables_used.add(vj.target_table) + + # Compute summary + total_valid = sum(len([v for v in m.validated_joins if v.is_valid]) for m in mappings) + csvs_with_valid = sum(1 for m in mappings if m.primary_join is not None) + csvs_without_valid = len(mappings) - csvs_with_valid + + primary_rates = [m.primary_join.match_rate for m in mappings if m.primary_join is not None] + avg_rate = sum(primary_rates) / len(primary_rates) if primary_rates else 0.0 + + needing_review = [ + m.csv_file for m in mappings if m.primary_join is None or m.primary_join.match_rate < 0.90 + ] + + summary = ReportSummary( + total_csvs_analyzed=len(mappings), + total_valid_joins=total_valid, + total_csvs_with_valid_join=csvs_with_valid, + total_csvs_without_valid_join=csvs_without_valid, + average_match_rate=avg_rate, + csvs_needing_review=needing_review, + ) + + metadata = ReportMetadata( + fnm_path=str(fnm_path), + intermediate_dir=str(intermediate_dir), + match_rate_threshold=match_threshold, + report_timestamp=datetime.now(timezone.utc).isoformat(), + ) + + return JoinKeyReport( + csv_mappings=mappings, + csvs_found=csvs_found, + csvs_missing=csvs_missing, + intermediate_tables_used=sorted(tables_used), + overall_summary=summary, + metadata=metadata, + ) + + +# --------------------------------------------------------------------------- +# Serialization +# --------------------------------------------------------------------------- + + +def _serialize(obj: Any) -> Any: + """Recursively serialize dataclasses, enums, and other types to JSON-safe values.""" + if isinstance(obj, Enum): + return obj.value + if isinstance(obj, Path): + return str(obj) + if hasattr(obj, "__dataclass_fields__"): + return {k: _serialize(v) for k, v in asdict(obj).items()} + if isinstance(obj, list): + return [_serialize(item) for item in obj] + if isinstance(obj, dict): + return {k: _serialize(v) for k, v in obj.items()} + if isinstance(obj, tuple): + return [_serialize(item) for item in obj] + if isinstance(obj, set): + return sorted(_serialize(item) for item in obj) + return obj + + +def report_to_dict(report: JoinKeyReport) -> dict: + """Convert a JoinKeyReport to a JSON-serializable dict. + + All enum values are serialized as their string values. All dataclass + fields are recursively converted. + + Args: + report: The join-key report to serialize. + + Returns: + A dict safe for JSON serialization. + """ + return _serialize(report) + + +def report_to_markdown(report: JoinKeyReport) -> str: + """Render a JoinKeyReport as a markdown document. + + Args: + report: The join-key report to render. + + Returns: + A complete markdown string. + """ + lines: list[str] = [] + s = report.overall_summary + + lines.append("# Supplemental CSV Join-Key Mapping Report") + lines.append("") + lines.append("## Executive Summary") + lines.append("") + lines.append(f"- **CSVs analyzed:** {s.total_csvs_analyzed}") + lines.append(f"- **Valid joins found:** {s.total_valid_joins}") + lines.append(f"- **CSVs with valid join:** {s.total_csvs_with_valid_join}") + lines.append(f"- **CSVs without valid join:** {s.total_csvs_without_valid_join}") + lines.append(f"- **Average match rate:** {s.average_match_rate:.1%}") + if s.csvs_needing_review: + lines.append(f"- **CSVs needing review:** {', '.join(s.csvs_needing_review)}") + lines.append("") + + # Per-CSV sections + for mapping in report.csv_mappings: + lines.append(f"## {mapping.csv_file}") + lines.append("") + lines.append(f"- **Row count:** {mapping.csv_row_count}") + lines.append(f"- **Columns:** {', '.join(mapping.csv_columns)}") + lines.append("") + + if mapping.primary_join: + pj = mapping.primary_join + lines.append("### Primary Join") + lines.append("") + lines.append(f"- **Target table:** {pj.target_table}") + lines.append( + f"- **Key columns:** {', '.join(pj.candidate.csv_columns)} " + f"-> {', '.join(pj.target_columns)}" + ) + lines.append(f"- **Cardinality:** {pj.cardinality.value}") + lines.append(f"- **Match rate:** {pj.match_rate:.1%}") + lines.append(f"- **Matched/Total:** {pj.matched_row_count}/{pj.csv_row_count}") + lines.append("") + + if pj.unmatched_sample: + lines.append("#### Unmatched Samples") + lines.append("") + for sample in pj.unmatched_sample[:5]: + lines.append(f" - {sample}") + lines.append("") + + if mapping.secondary_joins: + lines.append("### Secondary Joins") + lines.append("") + for sj in mapping.secondary_joins: + lines.append( + f"- **{sj.target_table}** via " + f"{', '.join(sj.candidate.csv_columns)}: " + f"{sj.match_rate:.1%} match rate, " + f"{sj.cardinality.value}" + ) + lines.append("") + + if not mapping.primary_join and not mapping.secondary_joins: + lines.append("*No valid join keys identified.*") + lines.append("") + + # Aggregate table + lines.append("## Join Summary Table") + lines.append("") + lines.append("| CSV File | Primary Join Target | Key Columns | Cardinality | Match Rate |") + lines.append("|----------|-------------------|-------------|-------------|------------|") + for mapping in report.csv_mappings: + if mapping.primary_join: + pj = mapping.primary_join + lines.append( + f"| {mapping.csv_file} | {pj.target_table} | " + f"{', '.join(pj.candidate.csv_columns)} | " + f"{pj.cardinality.value} | {pj.match_rate:.1%} |" + ) + else: + lines.append(f"| {mapping.csv_file} | — | — | — | — |") + lines.append("") + + # Missing CSVs + if report.csvs_missing: + lines.append("## Missing CSVs") + lines.append("") + for name in report.csvs_missing: + lines.append(f"- {name}") + lines.append("") + + # Methodology + lines.append("## Methodology") + lines.append("") + lines.append(f"- **Match rate threshold:** {report.metadata.match_rate_threshold:.0%}") + lines.append("- **Discovery:** Column name pattern matching (case-insensitive)") + lines.append("- **Cardinality:** Inferred from CSV key value distribution vs target table") + lines.append(f"- **Report generated:** {report.metadata.report_timestamp}") + lines.append("") + + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> None: + """CLI entry point for the supplemental CSV join-key mapping. + + Args: + argv: Command-line arguments. If ``None``, reads from ``sys.argv[1:]``. + """ + import sys + + parser = argparse.ArgumentParser( + description="Analyze supplemental CSV join keys against intermediate format tables." + ) + parser.add_argument( + "--fnm-path", + type=Path, + default=None, + help="Path to FNM data directory. Falls back to FNM_PATH env var.", + ) + parser.add_argument( + "--intermediate-dir", + type=Path, + default=Path("data/fnm/intermediate/canonical"), + help="Path to intermediate format tables directory.", + ) + parser.add_argument( + "-o", + "--output-dir", + type=Path, + default=Path("data/fnm/intermediate/csv_join_keys"), + help="Output directory for report files.", + ) + parser.add_argument( + "--match-threshold", + type=float, + default=0.80, + help="Minimum match rate for a join to be valid (default: 0.80).", + ) + + args = parser.parse_args(argv) + + fnm_path = args.fnm_path + if fnm_path is None: + env_val = os.environ.get("FNM_PATH") + if env_val is None: + print("Error: --fnm-path not provided and FNM_PATH env var not set.", file=sys.stderr) + sys.exit(2) + fnm_path = Path(env_val) + + fnm_path = fnm_path.expanduser().resolve() + if not fnm_path.is_dir(): + print(f"Error: FNM path is not a directory: {fnm_path}", file=sys.stderr) + sys.exit(2) + + intermediate_dir = args.intermediate_dir.resolve() + if not intermediate_dir.is_dir(): + print( + f"Error: Intermediate directory not found: {intermediate_dir}", + file=sys.stderr, + ) + sys.exit(2) + + report = build_join_key_report( + fnm_path=fnm_path, + intermediate_dir=intermediate_dir, + match_threshold=args.match_threshold, + ) + + output_dir = args.output_dir + output_dir.mkdir(parents=True, exist_ok=True) + + # Write JSON + json_path = output_dir / "join_key_report.json" + json_path.write_text(json.dumps(report_to_dict(report), indent=2) + "\n", encoding="utf-8") + + # Write markdown + md_path = output_dir / "join_key_report.md" + md_path.write_text(report_to_markdown(report), encoding="utf-8") + + print(f"Report written to {output_dir}") + print(f" JSON: {json_path}") + print(f" Markdown: {md_path}") + + # Exit code 1 if any CSV has no valid join + has_failures = any(m.primary_join is None for m in report.csv_mappings) + if has_failures: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/data/fnm/scripts/dcpf_acpf_characterization.py b/data/fnm/scripts/dcpf_acpf_characterization.py new file mode 100644 index 00000000..a49f9710 --- /dev/null +++ b/data/fnm/scripts/dcpf_acpf_characterization.py @@ -0,0 +1,1753 @@ +"""DCPF-vs-ACPF Characterization for FNM Annual S01. + +Compares the DCPF reference solution (Phase 3 D3) against the ACPF reference +solution (Phase 3 D2) to characterize how well the DC power flow approximation +represents the full AC solution. Quantifies per-bus angle deviations, per-branch +active power flow deviations, aggregate distribution statistics, and identifies +worst-case elements with probable physical causes for the largest discrepancies. + +Output files: +- ``data/fnm/reference/dcpf_vs_acpf_characterization.json`` (machine-readable) +- ``data/fnm/reference/dcpf_vs_acpf_characterization.md`` (human-readable) + +Uses only Python stdlib (no numpy/scipy). +""" + +from __future__ import annotations + +import argparse +import csv +import json +import logging +import math +import statistics +import sys +from dataclasses import dataclass, field +from datetime import datetime, timezone +from enum import Enum +from pathlib import Path + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +NEAR_ZERO_FLOW_THRESHOLD_MW: float = 1.0 +"""Branches with |P_from_acpf| <= this value are excluded from percentage +deviation calculations. Standard practice for power flow comparisons.""" + +ANGLE_COMPLIANCE_THRESHOLDS_DEG: list[float] = [0.5, 1.0, 2.0, 3.0, 5.0, 10.0] +"""Thresholds (degrees) for cumulative angle deviation compliance fractions.""" + +FLOW_COMPLIANCE_THRESHOLDS_PCT: list[float] = [1.0, 2.0, 5.0, 10.0, 20.0, 50.0] +"""Thresholds (percent) for cumulative flow deviation compliance fractions.""" + +EXPECTED_ANGLE_COMPLIANCE_PCT: float = 95.0 +"""Expected: >95% of buses within 3 degrees.""" + +EXPECTED_ANGLE_THRESHOLD_DEG: float = 3.0 +"""Angle threshold for the expected-range check.""" + +EXPECTED_FLOW_COMPLIANCE_PCT: float = 90.0 +"""Expected: >90% of branches within 10%.""" + +EXPECTED_FLOW_THRESHOLD_PCT: float = 10.0 +"""Flow percentage threshold for the expected-range check.""" + +WORST_CASE_COUNT: int = 50 +"""Number of worst-case buses and branches to include in the report.""" + + +# --------------------------------------------------------------------------- +# Data Structures +# --------------------------------------------------------------------------- + + +class DeviationCause(Enum): + """Probable cause categories for worst-case DC-vs-AC deviations.""" + + PHASE_SHIFTER = "phase_shifter" + HIGH_REACTANCE = "high_reactance" + HEAVY_LOADING = "heavy_loading" + LOW_VOLTAGE = "low_voltage" + HIGH_VOLTAGE = "high_voltage" + TRANSFORMER_TAP = "transformer_tap" + SLACK_BUS_VICINITY = "slack_bus_vicinity" + UNCATEGORIZED = "uncategorized" + + +@dataclass(frozen=True) +class BusDeviation: + """Per-bus angle deviation record.""" + + bus: int + VA_acpf_deg: float + VA_dcpf_deg: float + delta_VA_deg: float + abs_delta_VA_deg: float + VM_acpf_pu: float + base_kv: float + area: int + causes: list[DeviationCause] = field(default_factory=list) + + +@dataclass(frozen=True) +class BranchDeviation: + """Per-branch flow deviation record.""" + + from_bus: int + to_bus: int + ckt: str + P_from_acpf_MW: float + P_flow_dcpf_MW: float + delta_P_MW: float + abs_delta_P_MW: float + delta_P_pct: float | None + """None when |P_from_acpf| <= near-zero threshold.""" + abs_delta_P_pct: float | None + x_pu: float + tap_ratio: float + shift_deg: float + is_transformer: bool + causes: list[DeviationCause] = field(default_factory=list) + + +@dataclass(frozen=True) +class AggregateStats: + """Aggregate statistics for a deviation distribution.""" + + count: int + mean: float + median: float + std: float + min: float + max: float + p05: float + p95: float + + +@dataclass(frozen=True) +class ComplianceFractions: + """Cumulative compliance fractions at multiple thresholds.""" + + thresholds: list[float] + """The threshold values (degrees or percent).""" + fractions: list[float] + """Fraction of elements within each threshold (0.0 to 1.0).""" + + +@dataclass(frozen=True) +class CharacterizationResult: + """Complete characterization output, serializable to JSON and markdown.""" + + bus_deviations: list[BusDeviation] + branch_deviations: list[BranchDeviation] + angle_stats_signed: AggregateStats + angle_stats_absolute: AggregateStats + angle_compliance: ComplianceFractions + flow_mw_stats_signed: AggregateStats + flow_mw_stats_absolute: AggregateStats + flow_pct_stats_signed: AggregateStats + flow_pct_stats_absolute: AggregateStats + flow_pct_compliance: ComplianceFractions + join_summary: dict[str, int] + system_level: dict[str, float] + expected_range_checks: dict[str, dict] + worst_buses: list[BusDeviation] + worst_branches: list[BranchDeviation] + warnings: list[str] + metadata: dict[str, str] = field(default_factory=dict) + + +# --------------------------------------------------------------------------- +# Data loading +# --------------------------------------------------------------------------- + + +def load_acpf_buses(acpf_buses_path: Path) -> list[dict]: + """Load ACPF bus reference data from buses_acpf.csv. + + Args: + acpf_buses_path: Path to ``buses_acpf.csv``. + + Returns: + List of dicts with keys ``bus`` (int), ``VM`` (float), ``VA`` (float). + + Raises: + FileNotFoundError: If the CSV does not exist. + ValueError: If required columns are missing. + """ + if not acpf_buses_path.exists(): + raise FileNotFoundError(f"ACPF bus CSV not found: {acpf_buses_path}") + + result: list[dict] = [] + with open(acpf_buses_path, encoding="utf-8") as f: + reader = csv.DictReader(f) + if reader.fieldnames is None: + raise ValueError(f"ACPF bus CSV is empty: {acpf_buses_path}") + fields = {fn.strip().lower() for fn in reader.fieldnames} + if "bus" not in fields or "va" not in fields: + raise ValueError(f"Required columns 'bus', 'VA' not found. Got: {reader.fieldnames}") + for row in reader: + result.append( + { + "bus": int(float(row["bus"].strip())), + "VM": float(row["VM"].strip()) if "VM" in row else 1.0, + "VA": float(row["VA"].strip()), + } + ) + return result + + +def load_dcpf_buses(dcpf_buses_path: Path) -> list[dict]: + """Load DCPF bus reference data from buses_dcpf.csv. + + Args: + dcpf_buses_path: Path to ``buses_dcpf.csv``. + + Returns: + List of dicts with keys ``bus`` (int), ``VA`` (float). + + Raises: + FileNotFoundError: If the CSV does not exist. + ValueError: If required columns are missing. + """ + if not dcpf_buses_path.exists(): + raise FileNotFoundError(f"DCPF bus CSV not found: {dcpf_buses_path}") + + result: list[dict] = [] + with open(dcpf_buses_path, encoding="utf-8") as f: + reader = csv.DictReader(f) + if reader.fieldnames is None: + raise ValueError(f"DCPF bus CSV is empty: {dcpf_buses_path}") + fields = {fn.strip().lower() for fn in reader.fieldnames} + if "bus" not in fields or "va" not in fields: + raise ValueError(f"Required columns 'bus', 'VA' not found. Got: {reader.fieldnames}") + for row in reader: + result.append( + { + "bus": int(float(row["bus"].strip())), + "VA": float(row["VA"].strip()), + } + ) + return result + + +def load_acpf_branches(acpf_branches_path: Path) -> list[dict]: + """Load ACPF branch reference data from branches_acpf.csv. + + Args: + acpf_branches_path: Path to ``branches_acpf.csv``. + + Returns: + List of dicts with keys ``from_bus`` (int), ``to_bus`` (int), + ``ckt`` (str), ``P_from`` (float), ``Q_from`` (float), + ``P_to`` (float), ``Q_to`` (float). + + Raises: + FileNotFoundError: If the CSV does not exist. + ValueError: If required columns are missing. + """ + if not acpf_branches_path.exists(): + raise FileNotFoundError(f"ACPF branch CSV not found: {acpf_branches_path}") + + result: list[dict] = [] + with open(acpf_branches_path, encoding="utf-8") as f: + reader = csv.DictReader(f) + if reader.fieldnames is None: + raise ValueError(f"ACPF branch CSV is empty: {acpf_branches_path}") + fields = {fn.strip().lower() for fn in reader.fieldnames} + for req in ["from_bus", "to_bus", "ckt", "p_from"]: + if req not in fields: + raise ValueError(f"Required column '{req}' not found. Got: {reader.fieldnames}") + for row in reader: + result.append( + { + "from_bus": int(float(row["from_bus"].strip())), + "to_bus": int(float(row["to_bus"].strip())), + "ckt": row["ckt"].strip(), + "P_from": float(row["P_from"].strip()), + "Q_from": float(row.get("Q_from", "0").strip()), + "P_to": float(row.get("P_to", "0").strip()), + "Q_to": float(row.get("Q_to", "0").strip()), + } + ) + return result + + +def load_dcpf_branches(dcpf_branches_path: Path) -> list[dict]: + """Load DCPF branch reference data from branches_dcpf.csv. + + Args: + dcpf_branches_path: Path to ``branches_dcpf.csv``. + + Returns: + List of dicts with keys ``from_bus`` (int), ``to_bus`` (int), + ``ckt`` (str), ``P_flow_MW`` (float). + + Raises: + FileNotFoundError: If the CSV does not exist. + ValueError: If required columns are missing. + """ + if not dcpf_branches_path.exists(): + raise FileNotFoundError(f"DCPF branch CSV not found: {dcpf_branches_path}") + + result: list[dict] = [] + with open(dcpf_branches_path, encoding="utf-8") as f: + reader = csv.DictReader(f) + if reader.fieldnames is None: + raise ValueError(f"DCPF branch CSV is empty: {dcpf_branches_path}") + fields = {fn.strip().lower() for fn in reader.fieldnames} + for req in ["from_bus", "to_bus", "ckt", "p_flow_mw"]: + if req not in fields: + raise ValueError(f"Required column '{req}' not found. Got: {reader.fieldnames}") + for row in reader: + result.append( + { + "from_bus": int(float(row["from_bus"].strip())), + "to_bus": int(float(row["to_bus"].strip())), + "ckt": row["ckt"].strip(), + "P_flow_MW": float(row["P_flow_MW"].strip()), + } + ) + return result + + +def load_summary_json(summary_path: Path) -> dict: + """Load a summary JSON file (ACPF or DCPF). + + Args: + summary_path: Path to ``summary_acpf.json`` or ``summary_dcpf.json``. + + Returns: + Parsed JSON as a dict. + + Raises: + FileNotFoundError: If the JSON does not exist. + json.JSONDecodeError: If the JSON is malformed. + """ + if not summary_path.exists(): + raise FileNotFoundError(f"Summary JSON not found: {summary_path}") + return json.loads(summary_path.read_text(encoding="utf-8")) + + +def load_intermediate_branches(intermediate_dir: Path) -> list[dict]: + """Load branch/transformer data from the canonical parser's intermediate format. + + Used for cause annotation heuristics (reactance, tap ratio, shift angle, + thermal rating). Auto-detects column names from MATPOWER and GridCal conventions. + + Args: + intermediate_dir: Directory containing the intermediate format CSVs. + + Returns: + List of dicts with keys: ``from_bus``, ``to_bus``, ``ckt``, ``x_pu``, + ``tap_ratio``, ``shift_deg``, ``rate_a_mw`` (or None), ``is_transformer``. + + Raises: + FileNotFoundError: If branch CSV does not exist in the directory. + """ + # Try common filenames + candidates = ["branch.csv", "branches.csv", "branch_data.csv"] + branch_path: Path | None = None + for name in candidates: + p = intermediate_dir / name + if p.exists(): + branch_path = p + break + + if branch_path is None: + raise FileNotFoundError(f"No branch CSV found in {intermediate_dir}. Tried: {candidates}") + + # Column name mappings (canonical -> variants) + from_bus_names = ["f_bus", "from_bus", "i", "fbus"] + to_bus_names = ["t_bus", "to_bus", "j", "tbus"] + x_names = ["br_x", "x"] + tap_names = ["tap", "windv1"] + shift_names = ["shift", "ang1"] + ckt_names = ["ckt", "circuit"] + rate_a_names = ["rate_a", "ratea", "rating_a", "mva_rating"] + + result: list[dict] = [] + with open(branch_path, encoding="utf-8") as f: + reader = csv.DictReader(f) + if reader.fieldnames is None: + return result + + lower_map = {fn.strip().lower(): fn.strip() for fn in reader.fieldnames} + + def _find(variants: list[str]) -> str | None: + for v in variants: + if v.lower() in lower_map: + return lower_map[v.lower()] + return None + + from_col = _find(from_bus_names) + to_col = _find(to_bus_names) + x_col = _find(x_names) + tap_col = _find(tap_names) + shift_col = _find(shift_names) + ckt_col = _find(ckt_names) + rate_a_col = _find(rate_a_names) + + if from_col is None or to_col is None: + raise ValueError( + f"Required columns from_bus/to_bus not found. Got: {reader.fieldnames}" + ) + + for row in reader: + tap_raw = float(row[tap_col].strip()) if tap_col and row.get(tap_col) else 0.0 + tap = tap_raw if tap_raw != 0.0 else 1.0 + shift = float(row[shift_col].strip()) if shift_col and row.get(shift_col) else 0.0 + x = float(row[x_col].strip()) if x_col and row.get(x_col) else 0.0 + + rate_a: float | None = None + if rate_a_col and row.get(rate_a_col): + try: + val = float(row[rate_a_col].strip()) + rate_a = val if val > 0 else None + except (ValueError, TypeError): + pass + + is_transformer = tap != 1.0 or shift != 0.0 + + result.append( + { + "from_bus": int(float(row[from_col].strip())), + "to_bus": int(float(row[to_col].strip())), + "ckt": row[ckt_col].strip() if ckt_col and row.get(ckt_col) else "1", + "x_pu": x, + "tap_ratio": tap, + "shift_deg": shift, + "rate_a_mw": rate_a, + "is_transformer": is_transformer, + } + ) + return result + + +def load_intermediate_buses(intermediate_dir: Path) -> list[dict]: + """Load bus data from the canonical parser's intermediate format. + + Used for cause annotation heuristics (base kV, area number) and for + enriching worst-case bus records. + + Args: + intermediate_dir: Directory containing the intermediate format CSVs. + + Returns: + List of dicts with keys: ``bus`` (int), ``base_kv`` (float), + ``area`` (int), ``bus_type`` (int). + + Raises: + FileNotFoundError: If bus CSV does not exist in the directory. + """ + candidates = ["bus.csv", "buses.csv", "bus_data.csv"] + bus_path: Path | None = None + for name in candidates: + p = intermediate_dir / name + if p.exists(): + bus_path = p + break + + if bus_path is None: + raise FileNotFoundError(f"No bus CSV found in {intermediate_dir}. Tried: {candidates}") + + bus_names = ["bus_i", "bus", "i", "number", "bus_number"] + kv_names = ["base_kv", "baskv", "basekv", "vnom"] + area_names = ["area"] + type_names = ["type", "bus_type", "ide"] + + result: list[dict] = [] + with open(bus_path, encoding="utf-8") as f: + reader = csv.DictReader(f) + if reader.fieldnames is None: + return result + + lower_map = {fn.strip().lower(): fn.strip() for fn in reader.fieldnames} + + def _find(variants: list[str]) -> str | None: + for v in variants: + if v.lower() in lower_map: + return lower_map[v.lower()] + return None + + bus_col = _find(bus_names) + kv_col = _find(kv_names) + area_col = _find(area_names) + type_col = _find(type_names) + + if bus_col is None: + raise ValueError(f"Required column 'bus' not found. Got: {reader.fieldnames}") + + for row in reader: + result.append( + { + "bus": int(float(row[bus_col].strip())), + "base_kv": float(row[kv_col].strip()) if kv_col and row.get(kv_col) else 0.0, + "area": int(float(row[area_col].strip())) + if area_col and row.get(area_col) + else 0, + "bus_type": ( + int(float(row[type_col].strip())) if type_col and row.get(type_col) else 1 + ), + } + ) + return result + + +# --------------------------------------------------------------------------- +# Join operations +# --------------------------------------------------------------------------- + + +def join_buses( + acpf_buses: list[dict], + dcpf_buses: list[dict], +) -> tuple[list[dict], dict[str, int]]: + """Inner-join ACPF and DCPF bus records on bus number. + + Args: + acpf_buses: ACPF bus records (bus, VM, VA). + dcpf_buses: DCPF bus records (bus, VA). + + Returns: + A tuple of: + - Matched records: list of dicts with keys ``bus``, ``VM_acpf``, + ``VA_acpf``, ``VA_dcpf``. + - Join summary: dict with keys ``buses_in_acpf``, ``buses_in_dcpf``, + ``buses_matched``, ``buses_acpf_only``, ``buses_dcpf_only``. + """ + acpf_map: dict[int, dict] = {b["bus"]: b for b in acpf_buses} + dcpf_map: dict[int, dict] = {b["bus"]: b for b in dcpf_buses} + + acpf_keys = set(acpf_map.keys()) + dcpf_keys = set(dcpf_map.keys()) + matched_keys = acpf_keys & dcpf_keys + + matched: list[dict] = [] + for bus_num in sorted(matched_keys): + acpf = acpf_map[bus_num] + dcpf = dcpf_map[bus_num] + matched.append( + { + "bus": bus_num, + "VM_acpf": acpf.get("VM", 1.0), + "VA_acpf": acpf["VA"], + "VA_dcpf": dcpf["VA"], + } + ) + + summary = { + "buses_in_acpf": len(acpf_buses), + "buses_in_dcpf": len(dcpf_buses), + "buses_matched": len(matched), + "buses_acpf_only": len(acpf_keys - dcpf_keys), + "buses_dcpf_only": len(dcpf_keys - acpf_keys), + } + + return matched, summary + + +def _normalize_branch_key(from_bus: int, to_bus: int, ckt: str) -> tuple[int, int, str, bool]: + """Normalize a branch key so (min, max, ckt) is canonical. + + Returns: + (normalized_from, normalized_to, ckt, was_swapped) + """ + if from_bus <= to_bus: + return from_bus, to_bus, ckt, False + return to_bus, from_bus, ckt, True + + +def join_branches( + acpf_branches: list[dict], + dcpf_branches: list[dict], +) -> tuple[list[dict], dict[str, int]]: + """Inner-join ACPF and DCPF branch records on (from_bus, to_bus, ckt). + + Normalizes branch keys to (min(from, to), max(from, to), ckt) before + joining. If from/to are swapped, the DCPF flow sign is negated. + + Args: + acpf_branches: ACPF branch records. + dcpf_branches: DCPF branch records. + + Returns: + A tuple of: + - Matched records: list of dicts with keys ``from_bus``, ``to_bus``, + ``ckt``, ``P_from_acpf``, ``P_flow_dcpf``. + - Join summary: dict with keys ``branches_in_acpf``, ``branches_in_dcpf``, + ``branches_matched``, ``branches_acpf_only``, ``branches_dcpf_only``. + """ + # Build ACPF lookup with normalized keys + acpf_map: dict[tuple[int, int, str], dict] = {} + for b in acpf_branches: + nf, nt, nc, swapped = _normalize_branch_key(b["from_bus"], b["to_bus"], b["ckt"]) + # If swapped, we use P_to (negated) as the "from" direction perspective + p_from = b["P_from"] + acpf_map[(nf, nt, nc)] = { + "from_bus": b["from_bus"], + "to_bus": b["to_bus"], + "ckt": b["ckt"], + "P_from_acpf": p_from, + } + + # Build DCPF lookup with normalized keys + dcpf_map: dict[tuple[int, int, str], dict] = {} + for b in dcpf_branches: + nf, nt, nc, swapped = _normalize_branch_key(b["from_bus"], b["to_bus"], b["ckt"]) + p_flow = -b["P_flow_MW"] if swapped else b["P_flow_MW"] + dcpf_map[(nf, nt, nc)] = { + "from_bus": b["from_bus"], + "to_bus": b["to_bus"], + "ckt": b["ckt"], + "P_flow_dcpf": p_flow, + } + + acpf_keys = set(acpf_map.keys()) + dcpf_keys = set(dcpf_map.keys()) + matched_keys = acpf_keys & dcpf_keys + + matched: list[dict] = [] + for key in sorted(matched_keys): + acpf = acpf_map[key] + dcpf = dcpf_map[key] + matched.append( + { + "from_bus": acpf["from_bus"], + "to_bus": acpf["to_bus"], + "ckt": acpf["ckt"], + "P_from_acpf": acpf["P_from_acpf"], + "P_flow_dcpf": dcpf["P_flow_dcpf"], + } + ) + + summary = { + "branches_in_acpf": len(acpf_branches), + "branches_in_dcpf": len(dcpf_branches), + "branches_matched": len(matched), + "branches_acpf_only": len(acpf_keys - dcpf_keys), + "branches_dcpf_only": len(dcpf_keys - acpf_keys), + } + + return matched, summary + + +# --------------------------------------------------------------------------- +# Deviation computation +# --------------------------------------------------------------------------- + + +def compute_bus_deviations( + matched_buses: list[dict], + intermediate_buses: list[dict], +) -> list[BusDeviation]: + """Compute per-bus angle deviation and enrich with intermediate format data. + + For each matched bus, computes: + - delta_VA_deg = VA_dcpf - VA_acpf (signed) + - abs_delta_VA_deg = |delta_VA_deg| + + Enriches with base_kv and area from the intermediate format bus table + (joined on bus number). If a bus is not found in the intermediate data, + base_kv defaults to 0.0 and area defaults to 0. + + Args: + matched_buses: Output of ``join_buses`` (matched records). + intermediate_buses: Output of ``load_intermediate_buses``. + + Returns: + List of BusDeviation records, one per matched bus. + """ + int_bus_map: dict[int, dict] = {b["bus"]: b for b in intermediate_buses} + + result: list[BusDeviation] = [] + for m in matched_buses: + delta = m["VA_dcpf"] - m["VA_acpf"] + int_data = int_bus_map.get(m["bus"], {}) + result.append( + BusDeviation( + bus=m["bus"], + VA_acpf_deg=m["VA_acpf"], + VA_dcpf_deg=m["VA_dcpf"], + delta_VA_deg=delta, + abs_delta_VA_deg=abs(delta), + VM_acpf_pu=m.get("VM_acpf", 1.0), + base_kv=int_data.get("base_kv", 0.0), + area=int_data.get("area", 0), + ) + ) + return result + + +def compute_branch_deviations( + matched_branches: list[dict], + intermediate_branches: list[dict], +) -> list[BranchDeviation]: + """Compute per-branch flow deviation and enrich with intermediate format data. + + For each matched branch, computes: + - delta_P_MW = P_flow_dcpf - P_from_acpf (signed) + - abs_delta_P_MW = |delta_P_MW| + - delta_P_pct = delta_P_MW / |P_from_acpf| * 100 (if |P_from_acpf| > threshold) + - abs_delta_P_pct = |delta_P_pct| (if applicable, else None) + + Enriches with x_pu, tap_ratio, shift_deg, is_transformer from the + intermediate format branch table (joined on from_bus, to_bus, ckt). + + Args: + matched_branches: Output of ``join_branches`` (matched records). + intermediate_branches: Output of ``load_intermediate_branches``. + + Returns: + List of BranchDeviation records, one per matched branch. + """ + # Build intermediate branch lookup with normalized keys + int_br_map: dict[tuple[int, int, str], dict] = {} + for b in intermediate_branches: + nf, nt, nc, _ = _normalize_branch_key(b["from_bus"], b["to_bus"], b["ckt"]) + int_br_map[(nf, nt, nc)] = b + + result: list[BranchDeviation] = [] + for m in matched_branches: + delta_mw = m["P_flow_dcpf"] - m["P_from_acpf"] + abs_delta_mw = abs(delta_mw) + + abs_p_acpf = abs(m["P_from_acpf"]) + if abs_p_acpf > NEAR_ZERO_FLOW_THRESHOLD_MW: + delta_pct = delta_mw / abs_p_acpf * 100.0 + abs_delta_pct: float | None = abs(delta_pct) + else: + delta_pct = None + abs_delta_pct = None + + nf, nt, nc, _ = _normalize_branch_key(m["from_bus"], m["to_bus"], m["ckt"]) + int_data = int_br_map.get((nf, nt, nc), {}) + + result.append( + BranchDeviation( + from_bus=m["from_bus"], + to_bus=m["to_bus"], + ckt=m["ckt"], + P_from_acpf_MW=m["P_from_acpf"], + P_flow_dcpf_MW=m["P_flow_dcpf"], + delta_P_MW=delta_mw, + abs_delta_P_MW=abs_delta_mw, + delta_P_pct=delta_pct, + abs_delta_P_pct=abs_delta_pct, + x_pu=int_data.get("x_pu", 0.0), + tap_ratio=int_data.get("tap_ratio", 1.0), + shift_deg=int_data.get("shift_deg", 0.0), + is_transformer=int_data.get("is_transformer", False), + ) + ) + return result + + +# --------------------------------------------------------------------------- +# Aggregate statistics +# --------------------------------------------------------------------------- + + +def _percentile(sorted_values: list[float], pct: float) -> float: + """Compute the pth percentile using linear interpolation. + + Args: + sorted_values: Sorted list of values. + pct: Percentile to compute (0-100). + + Returns: + The percentile value. + """ + n = len(sorted_values) + if n == 1: + return sorted_values[0] + + # Use the 'linear interpolation' method (same as numpy default) + k = (pct / 100.0) * (n - 1) + f = math.floor(k) + c = math.ceil(k) + if f == c: + return sorted_values[int(k)] + d = k - f + return sorted_values[int(f)] * (1 - d) + sorted_values[int(c)] * d + + +def compute_aggregate_stats(values: list[float]) -> AggregateStats: + """Compute aggregate statistics for a list of numeric values. + + Args: + values: Non-empty list of float values. + + Returns: + AggregateStats with mean, median, std, min, max, p05, p95. + + Raises: + ValueError: If values is empty. + """ + if not values: + raise ValueError("Cannot compute aggregate statistics for an empty list.") + + n = len(values) + mean_val = statistics.mean(values) + median_val = statistics.median(values) + + if n >= 2: + std_val = statistics.stdev(values) + else: + std_val = 0.0 + + sorted_vals = sorted(values) + min_val = sorted_vals[0] + max_val = sorted_vals[-1] + p05 = _percentile(sorted_vals, 5.0) + p95 = _percentile(sorted_vals, 95.0) + + return AggregateStats( + count=n, + mean=mean_val, + median=median_val, + std=std_val, + min=min_val, + max=max_val, + p05=p05, + p95=p95, + ) + + +def compute_compliance_fractions( + values: list[float], + thresholds: list[float], +) -> ComplianceFractions: + """Compute the fraction of values at or below each threshold. + + Args: + values: List of absolute (non-negative) deviation values. + thresholds: Sorted list of threshold values. + + Returns: + ComplianceFractions with fraction (0.0 to 1.0) at each threshold. + """ + n = len(values) + if n == 0: + return ComplianceFractions( + thresholds=list(thresholds), + fractions=[0.0] * len(thresholds), + ) + + sorted_vals = sorted(values) + fractions: list[float] = [] + for threshold in thresholds: + count = sum(1 for v in sorted_vals if v <= threshold) + fractions.append(count / n) + + return ComplianceFractions( + thresholds=list(thresholds), + fractions=fractions, + ) + + +# --------------------------------------------------------------------------- +# Cause annotation +# --------------------------------------------------------------------------- + + +def _build_bus_adjacency(intermediate_branches: list[dict]) -> dict[int, set[int]]: + """Build an adjacency dict from intermediate branch data. + + Args: + intermediate_branches: Intermediate format branch records. + + Returns: + Dict mapping bus number to set of neighboring bus numbers. + """ + adj: dict[int, set[int]] = {} + for b in intermediate_branches: + fb = b["from_bus"] + tb = b["to_bus"] + if fb not in adj: + adj[fb] = set() + if tb not in adj: + adj[tb] = set() + adj[fb].add(tb) + adj[tb].add(fb) + return adj + + +def _buses_within_n_hops( + start: int, + adjacency: dict[int, set[int]], + max_hops: int, +) -> set[int]: + """Find all buses within N topological hops of the start bus. + + Args: + start: Starting bus number. + adjacency: Adjacency dict. + max_hops: Maximum number of hops. + + Returns: + Set of bus numbers within max_hops (including the start bus). + """ + visited: set[int] = {start} + frontier: set[int] = {start} + for _ in range(max_hops): + next_frontier: set[int] = set() + for bus in frontier: + for neighbor in adjacency.get(bus, set()): + if neighbor not in visited: + visited.add(neighbor) + next_frontier.add(neighbor) + frontier = next_frontier + if not frontier: + break + return visited + + +def annotate_bus_causes( + bus_deviations: list[BusDeviation], + slack_bus: int, + bus_adjacency: dict[int, set[int]], +) -> list[BusDeviation]: + """Annotate each bus deviation with probable cause categories. + + Applies heuristics: + - low_voltage: VM_acpf < 0.95 + - high_voltage: VM_acpf > 1.05 + - slack_bus_vicinity: bus is within 2 hops of the slack bus in bus_adjacency + + Args: + bus_deviations: List of BusDeviation records (causes field empty). + slack_bus: Slack bus number from the ACPF summary. + bus_adjacency: Adjacency dict (bus -> set of neighbor buses) from + the intermediate format branch table. + + Returns: + New list of BusDeviation records with causes populated. + """ + slack_vicinity = _buses_within_n_hops(slack_bus, bus_adjacency, 2) + + result: list[BusDeviation] = [] + for bd in bus_deviations: + causes: list[DeviationCause] = [] + + if bd.VM_acpf_pu < 0.95: + causes.append(DeviationCause.LOW_VOLTAGE) + if bd.VM_acpf_pu > 1.05: + causes.append(DeviationCause.HIGH_VOLTAGE) + if bd.bus in slack_vicinity: + causes.append(DeviationCause.SLACK_BUS_VICINITY) + + if not causes: + causes.append(DeviationCause.UNCATEGORIZED) + + result.append( + BusDeviation( + bus=bd.bus, + VA_acpf_deg=bd.VA_acpf_deg, + VA_dcpf_deg=bd.VA_dcpf_deg, + delta_VA_deg=bd.delta_VA_deg, + abs_delta_VA_deg=bd.abs_delta_VA_deg, + VM_acpf_pu=bd.VM_acpf_pu, + base_kv=bd.base_kv, + area=bd.area, + causes=causes, + ) + ) + return result + + +def annotate_branch_causes( + branch_deviations: list[BranchDeviation], + acpf_buses: list[dict], + intermediate_branches: list[dict], +) -> list[BranchDeviation]: + """Annotate each branch deviation with probable cause categories. + + Applies heuristics (in priority order): + - phase_shifter: shift_deg != 0 + - high_reactance: x_pu > 0.5 + - heavy_loading: |P_from_acpf| > 0.8 * rate_a (if rate_a available) + - low_voltage: VM at either end-bus < 0.95 + - high_voltage: VM at either end-bus > 1.05 + - transformer_tap: is_transformer and tap_ratio != 1.0 + + Args: + branch_deviations: List of BranchDeviation records (causes field empty). + acpf_buses: ACPF bus records (for VM lookup by bus number). + intermediate_branches: Intermediate format branch data (for rate_a lookup). + + Returns: + New list of BranchDeviation records with causes populated. + """ + vm_map: dict[int, float] = {b["bus"]: b.get("VM", 1.0) for b in acpf_buses} + + # Build rate_a lookup with normalized keys + rate_a_map: dict[tuple[int, int, str], float | None] = {} + for b in intermediate_branches: + nf, nt, nc, _ = _normalize_branch_key(b["from_bus"], b["to_bus"], b["ckt"]) + rate_a_map[(nf, nt, nc)] = b.get("rate_a_mw") + + result: list[BranchDeviation] = [] + for bd in branch_deviations: + causes: list[DeviationCause] = [] + + # Phase shifter + if bd.shift_deg != 0.0: + causes.append(DeviationCause.PHASE_SHIFTER) + + # High reactance + if bd.x_pu > 0.5: + causes.append(DeviationCause.HIGH_REACTANCE) + + # Heavy loading + nf, nt, nc, _ = _normalize_branch_key(bd.from_bus, bd.to_bus, bd.ckt) + rate_a = rate_a_map.get((nf, nt, nc)) + if rate_a is not None and rate_a > 0: + if abs(bd.P_from_acpf_MW) > 0.8 * rate_a: + causes.append(DeviationCause.HEAVY_LOADING) + + # Low/high voltage at either end-bus + vm_from = vm_map.get(bd.from_bus, 1.0) + vm_to = vm_map.get(bd.to_bus, 1.0) + if vm_from < 0.95 or vm_to < 0.95: + causes.append(DeviationCause.LOW_VOLTAGE) + if vm_from > 1.05 or vm_to > 1.05: + causes.append(DeviationCause.HIGH_VOLTAGE) + + # Transformer tap + if bd.is_transformer and bd.tap_ratio != 1.0: + causes.append(DeviationCause.TRANSFORMER_TAP) + + if not causes: + causes.append(DeviationCause.UNCATEGORIZED) + + result.append( + BranchDeviation( + from_bus=bd.from_bus, + to_bus=bd.to_bus, + ckt=bd.ckt, + P_from_acpf_MW=bd.P_from_acpf_MW, + P_flow_dcpf_MW=bd.P_flow_dcpf_MW, + delta_P_MW=bd.delta_P_MW, + abs_delta_P_MW=bd.abs_delta_P_MW, + delta_P_pct=bd.delta_P_pct, + abs_delta_P_pct=bd.abs_delta_P_pct, + x_pu=bd.x_pu, + tap_ratio=bd.tap_ratio, + shift_deg=bd.shift_deg, + is_transformer=bd.is_transformer, + causes=causes, + ) + ) + return result + + +# --------------------------------------------------------------------------- +# Worst-case extraction +# --------------------------------------------------------------------------- + + +def extract_worst_buses( + bus_deviations: list[BusDeviation], + count: int = WORST_CASE_COUNT, +) -> list[BusDeviation]: + """Return the top N buses by absolute angle deviation, descending. + + Args: + bus_deviations: All bus deviations (with causes annotated). + count: Number of worst-case buses to return. + + Returns: + List of up to ``count`` BusDeviation records, sorted by + abs_delta_VA_deg descending. + """ + sorted_devs = sorted(bus_deviations, key=lambda d: d.abs_delta_VA_deg, reverse=True) + return sorted_devs[:count] + + +def extract_worst_branches( + branch_deviations: list[BranchDeviation], + count: int = WORST_CASE_COUNT, +) -> list[BranchDeviation]: + """Return the top N branches by absolute percentage flow deviation, descending. + + Only considers branches with non-null abs_delta_P_pct (i.e., those + above the near-zero flow threshold). + + Args: + branch_deviations: All branch deviations (with causes annotated). + count: Number of worst-case branches to return. + + Returns: + List of up to ``count`` BranchDeviation records, sorted by + abs_delta_P_pct descending. + """ + eligible = [bd for bd in branch_deviations if bd.abs_delta_P_pct is not None] + sorted_devs = sorted( + eligible, + key=lambda d: d.abs_delta_P_pct if d.abs_delta_P_pct is not None else 0.0, + reverse=True, + ) + return sorted_devs[:count] + + +# --------------------------------------------------------------------------- +# Report writing +# --------------------------------------------------------------------------- + + +def _stats_to_dict(stats: AggregateStats) -> dict: + """Convert AggregateStats to a JSON-serializable dict.""" + return { + "mean": round(stats.mean, 6), + "median": round(stats.median, 6), + "std": round(stats.std, 6), + "min": round(stats.min, 6), + "max": round(stats.max, 6), + "p05": round(stats.p05, 6), + "p95": round(stats.p95, 6), + } + + +def _compliance_to_dict( + comp: ComplianceFractions, + label_fmt: str, +) -> dict: + """Convert ComplianceFractions to a JSON dict with threshold-based keys. + + Args: + comp: Compliance fractions. + label_fmt: Format string for keys, e.g. "pct_within_{}_deg". + The ``{}`` is replaced with the threshold value formatted + with underscores for decimals (e.g., 0.5 -> "0_5"). + + Returns: + Dict mapping threshold label to percentage (0-100). + """ + result: dict[str, float] = {} + for threshold, fraction in zip(comp.thresholds, comp.fractions): + label = label_fmt.format(str(threshold).replace(".", "_")) + result[label] = round(fraction * 100.0, 4) + return result + + +def write_characterization_json( + result: CharacterizationResult, + output_path: Path, +) -> None: + """Write the characterization report as JSON. + + Serializes the CharacterizationResult into the JSON schema defined + in the Data Structures section. + + Args: + result: Complete characterization result. + output_path: Full path to the output JSON file. + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + + # Build angle deviation section + angle_dev = { + "unit": "degrees", + "count": result.angle_stats_absolute.count, + "signed": _stats_to_dict(result.angle_stats_signed), + "absolute": _stats_to_dict(result.angle_stats_absolute), + "compliance": _compliance_to_dict(result.angle_compliance, "pct_within_{}_deg"), + } + + # Build flow deviation MW section + flow_mw = { + "unit": "MW", + "count": result.flow_mw_stats_absolute.count, + "signed": _stats_to_dict(result.flow_mw_stats_signed), + "absolute": _stats_to_dict(result.flow_mw_stats_absolute), + } + + # Count near-zero flow branches + near_zero_count = sum(1 for bd in result.branch_deviations if bd.abs_delta_P_pct is None) + + # Build flow deviation pct section + flow_pct = { + "unit": "percent", + "count": result.flow_pct_stats_absolute.count, + "excluded_near_zero_flow": near_zero_count, + "near_zero_flow_threshold_mw": NEAR_ZERO_FLOW_THRESHOLD_MW, + "signed": _stats_to_dict(result.flow_pct_stats_signed), + "absolute": _stats_to_dict(result.flow_pct_stats_absolute), + "compliance": _compliance_to_dict(result.flow_pct_compliance, "pct_within_{}_pct"), + } + + # Build worst buses + worst_buses = [] + for bd in result.worst_buses: + worst_buses.append( + { + "bus": bd.bus, + "abs_delta_VA_deg": round(bd.abs_delta_VA_deg, 6), + "VA_acpf_deg": round(bd.VA_acpf_deg, 6), + "VA_dcpf_deg": round(bd.VA_dcpf_deg, 6), + "VM_acpf_pu": round(bd.VM_acpf_pu, 6), + "base_kv": round(bd.base_kv, 2), + "area": bd.area, + "primary_cause": bd.causes[0].value if bd.causes else "uncategorized", + "all_causes": [c.value for c in bd.causes], + } + ) + + # Build worst branches + worst_branches = [] + for bd in result.worst_branches: + worst_branches.append( + { + "from_bus": bd.from_bus, + "to_bus": bd.to_bus, + "ckt": bd.ckt, + "abs_delta_P_pct": ( + round(bd.abs_delta_P_pct, 6) if bd.abs_delta_P_pct is not None else None + ), + "abs_delta_P_MW": round(bd.abs_delta_P_MW, 6), + "P_from_acpf_MW": round(bd.P_from_acpf_MW, 6), + "P_flow_dcpf_MW": round(bd.P_flow_dcpf_MW, 6), + "x_pu": round(bd.x_pu, 6), + "tap_ratio": round(bd.tap_ratio, 6), + "shift_deg": round(bd.shift_deg, 6), + "is_transformer": bd.is_transformer, + "primary_cause": bd.causes[0].value if bd.causes else "uncategorized", + "all_causes": [c.value for c in bd.causes], + } + ) + + data = { + "metadata": result.metadata, + "join_summary": result.join_summary, + "system_level": { + k: round(v, 6) if isinstance(v, float) else v for k, v in result.system_level.items() + }, + "angle_deviation": angle_dev, + "flow_deviation_mw": flow_mw, + "flow_deviation_pct": flow_pct, + "expected_range_checks": result.expected_range_checks, + "worst_buses": worst_buses, + "worst_branches": worst_branches, + "warnings": result.warnings, + } + + output_path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + + +def write_characterization_md( + result: CharacterizationResult, + output_path: Path, +) -> None: + """Write the characterization report as human-readable markdown. + + Produces the markdown structure: Summary, System-Level Comparison, + Angle Deviation Distribution, Flow Deviation Distribution, + Expected Range Checks, Worst-Case Buses, Worst-Case Branches, + Methodology Notes. + + Args: + result: Complete characterization result. + output_path: Full path to the output markdown file. + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + + lines: list[str] = [] + lines.append("# DCPF-vs-ACPF Characterization Report\n") + + # Summary + lines.append("## Summary\n") + js = result.join_summary + lines.append( + f"Compared {js.get('buses_matched', 0)} matched buses and " + f"{js.get('branches_matched', 0)} matched branches. " + ) + + angle_check = result.expected_range_checks.get("angle_95pct_within_3deg", {}) + flow_check = result.expected_range_checks.get("flow_90pct_within_10pct", {}) + angle_met = angle_check.get("met", False) + flow_met = flow_check.get("met", False) + + lines.append( + f"Angle expected-range check (>95% within 3 deg): " + f"{'PASSED' if angle_met else 'WARNING'}. " + f"Flow expected-range check (>90% within 10%): " + f"{'PASSED' if flow_met else 'WARNING'}.\n" + ) + + # System-Level Comparison + lines.append("## System-Level Comparison\n") + sl = result.system_level + lines.append("| Metric | ACPF | DCPF |") + lines.append("|--------|------|------|") + lines.append( + f"| Total Generation (MW) | {sl.get('acpf_total_gen_mw', 'N/A'):.2f} | " + f"{sl.get('dcpf_total_gen_mw', 'N/A'):.2f} |" + ) + lines.append( + f"| Total Load (MW) | {sl.get('acpf_total_load_mw', 'N/A'):.2f} | " + f"{sl.get('dcpf_total_load_mw', 'N/A'):.2f} |" + ) + lines.append( + f"| Total Losses (MW) | {sl.get('acpf_total_loss_mw', 'N/A'):.2f} | N/A (lossless) |" + ) + lines.append(f"| Loss % of Generation | {sl.get('acpf_loss_pct_of_gen', 0):.2f}% | N/A |") + lines.append("") + + # Angle Deviation Distribution + lines.append("## Angle Deviation Distribution\n") + lines.append("### Aggregate Statistics (absolute)\n") + _append_stats_table(lines, result.angle_stats_absolute, "degrees") + lines.append("### Compliance Fractions\n") + _append_compliance_table(lines, result.angle_compliance, "deg") + + # Flow Deviation Distribution + lines.append("## Flow Deviation Distribution\n") + lines.append("### Aggregate Statistics - MW (absolute)\n") + _append_stats_table(lines, result.flow_mw_stats_absolute, "MW") + lines.append("### Aggregate Statistics - Percent (absolute)\n") + _append_stats_table(lines, result.flow_pct_stats_absolute, "%") + lines.append("### Compliance Fractions\n") + _append_compliance_table(lines, result.flow_pct_compliance, "%") + + # Expected Range Checks + lines.append("## Expected Range Checks\n") + for name, check in result.expected_range_checks.items(): + status = "PASSED" if check.get("met", False) else "WARNING" + actual = check.get("actual_pct", 0) + lines.append(f"- **{name}**: {status} (actual: {actual:.2f}%)") + lines.append("") + + # Worst-Case Buses + lines.append("## Worst-Case Buses (Top 50 by angle deviation)\n") + lines.append("| Bus | abs_dVA (deg) | VA_acpf | VA_dcpf | VM_acpf | kV | Cause |") + lines.append("|-----|---------------|---------|---------|---------|-----|-------|") + for bd in result.worst_buses: + cause = bd.causes[0].value if bd.causes else "uncategorized" + lines.append( + f"| {bd.bus} | {bd.abs_delta_VA_deg:.4f} | {bd.VA_acpf_deg:.4f} | " + f"{bd.VA_dcpf_deg:.4f} | {bd.VM_acpf_pu:.4f} | {bd.base_kv:.1f} | {cause} |" + ) + lines.append("") + + # Worst-Case Branches + lines.append("## Worst-Case Branches (Top 50 by percentage flow deviation)\n") + lines.append("| From | To | Ckt | abs_dP% | abs_dP_MW | P_acpf | P_dcpf | Cause |") + lines.append("|------|-----|-----|---------|-----------|--------|--------|-------|") + for bd in result.worst_branches: + cause = bd.causes[0].value if bd.causes else "uncategorized" + pct_str = f"{bd.abs_delta_P_pct:.2f}" if bd.abs_delta_P_pct is not None else "N/A" + lines.append( + f"| {bd.from_bus} | {bd.to_bus} | {bd.ckt} | {pct_str} | " + f"{bd.abs_delta_P_MW:.4f} | {bd.P_from_acpf_MW:.4f} | " + f"{bd.P_flow_dcpf_MW:.4f} | {cause} |" + ) + lines.append("") + + # Methodology Notes + lines.append("## Methodology Notes\n") + lines.append( + "- **Join strategy**: Inner join on bus number (buses) and " + "(from_bus, to_bus, ckt) normalized to (min, max, ckt) for branches.\n" + "- **Near-zero flow exclusion**: Branches with |P_from_acpf| <= " + f"{NEAR_ZERO_FLOW_THRESHOLD_MW} MW excluded from percentage metrics.\n" + "- **Cause annotation**: Rule-based heuristics applied in priority order: " + "phase_shifter, high_reactance, heavy_loading, low_voltage, high_voltage, " + "transformer_tap, slack_bus_vicinity, uncategorized.\n" + ) + + output_path.write_text("\n".join(lines), encoding="utf-8") + + +def _append_stats_table(lines: list[str], stats: AggregateStats, unit: str) -> None: + """Append a statistics table to the markdown lines.""" + lines.append(f"| Statistic | Value ({unit}) |") + lines.append("|-----------|---------------|") + lines.append(f"| Count | {stats.count} |") + lines.append(f"| Mean | {stats.mean:.6f} |") + lines.append(f"| Median | {stats.median:.6f} |") + lines.append(f"| Std Dev | {stats.std:.6f} |") + lines.append(f"| P05 | {stats.p05:.6f} |") + lines.append(f"| P95 | {stats.p95:.6f} |") + lines.append(f"| Max | {stats.max:.6f} |") + lines.append("") + + +def _append_compliance_table(lines: list[str], comp: ComplianceFractions, unit: str) -> None: + """Append a compliance fractions table to the markdown lines.""" + lines.append(f"| Threshold ({unit}) | % Within |") + lines.append("|-------------------|----------|") + for threshold, fraction in zip(comp.thresholds, comp.fractions): + lines.append(f"| {threshold} | {fraction * 100:.2f}% |") + lines.append("") + + +# --------------------------------------------------------------------------- +# Orchestrator +# --------------------------------------------------------------------------- + + +def build_characterization( + acpf_dir: Path, + dcpf_dir: Path, + intermediate_dir: Path, + output_dir: Path, +) -> Path: + """Top-level orchestrator for DCPF-vs-ACPF characterization. + + Steps: + 1. Load ACPF and DCPF bus and branch CSVs. + 2. Load ACPF and DCPF summary JSONs. + 3. Load intermediate format bus and branch data for cause annotations. + 4. Join buses on bus number; join branches on (from_bus, to_bus, ckt). + 5. Compute per-bus angle deviations and per-branch flow deviations. + 6. Compute aggregate statistics and compliance fractions. + 7. Check expected-range thresholds; log warnings if not met. + 8. Annotate worst-case elements with probable causes. + 9. Extract top-50 worst buses and branches. + 10. Build CharacterizationResult. + 11. Write JSON and markdown reports to output_dir. + + Args: + acpf_dir: Directory containing ACPF reference files + (``buses_acpf.csv``, ``branches_acpf.csv``, ``summary_acpf.json``). + dcpf_dir: Directory containing DCPF reference files + (``buses_dcpf.csv``, ``branches_dcpf.csv``, ``summary_dcpf.json``). + intermediate_dir: Directory containing intermediate format CSVs + from the canonical parser. + output_dir: Output directory for characterization report files. + Created if it does not exist. + + Returns: + Path to the output directory containing the JSON and markdown reports. + + Raises: + FileNotFoundError: If any required input file is missing. + ValueError: If join produces zero matched buses or branches. + """ + # 1. Load data + acpf_buses = load_acpf_buses(acpf_dir / "buses_acpf.csv") + dcpf_buses = load_dcpf_buses(dcpf_dir / "buses_dcpf.csv") + acpf_branches = load_acpf_branches(acpf_dir / "branches_acpf.csv") + dcpf_branches = load_dcpf_branches(dcpf_dir / "branches_dcpf.csv") + + # 2. Load summaries + acpf_summary = load_summary_json(acpf_dir / "summary_acpf.json") + dcpf_summary = load_summary_json(dcpf_dir / "summary_dcpf.json") + + # 3. Load intermediate data (gracefully handle missing) + try: + intermediate_buses = load_intermediate_buses(intermediate_dir) + except (FileNotFoundError, ValueError): + logger.warning("Could not load intermediate bus data; using empty list.") + intermediate_buses = [] + + try: + intermediate_branches = load_intermediate_branches(intermediate_dir) + except (FileNotFoundError, ValueError): + logger.warning("Could not load intermediate branch data; using empty list.") + intermediate_branches = [] + + # 4. Join + matched_buses, bus_join_summary = join_buses(acpf_buses, dcpf_buses) + matched_branches, branch_join_summary = join_branches(acpf_branches, dcpf_branches) + + if not matched_buses: + raise ValueError("Join produced zero matched buses.") + if not matched_branches: + raise ValueError("Join produced zero matched branches.") + + # Check for all-zero DCPF angles + all_dcpf_angles = [m["VA_dcpf"] for m in matched_buses] + if all(a == 0.0 for a in all_dcpf_angles): + raise ValueError("All DCPF angles are 0.0, indicating a solver failure or empty network.") + + # Log unmatched warnings + warnings: list[str] = [] + total_acpf_buses = bus_join_summary["buses_in_acpf"] + unmatched_buses = bus_join_summary["buses_acpf_only"] + bus_join_summary["buses_dcpf_only"] + if total_acpf_buses > 0 and unmatched_buses / total_acpf_buses > 0.01: + msg = ( + f"Unmatched bus count ({unmatched_buses}) exceeds 1% of ACPF buses " + f"({total_acpf_buses}). ACPF-only: {bus_join_summary['buses_acpf_only']}, " + f"DCPF-only: {bus_join_summary['buses_dcpf_only']}." + ) + warnings.append(msg) + logger.warning(msg) + + # Merge join summaries + join_summary = {**bus_join_summary, **branch_join_summary} + + # 5. Compute deviations + bus_devs = compute_bus_deviations(matched_buses, intermediate_buses) + branch_devs = compute_branch_deviations(matched_branches, intermediate_branches) + + # 6. Aggregate statistics + signed_angles = [bd.delta_VA_deg for bd in bus_devs] + abs_angles = [bd.abs_delta_VA_deg for bd in bus_devs] + angle_stats_signed = compute_aggregate_stats(signed_angles) + angle_stats_absolute = compute_aggregate_stats(abs_angles) + angle_compliance = compute_compliance_fractions(abs_angles, ANGLE_COMPLIANCE_THRESHOLDS_DEG) + + signed_mw = [bd.delta_P_MW for bd in branch_devs] + abs_mw = [bd.abs_delta_P_MW for bd in branch_devs] + flow_mw_stats_signed = compute_aggregate_stats(signed_mw) + flow_mw_stats_absolute = compute_aggregate_stats(abs_mw) + + # Percentage stats only for non-near-zero branches + signed_pct = [bd.delta_P_pct for bd in branch_devs if bd.delta_P_pct is not None] + abs_pct = [bd.abs_delta_P_pct for bd in branch_devs if bd.abs_delta_P_pct is not None] + + if signed_pct: + flow_pct_stats_signed = compute_aggregate_stats(signed_pct) + else: + flow_pct_stats_signed = AggregateStats(0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0) + + if abs_pct: + flow_pct_stats_absolute = compute_aggregate_stats(abs_pct) + else: + flow_pct_stats_absolute = AggregateStats(0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0) + + flow_pct_compliance = compute_compliance_fractions(abs_pct, FLOW_COMPLIANCE_THRESHOLDS_PCT) + + # 7. Expected-range checks + angle_3deg_compliance = ( + angle_compliance.fractions[ANGLE_COMPLIANCE_THRESHOLDS_DEG.index(3.0)] * 100.0 + ) + angle_check_met = angle_3deg_compliance >= EXPECTED_ANGLE_COMPLIANCE_PCT + + if abs_pct: + flow_10pct_compliance = ( + flow_pct_compliance.fractions[FLOW_COMPLIANCE_THRESHOLDS_PCT.index(10.0)] * 100.0 + ) + else: + flow_10pct_compliance = 0.0 + flow_check_met = flow_10pct_compliance >= EXPECTED_FLOW_COMPLIANCE_PCT + + expected_range_checks = { + "angle_95pct_within_3deg": { + "threshold_pct": EXPECTED_ANGLE_COMPLIANCE_PCT, + "threshold_deg": EXPECTED_ANGLE_THRESHOLD_DEG, + "actual_pct": round(angle_3deg_compliance, 4), + "met": angle_check_met, + }, + "flow_90pct_within_10pct": { + "threshold_pct": EXPECTED_FLOW_COMPLIANCE_PCT, + "threshold_flow_pct": EXPECTED_FLOW_THRESHOLD_PCT, + "actual_pct": round(flow_10pct_compliance, 4), + "met": flow_check_met, + }, + } + + if not angle_check_met: + msg = ( + f"Expected >95% of buses within 3 degrees, but only " + f"{angle_3deg_compliance:.2f}% met the threshold." + ) + warnings.append(msg) + logger.warning(msg) + + if not flow_check_met: + msg = ( + f"Expected >90% of branches within 10%, but only " + f"{flow_10pct_compliance:.2f}% met the threshold." + ) + warnings.append(msg) + logger.warning(msg) + + # 8. Annotate causes + bus_adjacency = _build_bus_adjacency(intermediate_branches) + slack_bus = _extract_slack_bus(acpf_summary, dcpf_summary) + bus_devs = annotate_bus_causes(bus_devs, slack_bus, bus_adjacency) + branch_devs = annotate_branch_causes(branch_devs, acpf_buses, intermediate_branches) + + # 9. Extract worst-case + worst_buses = extract_worst_buses(bus_devs) + worst_branches = extract_worst_branches(branch_devs) + + # 10. System-level comparison + system_level = _build_system_level(acpf_summary, dcpf_summary) + + # Build metadata + metadata = { + "acpf_summary_path": str(acpf_dir / "summary_acpf.json"), + "dcpf_summary_path": str(dcpf_dir / "summary_dcpf.json"), + "acpf_buses_path": str(acpf_dir / "buses_acpf.csv"), + "dcpf_buses_path": str(dcpf_dir / "buses_dcpf.csv"), + "acpf_branches_path": str(acpf_dir / "branches_acpf.csv"), + "dcpf_branches_path": str(dcpf_dir / "branches_dcpf.csv"), + "intermediate_dir": str(intermediate_dir), + "timestamp": datetime.now(timezone.utc).isoformat(), + } + + # 10. Build result + char_result = CharacterizationResult( + bus_deviations=bus_devs, + branch_deviations=branch_devs, + angle_stats_signed=angle_stats_signed, + angle_stats_absolute=angle_stats_absolute, + angle_compliance=angle_compliance, + flow_mw_stats_signed=flow_mw_stats_signed, + flow_mw_stats_absolute=flow_mw_stats_absolute, + flow_pct_stats_signed=flow_pct_stats_signed, + flow_pct_stats_absolute=flow_pct_stats_absolute, + flow_pct_compliance=flow_pct_compliance, + join_summary=join_summary, + system_level=system_level, + expected_range_checks=expected_range_checks, + worst_buses=worst_buses, + worst_branches=worst_branches, + warnings=warnings, + metadata=metadata, + ) + + # 11. Write reports + output_dir.mkdir(parents=True, exist_ok=True) + write_characterization_json(char_result, output_dir / "dcpf_vs_acpf_characterization.json") + write_characterization_md(char_result, output_dir / "dcpf_vs_acpf_characterization.md") + + logger.info("Characterization reports written to %s", output_dir) + return output_dir + + +def _extract_slack_bus(acpf_summary: dict, dcpf_summary: dict) -> int: + """Extract the slack bus number from the ACPF or DCPF summary. + + Args: + acpf_summary: Parsed ACPF summary JSON. + dcpf_summary: Parsed DCPF summary JSON. + + Returns: + Slack bus number. + """ + # Try ACPF summary first + sys_summary = acpf_summary.get("system_summary", {}) + slack = sys_summary.get("slack_bus") + if slack is not None: + return int(slack) + + # Try DCPF summary + settings = dcpf_summary.get("settings", {}) + slack = settings.get("slack_bus") + if slack is not None: + return int(slack) + + # Default to bus 1 + logger.warning("Could not determine slack bus from summaries; defaulting to bus 1.") + return 1 + + +def _build_system_level(acpf_summary: dict, dcpf_summary: dict) -> dict[str, float]: + """Build system-level comparison dict from summaries. + + Args: + acpf_summary: Parsed ACPF summary JSON. + dcpf_summary: Parsed DCPF summary JSON. + + Returns: + Dict with system-level comparison values. + """ + acpf_sys = acpf_summary.get("system_summary", {}) + dcpf_power = dcpf_summary.get("power_summary", {}) + + acpf_gen = float(acpf_sys.get("total_gen_mw", 0)) + acpf_load = float(acpf_sys.get("total_load_mw", 0)) + acpf_loss = float(acpf_sys.get("total_loss_mw", 0)) + dcpf_gen = float(dcpf_power.get("total_generation_mw", 0)) + dcpf_load = float(dcpf_power.get("total_load_mw", 0)) + + acpf_slack = int(acpf_sys.get("slack_bus", 0)) + dcpf_slack = int(dcpf_summary.get("settings", {}).get("slack_bus", 0)) + + loss_pct = (acpf_loss / acpf_gen * 100.0) if acpf_gen > 0 else 0.0 + + return { + "acpf_total_gen_mw": acpf_gen, + "dcpf_total_gen_mw": dcpf_gen, + "acpf_total_load_mw": acpf_load, + "dcpf_total_load_mw": dcpf_load, + "acpf_total_loss_mw": acpf_loss, + "acpf_loss_pct_of_gen": loss_pct, + "acpf_slack_bus": float(acpf_slack), + "dcpf_slack_bus": float(dcpf_slack), + } + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> None: + """CLI entry point for DCPF-vs-ACPF characterization. + + Usage:: + + python -m data.fnm.scripts.dcpf_acpf_characterization \\ + --acpf-dir data/fnm/reference/acpf/ \\ + --dcpf-dir data/fnm/reference/dcpf/ \\ + --intermediate-dir data/fnm/intermediate/canonical/ \\ + [-o data/fnm/reference/] + + Exit codes: + - 0: Characterization report produced successfully. + - 1: Input error (missing files, zero matches after join). + - 2: Unexpected computation error. + + Args: + argv: Command-line arguments. If None, reads from sys.argv[1:]. + """ + parser = argparse.ArgumentParser( + description="Compare DCPF and ACPF reference solutions to characterize " + "DC approximation quality." + ) + parser.add_argument( + "--acpf-dir", + type=Path, + required=True, + help="Directory containing ACPF reference files.", + ) + parser.add_argument( + "--dcpf-dir", + type=Path, + required=True, + help="Directory containing DCPF reference files.", + ) + parser.add_argument( + "--intermediate-dir", + type=Path, + required=True, + help="Directory containing intermediate format CSVs.", + ) + parser.add_argument( + "-o", + "--output-dir", + type=Path, + default=None, + help="Output directory (default: data/fnm/reference/).", + ) + + args = parser.parse_args(argv if argv is not None else sys.argv[1:]) + output_dir = args.output_dir or Path("data/fnm/reference") + + try: + result_dir = build_characterization( + acpf_dir=args.acpf_dir, + dcpf_dir=args.dcpf_dir, + intermediate_dir=args.intermediate_dir, + output_dir=output_dir, + ) + print(f"Characterization reports written to: {result_dir}") + except (FileNotFoundError, ValueError) as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1) + except Exception as exc: + print(f"Unexpected error: {exc}", file=sys.stderr) + sys.exit(2) + + +if __name__ == "__main__": + main() diff --git a/data/fnm/scripts/dcpf_reference.py b/data/fnm/scripts/dcpf_reference.py new file mode 100644 index 00000000..16a01d5c --- /dev/null +++ b/data/fnm/scripts/dcpf_reference.py @@ -0,0 +1,1624 @@ +"""DCPF Reference Solution Computation for FNM Annual S01. + +Computes the DC Power Flow (DCPF) reference solution by building and solving +the standard B' susceptance matrix formulation against the canonical parser's +intermediate format data. The DCPF reference is always solver-computed -- +PSS/E RAW files do not store DC solutions -- making this the sole source of +DCPF ground truth for downstream tool verification. + +Output directory: ``data/fnm/reference/dcpf/`` + +Implementation note: uses only Python stdlib (no numpy/scipy). The B-matrix +is represented as a dict-of-dicts sparse structure and solved via dense LU +factorization after assembly into a list-of-lists matrix. This is sufficient +for networks up to ~30K buses on modern hardware. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import logging +import math +import sys +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +ZERO_IMPEDANCE_REPLACEMENT: float = 0.0001 +"""Reactance (p.u.) assigned to zero-impedance branches (X=0) to avoid +division-by-zero in B-matrix construction. Small enough for negligible +angle error, large enough to avoid numerical ill-conditioning.""" + +ANGLE_TOLERANCE_DEG: float = 0.001 +"""Tolerance for flow-angle consistency validation (degrees).""" + +FLOW_TOLERANCE_MW: float = 0.1 +"""Tolerance for power balance validation (MW).""" + + +# --------------------------------------------------------------------------- +# Input data containers +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class BusRecord: + """A single bus from the intermediate format, relevant to DCPF.""" + + bus_number: int + """Bus number (unique identifier).""" + + bus_type: int + """Bus type: 1=PQ, 2=PV, 3=slack, 4=isolated.""" + + pd_mw: float + """Real power demand at the bus (MW).""" + + base_kv: float + """Bus base voltage (kV). Informational only for DCPF.""" + + +@dataclass(frozen=True) +class GeneratorRecord: + """A single generator from the intermediate format, relevant to DCPF.""" + + bus_number: int + """Bus number where this generator is connected.""" + + pg_mw: float + """Real power output (MW).""" + + status: int + """Generator status: 1=in-service, 0=out-of-service.""" + + machine_id: str + """Machine identifier (for multi-generator buses).""" + + +@dataclass(frozen=True) +class BranchRecord: + """A single branch from the intermediate format, relevant to DCPF.""" + + from_bus: int + """From bus number.""" + + to_bus: int + """To bus number.""" + + circuit_id: str + """Circuit identifier (distinguishes parallel branches).""" + + x_pu: float + """Series reactance (p.u. on system MVA base).""" + + tap_ratio: float + """Transformer off-nominal turns ratio (1.0 for lines).""" + + shift_deg: float + """Phase shift angle (degrees, 0.0 for non-phase-shifters).""" + + status: int + """Branch status: 1=in-service, 0=out-of-service.""" + + is_transformer: bool + """True if this branch is a transformer (has tap ratio != 1.0 or + originates from the Transformer record type).""" + + +# --------------------------------------------------------------------------- +# B-matrix and solution containers +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class BMatrixResult: + """Result of B-matrix construction.""" + + b_prime: list[list[float]] + """The (N-1) x (N-1) B' susceptance matrix as a dense list-of-lists.""" + + bus_index_map: dict[int, int] + """Mapping from bus number to matrix row/column index.""" + + slack_bus: int + """Bus number of the slack bus (angle fixed at 0.0).""" + + active_bus_count: int + """Total number of active buses (including slack).""" + + zero_impedance_branches: list[tuple[int, int, str]] + """List of (from_bus, to_bus, circuit_id) for branches that had X=0 + and were assigned the replacement reactance.""" + + excluded_branch_count: int + """Number of out-of-service branches excluded from the matrix.""" + + phase_shifter_count: int + """Number of branches with non-zero phase shift angle.""" + + base_mva: float + """System MVA base used for per-unit conversion.""" + + +@dataclass(frozen=True) +class BranchFlow: + """MW flow result for a single branch.""" + + from_bus: int + """From bus number.""" + + to_bus: int + """To bus number.""" + + circuit_id: str + """Circuit identifier.""" + + p_flow_mw: float + """Real power flow (MW). Positive = from -> to direction.""" + + angle_diff_deg: float + """Angle difference theta_from - theta_to (degrees).""" + + x_pu: float + """Branch reactance used in computation (p.u.). May differ from + original if zero-impedance replacement was applied.""" + + is_zero_impedance_replaced: bool + """True if this branch had X=0 and used the replacement reactance.""" + + +@dataclass(frozen=True) +class DCPFSolution: + """Complete DCPF solution for the network.""" + + bus_angles_deg: dict[int, float] + """Mapping from bus number to voltage angle (degrees). Slack bus is 0.0. + Only active (non-excluded) buses are included.""" + + branch_flows_mw: list[BranchFlow] + """Per-branch MW flow for all in-service branches.""" + + total_generation_mw: float + """Sum of all in-service generator Pg (MW).""" + + total_load_mw: float + """Sum of all active bus Pd (MW).""" + + slack_bus: int + """Bus number of the slack bus.""" + + slack_injection_mw: float + """Net power injection at the slack bus (MW).""" + + active_bus_count: int + """Number of active buses in the solution.""" + + active_branch_count: int + """Number of in-service branches in the solution.""" + + zero_impedance_branches: list[tuple[int, int, str]] + """Branches that had X=0, carried forward from BMatrixResult.""" + + base_mva: float + """System MVA base.""" + + +# --------------------------------------------------------------------------- +# Validation result +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class DCPFValidation: + """Internal consistency validation of the DCPF solution.""" + + power_balance_ok: bool + """True if |total_gen - total_load| < FLOW_TOLERANCE_MW.""" + + power_balance_residual_mw: float + """Absolute residual of the power balance check (MW).""" + + flow_angle_consistency_ok: bool + """True if all branch flows are consistent with angle differences + within FLOW_TOLERANCE_MW.""" + + flow_angle_max_deviation_mw: float + """Maximum absolute deviation between computed branch flow and + flow re-derived from angle differences (MW).""" + + slack_angle_zero: bool + """True if the slack bus angle is exactly 0.0 degrees.""" + + all_checks_passed: bool + """True if all three checks passed.""" + + +# --------------------------------------------------------------------------- +# Column name auto-detection helpers +# --------------------------------------------------------------------------- + +_BUS_COLUMN_MAP: dict[str, list[str]] = { + "BUS_I": ["bus_i", "bus", "bus_number", "number", "i"], + "BUS_TYPE": ["bus_type", "type", "ide"], + "PD": ["pd", "pl"], + "BASE_KV": ["base_kv", "basekv", "nom_kv", "baskv", "vnom"], +} + +_GEN_COLUMN_MAP: dict[str, list[str]] = { + "GEN_BUS": ["gen_bus", "bus", "bus_number", "i"], + "PG": ["pg"], + "GEN_STATUS": ["gen_status", "status", "stat"], + "ID": ["id", "machine_id", "mach_id"], +} + +_BRANCH_COLUMN_MAP: dict[str, list[str]] = { + "F_BUS": ["f_bus", "from_bus", "i", "fbus"], + "T_BUS": ["t_bus", "to_bus", "j", "tbus"], + "BR_X": ["br_x", "x"], + "TAP": ["tap", "windv1"], + "SHIFT": ["shift", "ang1"], + "BR_STATUS": ["br_status", "status", "st"], + "CKT": ["ckt", "circuit"], +} + +_TRANSFORMER_COLUMN_MAP: dict[str, list[str]] = { + "I": ["i", "from_bus", "f_bus", "fbus"], + "J": ["j", "to_bus", "t_bus", "tbus"], + "X1_2": ["x1_2", "x12", "br_x", "x"], + "WINDV1": ["windv1", "tap", "wind1"], + "ANG1": ["ang1", "shift", "angle1"], + "STAT": ["stat", "status", "st", "br_status"], + "CKT": ["ckt", "circuit"], +} + + +def _resolve_columns( + headers: list[str], + column_map: dict[str, list[str]], + required: list[str], +) -> dict[str, int]: + """Map normalized column names to CSV column indices. + + Args: + headers: Raw CSV header row. + column_map: Mapping from canonical name to list of variant names. + required: Canonical names that must be found. + + Returns: + Dict mapping canonical name to column index. + + Raises: + ValueError: If a required column cannot be found. + """ + lower_headers = [h.strip().lower() for h in headers] + result: dict[str, int] = {} + + for canonical, variants in column_map.items(): + canonical_lower = canonical.lower() + if canonical_lower in lower_headers: + result[canonical] = lower_headers.index(canonical_lower) + continue + for variant in variants: + if variant.lower() in lower_headers: + result[canonical] = lower_headers.index(variant.lower()) + break + + missing = [r for r in required if r not in result] + if missing: + raise ValueError(f"Required columns not found: {missing}. Available headers: {headers}") + return result + + +# --------------------------------------------------------------------------- +# Data loading +# --------------------------------------------------------------------------- + + +def load_bus_table(bus_csv_path: Path) -> list[BusRecord]: + """Load the bus table from the canonical parser's CSV output. + + Auto-detects column names from MATPOWER and GridCal conventions: + - Bus number: ``BUS_I``, ``bus``, ``bus_number``, ``NUMBER`` + - Bus type: ``BUS_TYPE``, ``type``, ``bus_type``, ``IDE`` + - Real power demand: ``PD``, ``Pd``, ``pd``, ``PL`` + - Base kV: ``BASE_KV``, ``base_kv``, ``basekv``, ``NOM_KV`` + + Args: + bus_csv_path: Path to the bus CSV file. + + Returns: + List of BusRecord instances for all buses in the file + (including isolated -- filtering is done later). + + Raises: + FileNotFoundError: If the CSV does not exist. + ValueError: If required columns cannot be identified. + """ + if not bus_csv_path.exists(): + raise FileNotFoundError(f"Bus CSV not found: {bus_csv_path}") + + with open(bus_csv_path, encoding="utf-8") as f: + reader = csv.reader(f) + rows = list(reader) + + if not rows: + raise ValueError(f"Bus CSV is empty: {bus_csv_path}") + + headers = rows[0] + col_map = _resolve_columns(headers, _BUS_COLUMN_MAP, required=["BUS_I", "BUS_TYPE", "PD"]) + data_rows = rows[1:] + + result: list[BusRecord] = [] + for row in data_rows: + if not row or all(cell.strip() == "" for cell in row): + continue + result.append( + BusRecord( + bus_number=int(float(row[col_map["BUS_I"]].strip())), + bus_type=int(float(row[col_map["BUS_TYPE"]].strip())), + pd_mw=float(row[col_map["PD"]].strip()), + base_kv=float(row[col_map["BASE_KV"]].strip()) if "BASE_KV" in col_map else 0.0, + ) + ) + return result + + +def load_generator_table(gen_csv_path: Path) -> list[GeneratorRecord]: + """Load the generator table from the canonical parser's CSV output. + + Auto-detects column names from MATPOWER and GridCal conventions: + - Bus number: ``GEN_BUS``, ``bus``, ``bus_number`` + - Real power: ``PG``, ``Pg``, ``pg`` + - Status: ``GEN_STATUS``, ``status``, ``gen_status`` + - Machine ID: ``ID``, ``machine_id``, ``MACH_ID`` (defaults to "1" if absent) + + Args: + gen_csv_path: Path to the generator CSV file. + + Returns: + List of GeneratorRecord instances for all generators. + + Raises: + FileNotFoundError: If the CSV does not exist. + ValueError: If required columns cannot be identified. + """ + if not gen_csv_path.exists(): + raise FileNotFoundError(f"Generator CSV not found: {gen_csv_path}") + + with open(gen_csv_path, encoding="utf-8") as f: + reader = csv.reader(f) + rows = list(reader) + + if not rows: + raise ValueError(f"Generator CSV is empty: {gen_csv_path}") + + headers = rows[0] + col_map = _resolve_columns(headers, _GEN_COLUMN_MAP, required=["GEN_BUS", "PG", "GEN_STATUS"]) + data_rows = rows[1:] + + result: list[GeneratorRecord] = [] + for row in data_rows: + if not row or all(cell.strip() == "" for cell in row): + continue + result.append( + GeneratorRecord( + bus_number=int(float(row[col_map["GEN_BUS"]].strip())), + pg_mw=float(row[col_map["PG"]].strip()), + status=int(float(row[col_map["GEN_STATUS"]].strip())), + machine_id=row[col_map["ID"]].strip() if "ID" in col_map else "1", + ) + ) + return result + + +def load_branch_table(branch_csv_path: Path) -> list[BranchRecord]: + """Load the branch table from the canonical parser's CSV output. + + Reads both simple branches and transformers. Auto-detects column names: + - From bus: ``F_BUS``, ``from_bus``, ``I`` + - To bus: ``T_BUS``, ``to_bus``, ``J`` + - Reactance: ``BR_X``, ``x``, ``X`` + - Tap ratio: ``TAP``, ``tap``, ``WINDV1`` (1.0 if absent or 0.0) + - Phase shift: ``SHIFT``, ``shift``, ``ANG1`` (0.0 if absent) + - Status: ``BR_STATUS``, ``status``, ``ST`` + - Circuit ID: ``CKT``, ``ckt``, ``circuit`` (defaults to "1" if absent) + + A tap ratio of 0.0 in MATPOWER convention means 1.0 (nominal). This + function normalizes 0.0 tap values to 1.0. + + Args: + branch_csv_path: Path to the branch CSV file. + + Returns: + List of BranchRecord instances for all branches. + + Raises: + FileNotFoundError: If the CSV does not exist. + ValueError: If required columns cannot be identified. + """ + if not branch_csv_path.exists(): + raise FileNotFoundError(f"Branch CSV not found: {branch_csv_path}") + + with open(branch_csv_path, encoding="utf-8") as f: + reader = csv.reader(f) + rows = list(reader) + + if not rows: + raise ValueError(f"Branch CSV is empty: {branch_csv_path}") + + headers = rows[0] + col_map = _resolve_columns( + headers, _BRANCH_COLUMN_MAP, required=["F_BUS", "T_BUS", "BR_X", "BR_STATUS"] + ) + data_rows = rows[1:] + + result: list[BranchRecord] = [] + for row in data_rows: + if not row or all(cell.strip() == "" for cell in row): + continue + + tap_raw = float(row[col_map["TAP"]].strip()) if "TAP" in col_map else 0.0 + tap = tap_raw if tap_raw != 0.0 else 1.0 + + shift = float(row[col_map["SHIFT"]].strip()) if "SHIFT" in col_map else 0.0 + + is_transformer = tap != 1.0 or shift != 0.0 + + result.append( + BranchRecord( + from_bus=int(float(row[col_map["F_BUS"]].strip())), + to_bus=int(float(row[col_map["T_BUS"]].strip())), + circuit_id=row[col_map["CKT"]].strip() if "CKT" in col_map else "1", + x_pu=float(row[col_map["BR_X"]].strip()), + tap_ratio=tap, + shift_deg=shift, + status=int(float(row[col_map["BR_STATUS"]].strip())), + is_transformer=is_transformer, + ) + ) + return result + + +def load_transformer_table(transformer_csv_path: Path) -> list[BranchRecord]: + """Load a separate transformer table and map it to BranchRecord instances. + + Reads PSS/E-style transformer columns (I, J, X1_2, WINDV1, ANG1, STAT, CKT) + and maps them to the unified BranchRecord structure used by the DCPF solver. + + A WINDV1 value of 0.0 is normalized to 1.0 (nominal tap ratio), matching + the MATPOWER convention used by ``load_branch_table``. + + STAT mapping follows PSS/E conventions: + - 0: out-of-service (status=0) + - 1: in-service (status=1) + - 2: only winding 1 in-service (status=1 for two-winding DCPF) + - 3: only winding 2 in-service (status=1 for two-winding DCPF) + - 4: in-service (status=1) + + Args: + transformer_csv_path: Path to the transformer CSV file. + + Returns: + List of BranchRecord instances for all transformers. + + Raises: + FileNotFoundError: If the CSV does not exist. + ValueError: If required columns cannot be identified. + """ + if not transformer_csv_path.exists(): + raise FileNotFoundError(f"Transformer CSV not found: {transformer_csv_path}") + + with open(transformer_csv_path, encoding="utf-8") as f: + reader = csv.reader(f) + rows = list(reader) + + if not rows: + raise ValueError(f"Transformer CSV is empty: {transformer_csv_path}") + + headers = rows[0] + col_map = _resolve_columns( + headers, _TRANSFORMER_COLUMN_MAP, required=["I", "J", "X1_2", "STAT"] + ) + data_rows = rows[1:] + + result: list[BranchRecord] = [] + for row in data_rows: + if not row or all(cell.strip() == "" for cell in row): + continue + + tap_raw = float(row[col_map["WINDV1"]].strip()) if "WINDV1" in col_map else 0.0 + tap = tap_raw if tap_raw != 0.0 else 1.0 + + shift = float(row[col_map["ANG1"]].strip()) if "ANG1" in col_map else 0.0 + + # PSS/E STAT: 0=out-of-service, 1-4=in-service for two-winding DCPF + stat_raw = int(float(row[col_map["STAT"]].strip())) + status = 0 if stat_raw == 0 else 1 + + result.append( + BranchRecord( + from_bus=int(float(row[col_map["I"]].strip())), + to_bus=int(float(row[col_map["J"]].strip())), + circuit_id=row[col_map["CKT"]].strip() if "CKT" in col_map else "1", + x_pu=float(row[col_map["X1_2"]].strip()), + tap_ratio=tap, + shift_deg=shift, + status=status, + is_transformer=True, + ) + ) + return result + + +def load_manifest(manifest_path: Path) -> dict: + """Load a manifest JSON sidecar file. + + Args: + manifest_path: Path to the manifest.json file. + + Returns: + Parsed manifest as a dict. + + Raises: + FileNotFoundError: If the manifest file does not exist. + ValueError: If the manifest is not valid JSON. + """ + if not manifest_path.exists(): + raise FileNotFoundError(f"Manifest file not found: {manifest_path}") + + try: + return json.loads(manifest_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid JSON in manifest: {manifest_path}: {exc}") from exc + + +def resolve_base_mva( + cli_base_mva: float | None, + manifest: dict | None, + default: float = 100.0, +) -> tuple[float, str]: + """Resolve the system MVA base with precedence: CLI > manifest > default. + + Args: + cli_base_mva: Value from ``--base-mva`` CLI argument, or None if not provided. + manifest: Parsed manifest dict (may contain ``sbase`` key), or None. + default: Fallback value if neither CLI nor manifest provides one. + + Returns: + Tuple of (base_mva, source_description) where source_description is one of + ``"cli"``, ``"manifest"``, or ``"default"``. + """ + if cli_base_mva is not None: + return cli_base_mva, "cli" + if manifest is not None and "sbase" in manifest: + return float(manifest["sbase"]), "manifest" + return default, "default" + + +def load_excluded_buses(exclusion_csv_path: Path) -> set[int]: + """Load the set of excluded bus numbers from the D1 bus exclusion registry. + + Reads the ``bus_number`` column from the exclusion CSV produced by + Phase 3 D1 (``data/fnm/reference/excluded_buses.csv``). + + Args: + exclusion_csv_path: Path to the excluded buses CSV. + + Returns: + Set of bus numbers to exclude from the DCPF computation. + + Raises: + FileNotFoundError: If the exclusion CSV does not exist. + """ + if not exclusion_csv_path.exists(): + raise FileNotFoundError(f"Exclusion CSV not found: {exclusion_csv_path}") + + excluded: set[int] = set() + with open(exclusion_csv_path, encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + excluded.add(int(float(row["bus_number"]))) + return excluded + + +# --------------------------------------------------------------------------- +# Bus filtering and injection computation +# --------------------------------------------------------------------------- + + +def filter_active_buses( + buses: list[BusRecord], + excluded_bus_numbers: set[int], +) -> list[BusRecord]: + """Filter buses to the active set for DCPF computation. + + Removes: + - Buses in the exclusion registry (isolated, de-energized, disconnected) + - Buses with bus_type == 4 (as a safety check, even if not in the registry) + + Args: + buses: All buses from the intermediate format. + excluded_bus_numbers: Set of bus numbers from the exclusion registry. + + Returns: + List of active BusRecord instances. + """ + return [ + bus for bus in buses if bus.bus_number not in excluded_bus_numbers and bus.bus_type != 4 + ] + + +def compute_bus_injections( + active_buses: list[BusRecord], + generators: list[GeneratorRecord], + excluded_bus_numbers: set[int], +) -> dict[int, float]: + """Compute net real power injection at each active bus. + + For each active bus: P_injection = sum(Pg for in-service generators + at this bus) - Pd (bus load). + + Generators on excluded buses are ignored. Out-of-service generators + (status=0) are ignored. + + Args: + active_buses: The filtered active bus set. + generators: All generators from the intermediate format. + excluded_bus_numbers: Set of excluded bus numbers (for filtering generators). + + Returns: + Dict mapping bus number to net injection (MW). Buses with no + generators and no load have injection = 0.0. + """ + active_bus_set = {bus.bus_number for bus in active_buses} + + # Initialize injections: -Pd for each active bus + injections: dict[int, float] = {} + for bus in active_buses: + injections[bus.bus_number] = -bus.pd_mw + + # Add generator contributions + for gen in generators: + if gen.status != 1: + continue + if gen.bus_number in excluded_bus_numbers: + continue + if gen.bus_number in active_bus_set: + injections[gen.bus_number] = injections.get(gen.bus_number, 0.0) + gen.pg_mw + + return injections + + +def identify_slack_bus(active_buses: list[BusRecord]) -> int: + """Identify the slack bus for DCPF computation. + + Selects the bus with type=3 (swing bus). If multiple type-3 buses + exist, selects the one with the lowest bus number. If no type-3 bus + exists, raises an error. + + Args: + active_buses: The filtered active bus set. + + Returns: + Bus number of the selected slack bus. + + Raises: + ValueError: If no type-3 bus exists in the active bus set. + """ + slack_candidates = sorted([bus.bus_number for bus in active_buses if bus.bus_type == 3]) + if not slack_candidates: + raise ValueError("No type-3 (slack/swing) bus found in the active bus set.") + return slack_candidates[0] + + +# --------------------------------------------------------------------------- +# B-matrix construction (pure Python, no numpy/scipy) +# --------------------------------------------------------------------------- + + +def build_b_matrix( + active_buses: list[BusRecord], + branches: list[BranchRecord], + excluded_bus_numbers: set[int], + slack_bus: int, + base_mva: float, +) -> BMatrixResult: + """Construct the B' susceptance matrix for DC power flow. + + Builds the (N-1) x (N-1) dense matrix by iterating over all in-service + branches. The slack bus row and column are excluded from the matrix + (its angle is the reference, fixed at 0.0). + + For each in-service branch: + 1. Skip if either endpoint is in the excluded set. + 2. If X == 0.0, replace with ZERO_IMPEDANCE_REPLACEMENT and log. + 3. Compute susceptance per the B-matrix formulation, accounting for + tap ratio and phase shift. + 4. Add susceptance contributions to diagonal and off-diagonal entries. + + Args: + active_buses: The filtered active bus set. + branches: All branches from the intermediate format. + excluded_bus_numbers: Bus numbers to exclude. + slack_bus: Bus number of the slack bus. + base_mva: System MVA base. + + Returns: + A BMatrixResult containing the dense matrix and metadata. + + Raises: + ValueError: If the active bus set is empty or contains only the + slack bus (trivial system). + """ + active_bus_set = {bus.bus_number for bus in active_buses} + + if len(active_bus_set) <= 1: + raise ValueError( + "Active bus set must contain at least 2 buses " + f"(including slack). Got {len(active_bus_set)}." + ) + + # Build bus index map: non-slack active buses -> 0-based index + non_slack_buses = sorted(b for b in active_bus_set if b != slack_bus) + bus_index_map: dict[int, int] = {bus: idx for idx, bus in enumerate(non_slack_buses)} + n = len(non_slack_buses) + + # Initialize dense matrix + b_prime: list[list[float]] = [[0.0] * n for _ in range(n)] + + zero_impedance_branches: list[tuple[int, int, str]] = [] + excluded_branch_count = 0 + phase_shifter_count = 0 + for branch in branches: + # Skip out-of-service branches + if branch.status != 1: + excluded_branch_count += 1 + continue + + from_bus = branch.from_bus + to_bus = branch.to_bus + + # Skip if either endpoint is excluded or not in active set + if from_bus in excluded_bus_numbers or to_bus in excluded_bus_numbers: + continue + if from_bus not in active_bus_set or to_bus not in active_bus_set: + continue + + x = branch.x_pu + + # Handle zero-impedance branches + if x == 0.0: + x = ZERO_IMPEDANCE_REPLACEMENT + zero_impedance_branches.append((from_bus, to_bus, branch.circuit_id)) + logger.warning( + "Zero-impedance branch %d-%d (ckt %s) assigned X=%f p.u.", + from_bus, + to_bus, + branch.circuit_id, + x, + ) + + # Log negative reactance + if x < 0.0: + logger.warning( + "Negative reactance X=%f on branch %d-%d (ckt %s)", + x, + from_bus, + to_bus, + branch.circuit_id, + ) + + # Count phase shifters + if branch.shift_deg != 0.0: + phase_shifter_count += 1 + + t = branch.tap_ratio + + # Compute B-matrix entries based on tap ratio + if t == 1.0: + # Simple branch (no tap adjustment) + b = 1.0 / x + # Add to matrix (only for non-slack buses) + if from_bus in bus_index_map: + b_prime[bus_index_map[from_bus]][bus_index_map[from_bus]] += b + if to_bus in bus_index_map: + b_prime[bus_index_map[to_bus]][bus_index_map[to_bus]] += b + if from_bus in bus_index_map and to_bus in bus_index_map: + i_idx = bus_index_map[from_bus] + j_idx = bus_index_map[to_bus] + b_prime[i_idx][j_idx] -= b + b_prime[j_idx][i_idx] -= b + else: + # Transformer with off-nominal tap ratio + # From-side diagonal: 1 / (X * t^2) + # To-side diagonal: 1 / X + # Off-diagonal: -1 / (X * t) [both i,j and j,i] + b_from_diag = 1.0 / (x * t * t) + b_to_diag = 1.0 / x + b_off = -1.0 / (x * t) + + if from_bus in bus_index_map: + b_prime[bus_index_map[from_bus]][bus_index_map[from_bus]] += b_from_diag + if to_bus in bus_index_map: + b_prime[bus_index_map[to_bus]][bus_index_map[to_bus]] += b_to_diag + if from_bus in bus_index_map and to_bus in bus_index_map: + i_idx = bus_index_map[from_bus] + j_idx = bus_index_map[to_bus] + b_prime[i_idx][j_idx] += b_off + b_prime[j_idx][i_idx] += b_off + + return BMatrixResult( + b_prime=b_prime, + bus_index_map=bus_index_map, + slack_bus=slack_bus, + active_bus_count=len(active_bus_set), + zero_impedance_branches=zero_impedance_branches, + excluded_branch_count=excluded_branch_count, + phase_shifter_count=phase_shifter_count, + base_mva=base_mva, + ) + + +def compute_phase_shift_injections( + branches: list[BranchRecord], + excluded_bus_numbers: set[int], + base_mva: float, +) -> dict[int, float]: + """Compute injection vector modifications from phase-shifting transformers. + + For each in-service branch with a non-zero phase shift angle, computes + the real power offset injected at each endpoint: + + P_shift = shift_rad / X * baseMVA + + This is subtracted from the from-bus injection and added to the to-bus + injection. + + Args: + branches: All branches from the intermediate format. + excluded_bus_numbers: Bus numbers to exclude. + base_mva: System MVA base. + + Returns: + Dict mapping bus number to cumulative phase-shift injection + modification (MW). Only buses affected by phase shifters appear. + """ + injections: dict[int, float] = {} + + for branch in branches: + if branch.status != 1: + continue + if branch.shift_deg == 0.0: + continue + if branch.from_bus in excluded_bus_numbers or branch.to_bus in excluded_bus_numbers: + continue + + x = branch.x_pu + if x == 0.0: + x = ZERO_IMPEDANCE_REPLACEMENT + + shift_rad = math.radians(branch.shift_deg) + p_shift = shift_rad / x * base_mva + + # Subtract from from-bus, add to to-bus + injections[branch.from_bus] = injections.get(branch.from_bus, 0.0) - p_shift + injections[branch.to_bus] = injections.get(branch.to_bus, 0.0) + p_shift + + return injections + + +# --------------------------------------------------------------------------- +# Dense LU solver (pure Python, no numpy/scipy) +# --------------------------------------------------------------------------- + + +def _solve_linear_system(a_matrix: list[list[float]], b_vector: list[float]) -> list[float]: + """Solve A * x = b using Gaussian elimination with partial pivoting. + + Args: + a_matrix: N x N coefficient matrix (will be modified in place). + b_vector: N-element right-hand side vector (will be modified in place). + + Returns: + N-element solution vector x. + + Raises: + ValueError: If the matrix is singular. + """ + n = len(b_vector) + + # Make copies to avoid modifying inputs + a = [row[:] for row in a_matrix] + b = b_vector[:] + + # Forward elimination with partial pivoting + for col in range(n): + # Find pivot + max_val = abs(a[col][col]) + max_row = col + for row in range(col + 1, n): + if abs(a[row][col]) > max_val: + max_val = abs(a[row][col]) + max_row = row + + if max_val < 1e-15: + raise ValueError( + f"Singular or near-singular matrix at column {col}. " + "This may indicate disconnected sub-networks in the active bus set." + ) + + # Swap rows + if max_row != col: + a[col], a[max_row] = a[max_row], a[col] + b[col], b[max_row] = b[max_row], b[col] + + # Eliminate below + pivot = a[col][col] + for row in range(col + 1, n): + factor = a[row][col] / pivot + for k in range(col + 1, n): + a[row][k] -= factor * a[col][k] + a[row][col] = 0.0 + b[row] -= factor * b[col] + + # Back substitution + x = [0.0] * n + for row in range(n - 1, -1, -1): + val = b[row] + for col in range(row + 1, n): + val -= a[row][col] * x[col] + x[row] = val / a[row][row] + + return x + + +# --------------------------------------------------------------------------- +# Solver +# --------------------------------------------------------------------------- + + +def solve_dcpf( + b_matrix: BMatrixResult, + bus_injections: dict[int, float], + phase_shift_injections: dict[int, float], + branches: list[BranchRecord], + excluded_bus_numbers: set[int], +) -> DCPFSolution: + """Solve the DC power flow: B' * theta = P_injection. + + Steps: + 1. Assemble the injection vector P for the (N-1) non-slack buses, + incorporating phase-shift modifications. + 2. Convert injections to per-unit on system MVA base. + 3. Solve the linear system using Gaussian elimination. + 4. Convert theta from radians to degrees and map back to bus numbers. + 5. Compute per-branch MW flows from angle differences. + + Args: + b_matrix: The BMatrixResult from build_b_matrix. + bus_injections: Net real power injection per bus (MW). + phase_shift_injections: Phase-shift injection modifications (MW). + branches: All branches (for computing branch flows). + excluded_bus_numbers: Excluded bus numbers. + + Returns: + A complete DCPFSolution. + + Raises: + ValueError: If the B-matrix is singular (disconnected network). + """ + bus_index_map = b_matrix.bus_index_map + n = len(bus_index_map) + base_mva = b_matrix.base_mva + slack_bus = b_matrix.slack_bus + + # Assemble injection vector in per-unit + p_vector = [0.0] * n + for bus_num, idx in bus_index_map.items(): + inj_mw = bus_injections.get(bus_num, 0.0) + # Add phase-shift modifications + inj_mw += phase_shift_injections.get(bus_num, 0.0) + # Convert to per-unit + p_vector[idx] = inj_mw / base_mva + + # Solve B' * theta = P + theta_rad = _solve_linear_system(b_matrix.b_prime, p_vector) + + # Build bus_angles_rad and bus_angles_deg maps + bus_angles_rad: dict[int, float] = {slack_bus: 0.0} + bus_angles_deg: dict[int, float] = {slack_bus: 0.0} + + for bus_num, idx in bus_index_map.items(): + bus_angles_rad[bus_num] = theta_rad[idx] + bus_angles_deg[bus_num] = math.degrees(theta_rad[idx]) + + # Compute branch flows + branch_flows = compute_branch_flows(branches, bus_angles_rad, excluded_bus_numbers, base_mva) + + # Compute totals + total_gen = 0.0 + total_load = 0.0 + for bus_num, inj in bus_injections.items(): + # injection = gen - load, so gen contributes positive, load contributes negative + if inj > 0: + total_gen += inj + else: + total_load += abs(inj) + + slack_inj = bus_injections.get(slack_bus, 0.0) + + return DCPFSolution( + bus_angles_deg=bus_angles_deg, + branch_flows_mw=branch_flows, + total_generation_mw=0.0, # Placeholder, set by orchestrator + total_load_mw=0.0, # Placeholder, set by orchestrator + slack_bus=slack_bus, + slack_injection_mw=slack_inj, + active_bus_count=b_matrix.active_bus_count, + active_branch_count=len(branch_flows), + zero_impedance_branches=b_matrix.zero_impedance_branches, + base_mva=base_mva, + ) + + +def compute_branch_flows( + branches: list[BranchRecord], + bus_angles_rad: dict[int, float], + excluded_bus_numbers: set[int], + base_mva: float, +) -> list[BranchFlow]: + """Compute MW flow for each in-service branch from angle differences. + + For each branch: + P_flow = (theta_from - theta_to) / X * baseMVA + + For transformers with tap ratio t: + P_flow = (theta_from - theta_to) / (X * t) * baseMVA + + Phase shift is already incorporated in the angle solution via + injection vector modification, so it does not appear here. + + Args: + branches: All branches from the intermediate format. + bus_angles_rad: Bus angles in radians (keyed by bus number). + excluded_bus_numbers: Excluded bus numbers. + base_mva: System MVA base. + + Returns: + List of BranchFlow instances for all in-service branches with + both endpoints in the active set. + """ + flows: list[BranchFlow] = [] + + for branch in branches: + if branch.status != 1: + continue + + from_bus = branch.from_bus + to_bus = branch.to_bus + + if from_bus in excluded_bus_numbers or to_bus in excluded_bus_numbers: + continue + if from_bus not in bus_angles_rad or to_bus not in bus_angles_rad: + continue + + x = branch.x_pu + is_zero_replaced = False + if x == 0.0: + x = ZERO_IMPEDANCE_REPLACEMENT + is_zero_replaced = True + + t = branch.tap_ratio + # Effective reactance includes tap ratio for transformers + x_eff = x * t + + theta_from = bus_angles_rad[from_bus] + theta_to = bus_angles_rad[to_bus] + angle_diff_rad = theta_from - theta_to + + # P_flow = (theta_from - theta_to) / (X * t) * baseMVA + p_flow_mw = angle_diff_rad / x_eff * base_mva + + flows.append( + BranchFlow( + from_bus=from_bus, + to_bus=to_bus, + circuit_id=branch.circuit_id, + p_flow_mw=p_flow_mw, + angle_diff_deg=math.degrees(angle_diff_rad), + x_pu=x_eff, + is_zero_impedance_replaced=is_zero_replaced, + ) + ) + + return flows + + +# --------------------------------------------------------------------------- +# Validation +# --------------------------------------------------------------------------- + + +def validate_dcpf_solution(solution: DCPFSolution) -> DCPFValidation: + """Run internal consistency checks on the DCPF solution. + + Three checks: + 1. **Power balance:** |total_generation - total_load| < FLOW_TOLERANCE_MW + (lossless DC assumption -- generation must equal load after slack + bus adjustment). + 2. **Flow-angle consistency:** For each branch, the stored P_flow_MW + matches (theta_from - theta_to) / X * baseMVA within FLOW_TOLERANCE_MW. + This catches indexing errors in the B-matrix. + 3. **Slack angle zero:** The slack bus angle is exactly 0.0 degrees. + + Args: + solution: The DCPF solution to validate. + + Returns: + A DCPFValidation with all check results. + """ + # Check 1: Power balance + power_residual = abs(solution.total_generation_mw - solution.total_load_mw) + power_balance_ok = power_residual < FLOW_TOLERANCE_MW + + # Check 2: Flow-angle consistency + # x_pu in BranchFlow stores the effective reactance (X * tap_ratio), + # so the formula P_flow = angle_diff / x_pu * baseMVA should match. + max_deviation = 0.0 + for flow in solution.branch_flows_mw: + from_angle_rad = math.radians(solution.bus_angles_deg.get(flow.from_bus, 0.0)) + to_angle_rad = math.radians(solution.bus_angles_deg.get(flow.to_bus, 0.0)) + angle_diff = from_angle_rad - to_angle_rad + + expected_flow = angle_diff / flow.x_pu * solution.base_mva + deviation = abs(flow.p_flow_mw - expected_flow) + if deviation > max_deviation: + max_deviation = deviation + + flow_angle_ok = max_deviation < FLOW_TOLERANCE_MW + + # Check 3: Slack angle zero + slack_angle = solution.bus_angles_deg.get(solution.slack_bus, float("nan")) + slack_angle_zero = slack_angle == 0.0 + + all_passed = power_balance_ok and flow_angle_ok and slack_angle_zero + + return DCPFValidation( + power_balance_ok=power_balance_ok, + power_balance_residual_mw=power_residual, + flow_angle_consistency_ok=flow_angle_ok, + flow_angle_max_deviation_mw=max_deviation, + slack_angle_zero=slack_angle_zero, + all_checks_passed=all_passed, + ) + + +# --------------------------------------------------------------------------- +# Output writing +# --------------------------------------------------------------------------- + + +def write_buses_csv( + solution: DCPFSolution, + output_path: Path, +) -> None: + """Write the bus angles CSV file. + + Output schema: + + | Column | Type | Unit | Description | + |--------|------|------|-------------| + | bus | int | -- | Bus number | + | VA | float | degrees | Voltage angle | + + Sorted by bus number ascending. Only active (non-excluded) buses. + + Args: + solution: The DCPF solution. + output_path: Path for the output CSV file. + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + + sorted_buses = sorted(solution.bus_angles_deg.items(), key=lambda x: x[0]) + + with open(output_path, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + writer.writerow(["bus", "VA"]) + for bus_num, angle_deg in sorted_buses: + writer.writerow([bus_num, f"{angle_deg:.6f}"]) + + +def write_branches_csv( + solution: DCPFSolution, + output_path: Path, +) -> None: + """Write the branch flows CSV file. + + Output schema: + + | Column | Type | Unit | Description | + |--------|------|------|-------------| + | from_bus | int | -- | From bus number | + | to_bus | int | -- | To bus number | + | ckt | str | -- | Circuit identifier | + | P_flow_MW | float | MW | Real power flow (positive = from->to) | + + Sorted by (from_bus, to_bus, ckt) ascending. Only in-service branches + with both endpoints in the active set. + + Args: + solution: The DCPF solution. + output_path: Path for the output CSV file. + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + + sorted_flows = sorted( + solution.branch_flows_mw, + key=lambda f: (f.from_bus, f.to_bus, f.circuit_id), + ) + + with open(output_path, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + writer.writerow(["from_bus", "to_bus", "ckt", "P_flow_MW"]) + for flow in sorted_flows: + writer.writerow( + [ + flow.from_bus, + flow.to_bus, + flow.circuit_id, + f"{flow.p_flow_mw:.6f}", + ] + ) + + +def write_summary_json( + solution: DCPFSolution, + validation: DCPFValidation, + output_path: Path, + *, + canonical_parser: str = "", +) -> None: + """Write the DCPF summary JSON file. + + Args: + solution: The DCPF solution. + validation: The validation results. + output_path: Path for the output JSON file. + canonical_parser: Name of the canonical parser (for metadata). + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + + # Compute angle statistics + angles = list(solution.bus_angles_deg.values()) + max_angle = max(angles) if angles else 0.0 + min_angle = min(angles) if angles else 0.0 + mean_angle = sum(angles) / len(angles) if angles else 0.0 + variance = sum((a - mean_angle) ** 2 for a in angles) / len(angles) if angles else 0.0 + std_angle = math.sqrt(variance) + + # Compute flow statistics + flows_mw = [f.p_flow_mw for f in solution.branch_flows_mw] + max_flow = max(flows_mw) if flows_mw else 0.0 + min_flow = min(flows_mw) if flows_mw else 0.0 + + summary = { + "solver": "stdlib_gaussian_elimination", + "formulation": "standard_b_prime", + "base_mva": solution.base_mva, + "settings": { + "zero_impedance_replacement_pu": ZERO_IMPEDANCE_REPLACEMENT, + "voltage_magnitude_assumption": 1.0, + "loss_model": "lossless", + "slack_bus": solution.slack_bus, + "slack_angle_deg": 0.0, + }, + "network_summary": { + "active_bus_count": solution.active_bus_count, + "active_branch_count": solution.active_branch_count, + "excluded_bus_count": 0, # Set by orchestrator if available + "out_of_service_branch_count": 0, # Set by orchestrator if available + "zero_impedance_branch_count": len(solution.zero_impedance_branches), + "phase_shifter_count": 0, # Set by orchestrator if available + }, + "power_summary": { + "total_generation_mw": solution.total_generation_mw, + "total_load_mw": solution.total_load_mw, + "slack_injection_mw": solution.slack_injection_mw, + "max_branch_flow_mw": max_flow, + "min_branch_flow_mw": min_flow, + }, + "angle_summary": { + "max_angle_deg": max_angle, + "min_angle_deg": min_angle, + "mean_angle_deg": mean_angle, + "std_angle_deg": std_angle, + }, + "validation": { + "power_balance_ok": validation.power_balance_ok, + "power_balance_residual_mw": validation.power_balance_residual_mw, + "flow_angle_consistency_ok": validation.flow_angle_consistency_ok, + "flow_angle_max_deviation_mw": validation.flow_angle_max_deviation_mw, + "slack_angle_zero": validation.slack_angle_zero, + "all_checks_passed": validation.all_checks_passed, + }, + "zero_impedance_branches": [ + {"from_bus": fb, "to_bus": tb, "ckt": ckt} + for fb, tb, ckt in solution.zero_impedance_branches + ], + "excluded_element_types": [], + "timestamp": datetime.now(timezone.utc).isoformat(), + "canonical_parser": canonical_parser, + } + + output_path.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + + +# --------------------------------------------------------------------------- +# Orchestration +# --------------------------------------------------------------------------- + + +def run_dcpf_reference( + bus_csv_path: Path, + gen_csv_path: Path, + branch_csv_path: Path, + exclusion_csv_path: Path, + output_dir: Path, + *, + base_mva: float = 100.0, + canonical_parser: str = "", + transformer_csv_path: Path | None = None, +) -> DCPFSolution: + """Orchestrate the full DCPF reference computation pipeline. + + Steps: + 1. Load bus, generator, and branch tables from CSVs. + If ``transformer_csv_path`` is provided, load transformers from a + separate file and concatenate them with the branch list. + 2. Load excluded bus set from D1 exclusion registry. + 3. Filter to active buses. + 4. Identify the slack bus. + 5. Compute bus injections. + 6. Build the B' susceptance matrix. + 7. Compute phase-shift injection modifications. + 8. Solve the linear system. + 9. Validate the solution. + 10. Write buses_dcpf.csv, branches_dcpf.csv, and summary_dcpf.json. + + Args: + bus_csv_path: Path to the canonical parser's bus CSV. + gen_csv_path: Path to the canonical parser's generator CSV. + branch_csv_path: Path to the canonical parser's branch CSV. + exclusion_csv_path: Path to the D1 bus exclusion registry CSV. + output_dir: Directory for output files. Created if it does not exist. + base_mva: System MVA base (default 100.0). + canonical_parser: Name of the canonical parser (for metadata). + transformer_csv_path: Optional path to a separate transformer CSV. + When provided, transformers are loaded from this file and + appended to the branch list instead of being embedded in the + branch CSV. + + Returns: + The DCPFSolution (also written to disk). + + Raises: + FileNotFoundError: If any input CSV does not exist. + ValueError: If the network has no active buses or no slack bus. + """ + # 1. Load tables + buses = load_bus_table(bus_csv_path) + generators = load_generator_table(gen_csv_path) + branches = load_branch_table(branch_csv_path) + + # 1b. Optionally load and concatenate separate transformer table + if transformer_csv_path is not None: + transformers = load_transformer_table(transformer_csv_path) + branches = branches + transformers + + # 2. Load exclusion set + excluded = load_excluded_buses(exclusion_csv_path) + + # 3. Filter active buses + active_buses = filter_active_buses(buses, excluded) + if not active_buses: + raise ValueError("No active buses after exclusion filtering.") + + # 4. Identify slack bus + slack_bus = identify_slack_bus(active_buses) + + # 5. Compute bus injections + injections = compute_bus_injections(active_buses, generators, excluded) + + # 6. Build B-matrix + b_result = build_b_matrix(active_buses, branches, excluded, slack_bus, base_mva) + + # 7. Compute phase-shift injections + phase_injections = compute_phase_shift_injections(branches, excluded, base_mva) + + # 8. Solve + solution = solve_dcpf(b_result, injections, phase_injections, branches, excluded) + + # Compute total generation and total load from original data + active_bus_set = {bus.bus_number for bus in active_buses} + total_gen = sum( + gen.pg_mw + for gen in generators + if gen.status == 1 and gen.bus_number not in excluded and gen.bus_number in active_bus_set + ) + total_load = sum(bus.pd_mw for bus in active_buses) + + # Reconstruct solution with correct totals + solution = DCPFSolution( + bus_angles_deg=solution.bus_angles_deg, + branch_flows_mw=solution.branch_flows_mw, + total_generation_mw=total_gen, + total_load_mw=total_load, + slack_bus=solution.slack_bus, + slack_injection_mw=injections.get(slack_bus, 0.0), + active_bus_count=solution.active_bus_count, + active_branch_count=solution.active_branch_count, + zero_impedance_branches=solution.zero_impedance_branches, + base_mva=solution.base_mva, + ) + + # 9. Validate + validation = validate_dcpf_solution(solution) + + # 10. Write output + output_dir.mkdir(parents=True, exist_ok=True) + write_buses_csv(solution, output_dir / "buses_dcpf.csv") + write_branches_csv(solution, output_dir / "branches_dcpf.csv") + write_summary_json( + solution, + validation, + output_dir / "summary_dcpf.json", + canonical_parser=canonical_parser, + ) + + return solution + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> None: + """CLI entry point for DCPF reference computation. + + Usage:: + + python -m data.fnm.scripts.dcpf_reference \\ + --bus-csv path/to/bus.csv \\ + --gen-csv path/to/gen.csv \\ + --branch-csv path/to/branch.csv \\ + --exclusion-csv path/to/excluded_buses.csv \\ + [-o output_dir] \\ + [--base-mva 100.0] \\ + [--canonical-parser gridcal] + + If ``-o`` is omitted, writes to ``data/fnm/reference/dcpf/``. + + Exit codes: + - 0: DCPF computed and all validation checks passed. + - 1: DCPF computed but one or more validation checks failed. + - 2: Input error (missing files, no active buses, singular B-matrix). + + Args: + argv: Command-line arguments. If ``None``, reads from ``sys.argv[1:]``. + """ + parser = argparse.ArgumentParser( + description="Compute DCPF reference solution from intermediate format CSVs." + ) + parser.add_argument( + "--bus-csv", + type=Path, + required=True, + help="Path to the bus table CSV.", + ) + parser.add_argument( + "--gen-csv", + type=Path, + required=True, + help="Path to the generator table CSV.", + ) + parser.add_argument( + "--branch-csv", + type=Path, + required=True, + help="Path to the branch table CSV.", + ) + parser.add_argument( + "--exclusion-csv", + type=Path, + required=True, + help="Path to the D1 bus exclusion registry CSV.", + ) + parser.add_argument( + "-o", + "--output-dir", + type=Path, + default=None, + help="Output directory (default: data/fnm/reference/dcpf/).", + ) + parser.add_argument( + "--base-mva", + type=float, + default=None, + help="System MVA base. Overrides manifest sbase. Default: 100.0.", + ) + parser.add_argument( + "--canonical-parser", + type=str, + default="", + help="Name of the canonical parser (for metadata).", + ) + parser.add_argument( + "--transformer-csv", + type=Path, + default=None, + help="Path to a separate transformer table CSV (PSS/E column format).", + ) + parser.add_argument( + "--manifest", + type=Path, + default=None, + help="Path to manifest.json sidecar for baseMVA and metadata.", + ) + + args = parser.parse_args(argv if argv is not None else sys.argv[1:]) + + output_dir: Path = args.output_dir or Path("data/fnm/reference/dcpf") + + # Resolve baseMVA: --base-mva > manifest.sbase > 100.0 + manifest: dict | None = None + if args.manifest is not None: + manifest = load_manifest(args.manifest) + + base_mva, base_mva_source = resolve_base_mva(args.base_mva, manifest) + logger.info("baseMVA = %.1f (source: %s)", base_mva, base_mva_source) + + try: + solution = run_dcpf_reference( + bus_csv_path=args.bus_csv, + gen_csv_path=args.gen_csv, + branch_csv_path=args.branch_csv, + exclusion_csv_path=args.exclusion_csv, + output_dir=output_dir, + base_mva=base_mva, + canonical_parser=args.canonical_parser, + transformer_csv_path=args.transformer_csv, + ) + except (ValueError, FileNotFoundError) as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(2) + + # Run validation for exit code + validation = validate_dcpf_solution(solution) + + print(f"Active buses: {solution.active_bus_count}") + print(f"Active branches: {solution.active_branch_count}") + print(f"Total generation: {solution.total_generation_mw:.1f} MW") + print(f"Total load: {solution.total_load_mw:.1f} MW") + print(f"Slack bus: {solution.slack_bus}") + print(f"Validation passed: {validation.all_checks_passed}") + + if not validation.all_checks_passed: + print(f" Power balance residual: {validation.power_balance_residual_mw:.4f} MW") + print(f" Flow-angle max deviation: {validation.flow_angle_max_deviation_mw:.4f} MW") + print(f" Slack angle zero: {validation.slack_angle_zero}") + sys.exit(1) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/data/fnm/scripts/diagnose_acpf.m b/data/fnm/scripts/diagnose_acpf.m new file mode 100644 index 00000000..1c5c6c9c --- /dev/null +++ b/data/fnm/scripts/diagnose_acpf.m @@ -0,0 +1,149 @@ +% diagnose_acpf.m -- Diagnose why ACPF diverges on the FNM. +% +% Checks: islands, bus types, negative impedances, extreme values, Q limits + +script_dir = fileparts(mfilename('fullpath')); +fnm_dir = fullfile(script_dir, '..'); +repo_root = fullfile(fnm_dir, '..', '..'); +matpower_path = fullfile(repo_root, 'evaluations', 'matpower', 'matpower8.1'); +addpath(genpath(matpower_path)); + +mat_path = fullfile(fnm_dir, 'reference', 'matpower_parse', 'mpc_case.mat'); +load(mat_path, 'mpc'); + +fprintf('=== FNM Network Diagnostics ===\n\n'); + +bus = mpc.bus; +gen = mpc.gen; +branch = mpc.branch; + +fprintf('Raw counts: %d buses, %d branches, %d generators\n', ... + size(bus, 1), size(branch, 1), size(gen, 1)); + +% Bus type distribution +types = bus(:, 2); +fprintf('\nBus types:\n'); +fprintf(' Type 1 (PQ): %d\n', sum(types == 1)); +fprintf(' Type 2 (PV): %d\n', sum(types == 2)); +fprintf(' Type 3 (Slack): %d\n', sum(types == 3)); +fprintf(' Type 4 (Isolated): %d\n', sum(types == 4)); + +% Branch status +fprintf('\nBranch status:\n'); +fprintf(' In-service: %d\n', sum(branch(:, 11) == 1)); +fprintf(' Out-of-service: %d\n', sum(branch(:, 11) == 0)); + +% Generator status +fprintf('\nGenerator status:\n'); +fprintf(' In-service: %d\n', sum(gen(:, 8) > 0)); +fprintf(' Out-of-service: %d\n', sum(gen(:, 8) <= 0)); + +% Check for problematic impedances +fprintf('\nBranch impedance issues (in-service only):\n'); +in_service = branch(:, 11) == 1; +br_is = branch(in_service, :); +fprintf(' Zero R and X: %d\n', sum(br_is(:, 3) == 0 & br_is(:, 4) == 0)); +fprintf(' Zero X only: %d\n', sum(br_is(:, 4) == 0 & br_is(:, 3) ~= 0)); +fprintf(' Negative X: %d\n', sum(br_is(:, 4) < 0)); +fprintf(' Negative R: %d\n', sum(br_is(:, 3) < 0)); +fprintf(' X < 1e-6: %d\n', sum(abs(br_is(:, 4)) < 1e-6 & br_is(:, 4) ~= 0)); + +% Check for extreme values +fprintf('\nExtreme branch values (in-service):\n'); +fprintf(' Max |X|: %.6f pu\n', max(abs(br_is(:, 4)))); +fprintf(' Min |X| (nonzero): %.8f pu\n', min(abs(br_is(br_is(:, 4) ~= 0, 4)))); +fprintf(' Max |R|: %.6f pu\n', max(abs(br_is(:, 3)))); +fprintf(' Max tap ratio: %.4f\n', max(br_is(:, 9))); +fprintf(' Min tap ratio (nonzero): %.4f\n', min(br_is(br_is(:, 9) ~= 0, 9))); +fprintf(' Phase shifters (nonzero SHIFT): %d\n', sum(br_is(:, 10) ~= 0)); + +% Check for islands using graph connectivity +fprintf('\n=== Island Detection ===\n'); +% Build adjacency from in-service branches +n_bus = size(bus, 1); +bus_nums = bus(:, 1); +bus_map = containers.Map(bus_nums, 1:n_bus); + +% Union-Find (path compression inline) +parent = 1:n_bus; + +for i = 1:size(br_is, 1) + fb = br_is(i, 1); + tb = br_is(i, 2); + if bus_map.isKey(fb) && bus_map.isKey(tb) + % find root of fb + fi = bus_map(fb); + while parent(fi) ~= fi + fi = parent(fi); + end + % find root of tb + ti = bus_map(tb); + while parent(ti) ~= ti + ti = parent(ti); + end + if fi ~= ti + parent(fi) = ti; + end + end +end + +% Count islands (among non-isolated buses) +active = find(types ~= 4); +island_roots = zeros(length(active), 1); +for i = 1:length(active) + x = active(i); + while parent(x) ~= x + x = parent(x); + end + island_roots(i) = x; +end +[unique_islands, ~, ic] = unique(island_roots); +island_sizes = accumarray(ic, 1); +fprintf('Number of islands (non-isolated buses): %d\n', length(unique_islands)); +fprintf('Island size distribution:\n'); +sorted_sizes = sort(island_sizes, 'descend'); +for i = 1:min(10, length(sorted_sizes)) + fprintf(' Island %d: %d buses\n', i, sorted_sizes(i)); +end + +% Check which islands have a slack bus +fprintf('\nSlack bus distribution across islands:\n'); +slack_buses = find(types == 3); +for i = 1:length(slack_buses) + si = slack_buses(i); + x = si; + while parent(x) ~= x + x = parent(x); + end + island_id = x; + island_size = sum(island_roots == island_id); + fprintf(' Slack bus %d (index %d) in island of %d buses\n', ... + bus(si, 1), si, island_size); +end + +% Check PV buses without generators +pv_buses = bus(types == 2, 1); +gen_buses = unique(gen(gen(:, 8) > 0, 1)); +orphan_pv = setdiff(pv_buses, gen_buses); +fprintf('\nPV buses without in-service generators: %d\n', length(orphan_pv)); + +% Generator Q limits +fprintf('\nGenerator Q-limit issues:\n'); +active_gen = gen(gen(:, 8) > 0, :); +fprintf(' Qmax == Qmin: %d\n', sum(active_gen(:, 4) == active_gen(:, 5))); +fprintf(' Qmax < Qmin: %d\n', sum(active_gen(:, 4) < active_gen(:, 5))); +fprintf(' Qmax == 0 and Qmin == 0: %d\n', ... + sum(active_gen(:, 4) == 0 & active_gen(:, 5) == 0)); + +% Check Vg (voltage setpoint) range for active generators +fprintf('\nGenerator voltage setpoints:\n'); +vg = active_gen(:, 6); +fprintf(' Min Vg: %.4f pu\n', min(vg)); +fprintf(' Max Vg: %.4f pu\n', max(vg)); +fprintf(' Mean Vg: %.4f pu\n', mean(vg)); +fprintf(' Vg == 0: %d\n', sum(vg == 0)); +fprintf(' Vg == 1.0: %d\n', sum(vg == 1.0)); +fprintf(' Vg > 1.1: %d\n', sum(vg > 1.1)); +fprintf(' Vg < 0.9: %d\n', sum(vg < 0.9)); + +fprintf('\n=== Diagnostics Complete ===\n'); diff --git a/data/fnm/scripts/export_intermediate_csvs.py b/data/fnm/scripts/export_intermediate_csvs.py new file mode 100644 index 00000000..79eeaea6 --- /dev/null +++ b/data/fnm/scripts/export_intermediate_csvs.py @@ -0,0 +1,1364 @@ +"""Export Pipeline Script -- MATPOWER .mat to intermediate-format CSVs. + +Reads the cleaned MATPOWER case file, extracts all 17 PSS/E v31 record types +into separate intermediate-format CSV files, writes a sidecar manifest.json, +and validates every output artifact against existing JSON Schema files. + +Key transformations: + - Splits MATPOWER branch matrix into separate branch.csv and transformer.csv + - Converts MATPOWER tap=0 sentinel to PSS/E-standard 1.0 + - Filters to main-island buses using the excluded bus registry +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path + +import jsonschema + +from fnm.scripts.intermediate_schema import ( + FieldSpec, + TableSchema, + get_table_schemas, +) +from fnm.scripts.raw_record_counter import PSSE_V31_SECTION_NAMES + +# --------------------------------------------------------------------------- +# MATPOWER column definitions (0-indexed) +# --------------------------------------------------------------------------- + +# bus columns: bus_i(0), type(1), Pd(2), Qd(3), Gs(4), Bs(5), area(6), +# Vm(7), Va(8), baseKV(9), zone(10), Vmax(11), Vmin(12) +_BUS_COLS = 13 + +# gen columns: bus(0), Pg(1), Qg(2), Qmax(3), Qmin(4), Vg(5), mBase(6), +# status(7), Pmax(8), Pmin(9), ...21 total +_GEN_COLS = 21 + +# branch columns: fbus(0), tbus(1), r(2), x(3), b(4), rateA(5), rateB(6), +# rateC(7), tap(8), shift(9), status(10), angmin(11), angmax(12) +_BRANCH_COLS = 13 + +# gencost: model(0), startup(1), shutdown(2), ncost(3), cost_coeffs(4+) + +# MATPOWER branch column indices for transformer detection +_TAP_COL = 8 +_SHIFT_COL = 9 + + +# --------------------------------------------------------------------------- +# Data structures +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class MatpowerCase: + """Parsed contents of a MATPOWER .mat case file.""" + + baseMVA: float + version: str + bus: list[list[float]] + gen: list[list[float]] + branch: list[list[float]] + gencost: list[list[float]] + areas: list[list[float]] + bus_name: list[str] + dcline: list[list[float]] + + +@dataclass(frozen=True) +class TableExport: + """Metadata about one exported CSV table.""" + + table_name: str + record_type: str + file_name: str + file_path: Path + record_count: int + column_count: int + schema_file: str + + +@dataclass(frozen=True) +class ExportManifest: + """Top-level manifest for the intermediate CSV export.""" + + sbase: float + basfrq: float + rev: float + case_id: str + canonical_parser: str + tables: list[TableExport] + total_records: int + total_tables: int + non_empty_record_types: list[str] + schema_version: str + generated_timestamp: str + + +@dataclass(frozen=True) +class ValidationResult: + """Result of validating one CSV or manifest against its JSON Schema.""" + + artifact_name: str + is_valid: bool + errors: list[str] + rows_checked: int + + +@dataclass +class ExportResult: + """Aggregate result of the full export pipeline.""" + + manifest: ExportManifest + table_exports: list[TableExport] + validations: list[ValidationResult] + output_dir: Path + success: bool + errors: list[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Table name helpers +# --------------------------------------------------------------------------- + +_RECORD_TYPE_TO_TABLE: dict[str, str] = { + rt: rt.lower().replace(" ", "_").replace("-", "_") for rt in PSSE_V31_SECTION_NAMES +} + + +def _table_name(record_type: str) -> str: + return _RECORD_TYPE_TO_TABLE[record_type] + + +# --------------------------------------------------------------------------- +# Core functions +# --------------------------------------------------------------------------- + + +def load_matpower_case(mat_path: Path) -> MatpowerCase: + """Load a MATPOWER .mat case file into a structured container. + + Supports Octave text-format ``.mat`` files (the format produced by + ``save -text``). Falls back to ``scipy.io.loadmat`` for MATLAB v5 binary + files. + + Args: + mat_path: Path to the .mat file. + + Returns: + A MatpowerCase with all parsed matrices. + + Raises: + FileNotFoundError: If the file does not exist. + KeyError: If the .mat file lacks the expected ``mpc`` struct fields. + """ + if not mat_path.exists(): + raise FileNotFoundError(f"MAT file not found: {mat_path}") + + # Detect file format by reading the first line + with open(mat_path, encoding="utf-8", errors="replace") as f: + first_line = f.readline() + + if first_line.startswith("# Created by Octave") or first_line.startswith("#"): + return _load_octave_text_mat(mat_path) + + return _load_scipy_mat(mat_path) + + +def _load_octave_text_mat(mat_path: Path) -> MatpowerCase: + """Parse an Octave text-format .mat file containing a MATPOWER case struct. + + The Octave text format uses ``# name:``, ``# type:``, ``# rows:``, + ``# columns:`` directives followed by data lines. + """ + with open(mat_path, encoding="utf-8", errors="replace") as f: + lines = f.readlines() + + # Parse into named fields within the top-level struct + fields: dict[str, list[list[float]] | float | str | list[str]] = {} + i = 0 + n = len(lines) + + # Skip until we find the struct + while i < n: + line = lines[i].strip() + if line.startswith("# type: scalar struct"): + i += 1 + break + i += 1 + + # Skip ndims and length lines + while i < n: + line = lines[i].strip() + if line.startswith("# length:"): + i += 1 + break + i += 1 + + # Now parse struct fields + while i < n: + line = lines[i].strip() + if not line.startswith("# name:"): + i += 1 + continue + + field_name = line.split(":", 1)[1].strip() + i += 1 + if i >= n: + break + + type_line = lines[i].strip() + if not type_line.startswith("# type:"): + continue + field_type = type_line.split(":", 1)[1].strip() + i += 1 + + if field_type == "scalar": + # Read scalar value + while i < n and (not lines[i].strip() or lines[i].startswith("#")): + i += 1 + if i < n: + try: + fields[field_name] = float(lines[i].strip()) + except ValueError: + fields[field_name] = lines[i].strip() + i += 1 + + elif field_type == "matrix": + # Read rows/columns then data + rows = 0 + while i < n: + rl = lines[i].strip() + if rl.startswith("# rows:"): + rows = int(rl.split(":", 1)[1].strip()) + elif rl.startswith("# columns:"): + _ = int(rl.split(":", 1)[1].strip()) + i += 1 + break + i += 1 + + matrix: list[list[float]] = [] + for _ in range(rows): + if i >= n: + break + data_line = lines[i].strip() + if data_line and not data_line.startswith("#"): + vals = [float(v) for v in data_line.split()] + matrix.append(vals) + i += 1 + fields[field_name] = matrix + + elif field_type == "sq_string": + # Read string value + # Skip elements/length lines + while i < n: + sl = lines[i].strip() + if sl.startswith("# elements:") or sl.startswith("# length:"): + i += 1 + continue + break + if i < n: + fields[field_name] = lines[i].strip() + i += 1 + + elif field_type == "cell": + # Cell array of strings + cell_rows = 0 + while i < n: + cl = lines[i].strip() + if cl.startswith("# rows:"): + cell_rows = int(cl.split(":", 1)[1].strip()) + elif cl.startswith("# columns:"): + i += 1 + break + i += 1 + + string_list: list[str] = [] + for _ in range(cell_rows): + # Skip cell-element and type headers + while i < n: + cl = lines[i].strip() + if cl.startswith("# name: "): + i += 1 + continue + if cl.startswith("# type:"): + i += 1 + continue + if cl.startswith("# elements:"): + i += 1 + continue + if cl.startswith("# length:"): + i += 1 + break + i += 1 + # Read the string value + if i < n: + string_list.append(lines[i].rstrip("\n")) + i += 1 + fields[field_name] = string_list + + else: + i += 1 + + # Extract fields + baseMVA = float(fields.get("baseMVA", 100.0)) + version = str(fields.get("version", "2")) + + def _get_matrix(name: str) -> list[list[float]]: + val = fields.get(name, []) + if isinstance(val, list): + return val # type: ignore[return-value] + return [] + + def _get_strings(name: str) -> list[str]: + val = fields.get(name, []) + if isinstance(val, list) and all(isinstance(v, str) for v in val): + return val # type: ignore[return-value] + return [] + + return MatpowerCase( + baseMVA=baseMVA, + version=version, + bus=_get_matrix("bus"), + gen=_get_matrix("gen"), + branch=_get_matrix("branch"), + gencost=_get_matrix("gencost"), + areas=_get_matrix("areas"), + bus_name=_get_strings("bus_name"), + dcline=_get_matrix("dcline"), + ) + + +def _load_scipy_mat(mat_path: Path) -> MatpowerCase: + """Load a MATLAB v5 binary .mat file using scipy.""" + import scipy.io + + mat = scipy.io.loadmat(str(mat_path), squeeze_me=False) + + mpc = None + for key in mat: + if key.startswith("_"): + continue + val = mat[key] + if hasattr(val, "dtype") and val.dtype.names is not None: + mpc = val + break + + if mpc is None: + raise KeyError("No MATPOWER case struct found in .mat file") + + def _extract_scalar(name: str, default: float | str = 0.0) -> float | str: + try: + v = mpc[name][0, 0] + if hasattr(v, "flat"): + return float(v.flat[0]) + return v + except (KeyError, IndexError, ValueError): + return default + + def _extract_matrix(name: str) -> list[list[float]]: + try: + arr = mpc[name][0, 0] + return arr.tolist() + except (KeyError, IndexError): + return [] + + def _extract_strings(name: str) -> list[str]: + try: + arr = mpc[name][0, 0] + result: list[str] = [] + for row in arr: + if hasattr(row, "__len__") and not isinstance(row, str): + if len(row) > 0 and hasattr(row[0], "__len__"): + result.append(str(row[0]).strip()) + else: + result.append(str(row).strip()) + else: + result.append(str(row).strip()) + return result + except (KeyError, IndexError): + return [] + + baseMVA = float(_extract_scalar("baseMVA", 100.0)) + version = str(_extract_scalar("version", "2")) + + return MatpowerCase( + baseMVA=baseMVA, + version=version, + bus=_extract_matrix("bus"), + gen=_extract_matrix("gen"), + branch=_extract_matrix("branch"), + gencost=_extract_matrix("gencost"), + areas=_extract_matrix("areas"), + bus_name=_extract_strings("bus_name"), + dcline=_extract_matrix("dcline"), + ) + + +def load_excluded_buses(excluded_buses_path: Path) -> set[int]: + """Load the set of excluded bus numbers from the JSON registry. + + Supports both formats: + - Full registry JSON with top-level ``excluded_buses`` array of objects + (each with a ``bus_number`` field) + - Simple JSON array of integers + + Args: + excluded_buses_path: Path to the excluded buses JSON file. + + Returns: + Set of integer bus numbers to exclude. + + Raises: + FileNotFoundError: If the file does not exist. + """ + if not excluded_buses_path.exists(): + raise FileNotFoundError(f"Excluded buses file not found: {excluded_buses_path}") + + data = json.loads(excluded_buses_path.read_text(encoding="utf-8")) + + if isinstance(data, list): + return {int(b) for b in data} + + # Full registry format + buses = data.get("excluded_buses", []) + return {int(b["bus_number"]) for b in buses} + + +def normalize_tap_ratio(tap: float) -> float: + """Convert MATPOWER tap=0 sentinel to PSS/E-standard 1.0. + + In MATPOWER, a tap ratio of 0.0 means "nominal turns ratio" (i.e. 1.0). + PSS/E uses an explicit 1.0 value instead. This function performs that + conversion. + + Args: + tap: The tap ratio value from MATPOWER. + + Returns: + 1.0 if tap == 0.0, otherwise the original value. + """ + return 1.0 if tap == 0.0 else tap + + +def split_branches_and_transformers( + branch_matrix: list[list[float]], + bus_numbers: set[int], +) -> tuple[list[dict[str, int | float | str]], list[dict[str, int | float | str]]]: + """Split the MATPOWER branch matrix into branch and transformer rows. + + A row is classified as a transformer if tap ratio (col 8) != 0 or + phase shift (col 9) != 0. Plain branches have both at 0. + + Args: + branch_matrix: The MATPOWER branch matrix (list of rows). + bus_numbers: Set of main-island bus numbers for filtering. + + Returns: + Tuple of (branch_rows, transformer_rows) where each row is a dict + with PSS/E field names as keys. + """ + branch_schema = _get_schema_by_record_type("Branch") + transformer_schema = _get_schema_by_record_type("Transformer") + + branch_fields = branch_schema.fields + transformer_fields = transformer_schema.fields + + branches: list[dict[str, int | float | str]] = [] + transformers: list[dict[str, int | float | str]] = [] + + for row in branch_matrix: + fbus = int(row[0]) + tbus = int(row[1]) + + # Filter: both endpoints must be in bus_numbers + if fbus not in bus_numbers or tbus not in bus_numbers: + continue + + tap = row[_TAP_COL] if len(row) > _TAP_COL else 0.0 + shift = row[_SHIFT_COL] if len(row) > _SHIFT_COL else 0.0 + + is_transformer = (tap != 0.0) or (shift != 0.0) + + if is_transformer: + xfmr_row = _matpower_branch_to_transformer(row, transformer_fields) + transformers.append(xfmr_row) + else: + br_row = _matpower_branch_to_branch(row, branch_fields) + branches.append(br_row) + + return branches, transformers + + +def _matpower_branch_to_branch( + row: list[float], + fields: list[FieldSpec], +) -> dict[str, int | float | str]: + """Convert a MATPOWER branch row to a PSS/E branch dict.""" + # MATPOWER branch: fbus(0), tbus(1), r(2), x(3), b(4), rateA(5), + # rateB(6), rateC(7), tap(8), shift(9), status(10), angmin(11), angmax(12) + d: dict[str, int | float | str] = {} + + mapping = { + "I": (0, "integer"), + "J": (1, "integer"), + "CKT": (None, "string"), # default + "R": (2, "number"), + "X": (3, "number"), + "B": (4, "number"), + "RATEA": (5, "number"), + "RATEB": (6, "number"), + "RATEC": (7, "number"), + "GI": (None, "number"), + "BI": (None, "number"), + "GJ": (None, "number"), + "BJ": (None, "number"), + "ST": (10, "integer"), + "MET": (None, "integer"), + "LEN": (None, "number"), + "O1": (None, "integer"), + "F1": (None, "number"), + "O2": (None, "integer"), + "F2": (None, "number"), + "O3": (None, "integer"), + "F3": (None, "number"), + "O4": (None, "integer"), + "F4": (None, "number"), + } + + for f in fields: + if f.name in mapping: + col_idx, dtype = mapping[f.name] + if col_idx is not None and col_idx < len(row): + val = row[col_idx] + d[f.name] = _cast_value(val, dtype) + else: + d[f.name] = _default_for_field(f) + else: + d[f.name] = _default_for_field(f) + + return d + + +def _matpower_branch_to_transformer( + row: list[float], + fields: list[FieldSpec], +) -> dict[str, int | float | str]: + """Convert a MATPOWER branch row (transformer) to PSS/E transformer dict.""" + d: dict[str, int | float | str] = {} + + # Map MATPOWER branch columns to PSS/E transformer fields + for f in fields: + d[f.name] = _default_for_field(f) + + # Line 1 -- common identifiers + d["I"] = int(row[0]) + d["J"] = int(row[1]) + d["K"] = 0 # 2-winding + d["CKT"] = "1 " + d["CW"] = 1 + d["CZ"] = 1 + d["CM"] = 1 + d["STAT"] = int(row[10]) if len(row) > 10 else 1 + + # Line 2 -- impedance + d["R1_2"] = row[2] + d["X1_2"] = row[3] + d["SBASE1_2"] = 100.0 + + # Line 3 -- winding 1 + tap = row[_TAP_COL] if len(row) > _TAP_COL else 0.0 + d["WINDV1"] = normalize_tap_ratio(tap) + shift = row[_SHIFT_COL] if len(row) > _SHIFT_COL else 0.0 + d["ANG1"] = shift + + # Ratings + d["RATA1"] = row[5] if len(row) > 5 else 0.0 + d["RATB1"] = row[6] if len(row) > 6 else 0.0 + d["RATC1"] = row[7] if len(row) > 7 else 0.0 + + # Winding 2 (MATPOWER doesn't store winding-2 tap -- uses 1.0 default) + d["WINDV2"] = 1.0 + + return d + + +def _get_schema_by_record_type(record_type: str) -> TableSchema: + """Look up a TableSchema by PSS/E record type name.""" + for ts in get_table_schemas(): + if ts.record_type == record_type: + return ts + msg = f"Unknown record type: {record_type}" + raise ValueError(msg) + + +def _default_for_field(f: FieldSpec) -> int | float | str: + """Return the default value for a field spec.""" + if f.default_value is not None: + if f.data_type == "integer": + return int(f.default_value) + if f.data_type == "number": + return float(f.default_value) + return str(f.default_value) + # Required fields with no default -- use type-appropriate zero + if f.data_type == "integer": + return 0 + if f.data_type == "number": + return 0.0 + return "" + + +def _cast_value(val: float, dtype: str) -> int | float | str: + """Cast a numeric value to the specified type.""" + if dtype == "integer": + return int(val) + if dtype == "number": + return float(val) + return str(val) + + +def filter_rows_by_bus( + rows: list[dict[str, int | float | str]], + bus_numbers: set[int], + bus_key: str = "I", +) -> list[dict[str, int | float | str]]: + """Filter table rows to retain only those referencing main-island buses. + + Args: + rows: List of row dicts. + bus_numbers: Set of valid (main-island) bus numbers. + bus_key: Column name containing the bus number to check. + + Returns: + Filtered list of rows where the bus_key value is in bus_numbers. + """ + return [r for r in rows if int(r[bus_key]) in bus_numbers] + + +def _filter_branch_rows_by_bus( + rows: list[dict[str, int | float | str]], + bus_numbers: set[int], +) -> list[dict[str, int | float | str]]: + """Filter branch/transformer rows requiring both I and J in bus_numbers.""" + return [r for r in rows if int(r["I"]) in bus_numbers and int(r["J"]) in bus_numbers] + + +# --------------------------------------------------------------------------- +# MATPOWER -> intermediate format conversion helpers +# --------------------------------------------------------------------------- + + +def _matpower_bus_to_psse( + bus_matrix: list[list[float]], + bus_names: list[str], + bus_numbers: set[int], +) -> list[dict[str, int | float | str]]: + """Convert MATPOWER bus matrix rows to PSS/E bus dicts.""" + rows: list[dict[str, int | float | str]] = [] + + for idx, brow in enumerate(bus_matrix): + bus_num = int(brow[0]) + if bus_num not in bus_numbers: + continue + + d: dict[str, int | float | str] = {} + # I, NAME, BASKV, IDE, AREA, ZONE, OWNER, VM, VA, NVHI, NVLO, EVHI, EVLO + d["I"] = bus_num + d["NAME"] = bus_names[idx] if idx < len(bus_names) else " " + d["BASKV"] = brow[9] if len(brow) > 9 else 0.0 + d["IDE"] = int(brow[1]) if len(brow) > 1 else 1 + d["AREA"] = int(brow[6]) if len(brow) > 6 else 1 + d["ZONE"] = int(brow[10]) if len(brow) > 10 else 1 + d["OWNER"] = 1 # MATPOWER doesn't store owner per bus + d["VM"] = brow[7] if len(brow) > 7 else 1.0 + d["VA"] = brow[8] if len(brow) > 8 else 0.0 + d["NVHI"] = brow[11] if len(brow) > 11 else 1.1 + d["NVLO"] = brow[12] if len(brow) > 12 else 0.9 + d["EVHI"] = brow[11] if len(brow) > 11 else 1.1 + d["EVLO"] = brow[12] if len(brow) > 12 else 0.9 + + rows.append(d) + return rows + + +def _matpower_gen_to_psse( + gen_matrix: list[list[float]], + bus_numbers: set[int], +) -> list[dict[str, int | float | str]]: + """Convert MATPOWER gen matrix rows to PSS/E generator dicts.""" + rows: list[dict[str, int | float | str]] = [] + + # Track per-bus generator count for ID assignment + bus_gen_count: dict[int, int] = {} + + for grow in gen_matrix: + bus_num = int(grow[0]) + if bus_num not in bus_numbers: + continue + + bus_gen_count[bus_num] = bus_gen_count.get(bus_num, 0) + 1 + gen_id = str(bus_gen_count[bus_num]) + if len(gen_id) < 2: + gen_id = gen_id + " " + + d: dict[str, int | float | str] = {} + d["I"] = bus_num + d["ID"] = gen_id + d["PG"] = grow[1] if len(grow) > 1 else 0.0 + d["QG"] = grow[2] if len(grow) > 2 else 0.0 + d["QT"] = grow[3] if len(grow) > 3 else 9999.0 + d["QB"] = grow[4] if len(grow) > 4 else -9999.0 + d["VS"] = grow[5] if len(grow) > 5 else 1.0 + d["IREG"] = 0 + d["MBASE"] = grow[6] if len(grow) > 6 else 100.0 + d["ZR"] = 0.0 + d["ZX"] = 1.0 + d["RT"] = 0.0 + d["XT"] = 0.0 + d["GTAP"] = 1.0 + d["STAT"] = int(grow[7]) if len(grow) > 7 else 1 + d["RMPCT"] = 100.0 + d["PT"] = grow[8] if len(grow) > 8 else 9999.0 + d["PB"] = grow[9] if len(grow) > 9 else -9999.0 + d["O1"] = 1 + d["F1"] = 1.0 + d["O2"] = 0 + d["F2"] = 0.0 + d["O3"] = 0 + d["F3"] = 0.0 + d["O4"] = 0 + d["F4"] = 0.0 + d["WMOD"] = 0 + d["WPF"] = 1.0 + + rows.append(d) + return rows + + +def _matpower_bus_to_load( + bus_matrix: list[list[float]], + bus_numbers: set[int], +) -> list[dict[str, int | float | str]]: + """Extract load records from MATPOWER bus matrix (Pd, Qd columns).""" + rows: list[dict[str, int | float | str]] = [] + + for brow in bus_matrix: + bus_num = int(brow[0]) + if bus_num not in bus_numbers: + continue + + pd = brow[2] if len(brow) > 2 else 0.0 + qd = brow[3] if len(brow) > 3 else 0.0 + + # Only create load record if nonzero + if pd == 0.0 and qd == 0.0: + continue + + d: dict[str, int | float | str] = {} + d["I"] = bus_num + d["ID"] = "1 " + d["STATUS"] = 1 + d["AREA"] = int(brow[6]) if len(brow) > 6 else 1 + d["ZONE"] = int(brow[10]) if len(brow) > 10 else 1 + d["PL"] = pd + d["QL"] = qd + d["IP"] = 0.0 + d["IQ"] = 0.0 + d["YP"] = 0.0 + d["YQ"] = 0.0 + d["OWNER"] = 1 + d["SCALE"] = 1 + + rows.append(d) + return rows + + +def _matpower_bus_to_fixed_shunt( + bus_matrix: list[list[float]], + bus_numbers: set[int], +) -> list[dict[str, int | float | str]]: + """Extract fixed shunt records from MATPOWER bus matrix (Gs, Bs columns).""" + rows: list[dict[str, int | float | str]] = [] + + for brow in bus_matrix: + bus_num = int(brow[0]) + if bus_num not in bus_numbers: + continue + + gs = brow[4] if len(brow) > 4 else 0.0 + bs = brow[5] if len(brow) > 5 else 0.0 + + if gs == 0.0 and bs == 0.0: + continue + + d: dict[str, int | float | str] = {} + d["I"] = bus_num + d["ID"] = "1 " + d["STATUS"] = 1 + d["GL"] = gs + d["BL"] = bs + + rows.append(d) + return rows + + +def _matpower_areas_to_psse( + areas_matrix: list[list[float]], +) -> list[dict[str, int | float | str]]: + """Convert MATPOWER areas matrix to PSS/E area dicts.""" + rows: list[dict[str, int | float | str]] = [] + for arow in areas_matrix: + if len(arow) < 2: + continue + d: dict[str, int | float | str] = {} + d["I"] = int(arow[0]) + d["ISW"] = 0 + d["PDES"] = arow[1] if len(arow) > 1 else 0.0 + d["PTOL"] = 10.0 + d["ARNAME"] = " " + rows.append(d) + return rows + + +def _extract_zones( + bus_matrix: list[list[float]], + bus_numbers: set[int], +) -> list[dict[str, int | float | str]]: + """Extract unique zones from bus data.""" + zones: set[int] = set() + for brow in bus_matrix: + bus_num = int(brow[0]) + if bus_num not in bus_numbers: + continue + zone = int(brow[10]) if len(brow) > 10 else 1 + zones.add(zone) + + return [{"I": z, "ZONAME": " "} for z in sorted(zones)] + + +def _extract_owners( + bus_numbers: set[int], +) -> list[dict[str, int | float | str]]: + """Create a minimal owner record. MATPOWER doesn't store owner data.""" + return [{"I": 1, "OWNAME": " "}] + + +def _extract_switched_shunts( + bus_matrix: list[list[float]], + bus_numbers: set[int], +) -> list[dict[str, int | float | str]]: + """MATPOWER merges switched shunts into fixed -- return empty list.""" + return [] + + +# --------------------------------------------------------------------------- +# CSV export +# --------------------------------------------------------------------------- + + +def export_table_to_csv( + rows: list[dict[str, int | float | str]], + schema_path: Path, + output_path: Path, +) -> TableExport: + """Write a list of row dicts to a CSV file with column order from the schema. + + Column order is determined by the ``properties`` key order in the JSON Schema + file. Integer-typed fields are written without decimal suffixes. + + Args: + rows: List of row dicts to write. + schema_path: Path to the JSON Schema for this table. + output_path: Path to write the CSV file. + + Returns: + A TableExport metadata record. + """ + schema = json.loads(schema_path.read_text(encoding="utf-8")) + columns = list(schema["properties"].keys()) + field_types = { + name: props.get("type", "string") for name, props in schema["properties"].items() + } + + output_path.parent.mkdir(parents=True, exist_ok=True) + + with open(output_path, "w", encoding="utf-8", newline="") as f: + writer = csv.DictWriter(f, fieldnames=columns, extrasaction="ignore") + writer.writeheader() + for row in rows: + # Format values: integers without .0 + formatted: dict[str, str] = {} + for col in columns: + val = row.get(col, "") + ftype = field_types.get(col, "string") + if ftype == "integer" and val != "" and val is not None: + try: + formatted[col] = str(int(float(val))) + except (ValueError, TypeError): + formatted[col] = str(val) + elif ftype == "number" and val != "" and val is not None: + formatted[col] = str(float(val)) + else: + formatted[col] = str(val) if val is not None else "" + writer.writerow(formatted) + + record_type = schema.get("title", output_path.stem) + table_name = output_path.stem + + return TableExport( + table_name=table_name, + record_type=record_type, + file_name=output_path.name, + file_path=output_path, + record_count=len(rows), + column_count=len(columns), + schema_file=schema_path.name, + ) + + +# --------------------------------------------------------------------------- +# Manifest +# --------------------------------------------------------------------------- + + +def build_manifest( + case: MatpowerCase, + table_exports: list[TableExport], + schema_version: str = "1.0", +) -> ExportManifest: + """Construct the export manifest from case metadata and table exports. + + Args: + case: The parsed MATPOWER case. + table_exports: List of TableExport records from CSV writing. + schema_version: Schema version string. + + Returns: + An ExportManifest with all fields populated. + """ + total_records = sum(te.record_count for te in table_exports) + non_empty = [te.record_type for te in table_exports if te.record_count > 0] + + return ExportManifest( + sbase=case.baseMVA, + basfrq=60.0, # North American grids use 60 Hz + rev=31.0, + case_id="fnm_main_island", + canonical_parser="matpower", + tables=table_exports, + total_records=total_records, + total_tables=len(table_exports), + non_empty_record_types=non_empty, + schema_version=schema_version, + generated_timestamp=datetime.now(timezone.utc).isoformat(), + ) + + +def write_manifest(manifest: ExportManifest, output_path: Path) -> None: + """Serialize an ExportManifest to JSON and write to disk. + + Args: + manifest: The manifest to serialize. + output_path: Path to write the JSON file. + """ + data = { + "sbase": manifest.sbase, + "basfrq": manifest.basfrq, + "rev": manifest.rev, + "case_id": manifest.case_id, + "canonical_parser": manifest.canonical_parser, + "tables": [ + { + "table_name": te.table_name, + "record_type": te.record_type, + "file_name": te.file_name, + "record_count": te.record_count, + "column_count": te.column_count, + "schema_file": te.schema_file, + } + for te in manifest.tables + ], + "total_records": manifest.total_records, + "total_tables": manifest.total_tables, + "non_empty_record_types": manifest.non_empty_record_types, + "schema_version": manifest.schema_version, + "generated_timestamp": manifest.generated_timestamp, + } + + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text( + json.dumps(data, indent=2) + "\n", + encoding="utf-8", + ) + + +# --------------------------------------------------------------------------- +# Validation +# --------------------------------------------------------------------------- + + +def validate_csv_against_schema( + csv_path: Path, + schema_path: Path, +) -> ValidationResult: + """Validate every row of a CSV file against its JSON Schema. + + Each row is converted to a dict using the CSV header, cast to the schema's + declared types, and validated with ``jsonschema.validate()``. + + Args: + csv_path: Path to the CSV file. + schema_path: Path to the JSON Schema file. + + Returns: + A ValidationResult indicating pass/fail and any error messages. + """ + schema = json.loads(schema_path.read_text(encoding="utf-8")) + errors: list[str] = [] + rows_checked = 0 + + if not csv_path.exists(): + return ValidationResult( + artifact_name=csv_path.name, + is_valid=False, + errors=[f"CSV not found: {csv_path}"], + rows_checked=0, + ) + + with open(csv_path, encoding="utf-8") as f: + reader = csv.DictReader(f) + for idx, row in enumerate(reader): + rows_checked += 1 + typed_row = _cast_csv_row(row, schema) + try: + jsonschema.validate(instance=typed_row, schema=schema) + except jsonschema.ValidationError as e: + errors.append(f"Row {idx}: {e.message}") + + return ValidationResult( + artifact_name=csv_path.name, + is_valid=len(errors) == 0, + errors=errors, + rows_checked=rows_checked, + ) + + +def validate_manifest_against_schema( + manifest_path: Path, + schema_path: Path, +) -> ValidationResult: + """Validate a manifest.json file against manifest.schema.json. + + Args: + manifest_path: Path to the manifest JSON file. + schema_path: Path to the manifest JSON Schema file. + + Returns: + A ValidationResult indicating pass/fail and any error messages. + """ + errors: list[str] = [] + + if not manifest_path.exists(): + return ValidationResult( + artifact_name="manifest.json", + is_valid=False, + errors=["Manifest not found"], + rows_checked=0, + ) + + manifest_data = json.loads(manifest_path.read_text(encoding="utf-8")) + schema = json.loads(schema_path.read_text(encoding="utf-8")) + + try: + jsonschema.validate(instance=manifest_data, schema=schema) + except jsonschema.ValidationError as e: + errors.append(e.message) + + return ValidationResult( + artifact_name="manifest.json", + is_valid=len(errors) == 0, + errors=errors, + rows_checked=1, + ) + + +def _cast_csv_row( + row: dict[str, str], + schema: dict, +) -> dict[str, int | float | str]: + """Cast a CSV row's string values to the types declared in the schema.""" + properties = schema.get("properties", {}) + typed: dict[str, int | float | str] = {} + + for col, val in row.items(): + if col not in properties: + typed[col] = val + continue + dtype = properties[col].get("type", "string") + if val == "" or val is None: + # Skip empty optional values + continue + try: + if dtype == "integer": + typed[col] = int(float(val)) + elif dtype == "number": + typed[col] = float(val) + else: + typed[col] = val + except (ValueError, TypeError): + typed[col] = val + + return typed + + +# --------------------------------------------------------------------------- +# Pipeline orchestration +# --------------------------------------------------------------------------- + + +def run_export_pipeline( + mat_path: Path, + excluded_buses_path: Path, + schema_dir: Path, + output_dir: Path, +) -> ExportResult: + """Orchestrate the full export pipeline. + + 1. Load the MATPOWER .mat case + 2. Load excluded buses + 3. Compute main-island bus set + 4. For each PSS/E v31 record type, extract data, filter, export CSV + 5. Build and write manifest + 6. Validate all artifacts + + Args: + mat_path: Path to the cleaned .mat file. + excluded_buses_path: Path to the excluded buses JSON. + schema_dir: Path to the directory containing JSON Schema files. + output_dir: Path to write CSV files and manifest. + + Returns: + An ExportResult with all metadata and validation results. + """ + errors: list[str] = [] + output_dir.mkdir(parents=True, exist_ok=True) + + # 1. Load case + case = load_matpower_case(mat_path) + + # 2. Load excluded buses + excluded = load_excluded_buses(excluded_buses_path) + + # 3. Compute main-island bus set + all_bus_nums = {int(row[0]) for row in case.bus} + main_island_buses = all_bus_nums - excluded + + # 4. Extract and export each record type + table_exports: list[TableExport] = [] + + # Map record types to extraction functions + record_type_data = _extract_all_record_types(case, main_island_buses) + + for record_type in PSSE_V31_SECTION_NAMES: + table_name = _table_name(record_type) + schema_file = f"{table_name}.schema.json" + schema_path = schema_dir / schema_file + csv_path = output_dir / f"{table_name}.csv" + + rows = record_type_data.get(record_type, []) + + if schema_path.exists(): + te = export_table_to_csv(rows, schema_path, csv_path) + else: + # Write with known column order from field definitions + te = _export_table_without_schema(rows, record_type, table_name, csv_path) + + table_exports.append(te) + + # 5. Build and write manifest + manifest = build_manifest(case, table_exports) + manifest_path = output_dir / "manifest.json" + write_manifest(manifest, manifest_path) + + # 6. Validate + validations: list[ValidationResult] = [] + + for te in table_exports: + schema_path = schema_dir / te.schema_file + if schema_path.exists(): + vr = validate_csv_against_schema(te.file_path, schema_path) + validations.append(vr) + + manifest_schema_path = schema_dir / "manifest.schema.json" + if manifest_schema_path.exists(): + mv = validate_manifest_against_schema(manifest_path, manifest_schema_path) + validations.append(mv) + + # Check for failures + for vr in validations: + if not vr.is_valid: + errors.extend(f"{vr.artifact_name}: {e}" for e in vr.errors) + + return ExportResult( + manifest=manifest, + table_exports=table_exports, + validations=validations, + output_dir=output_dir, + success=len(errors) == 0, + errors=errors, + ) + + +def _extract_all_record_types( + case: MatpowerCase, + bus_numbers: set[int], +) -> dict[str, list[dict[str, int | float | str]]]: + """Extract data for all 17 PSS/E v31 record types from the MATPOWER case.""" + # Split branches and transformers + branches, transformers = split_branches_and_transformers(case.branch, bus_numbers) + + data: dict[str, list[dict[str, int | float | str]]] = {} + + data["Bus"] = _matpower_bus_to_psse(case.bus, case.bus_name, bus_numbers) + data["Load"] = _matpower_bus_to_load(case.bus, bus_numbers) + data["Fixed Shunt"] = _matpower_bus_to_fixed_shunt(case.bus, bus_numbers) + data["Generator"] = _matpower_gen_to_psse(case.gen, bus_numbers) + data["Branch"] = branches + data["Transformer"] = transformers + data["Area"] = _matpower_areas_to_psse(case.areas) + data["Zone"] = _extract_zones(case.bus, bus_numbers) + data["Owner"] = _extract_owners(bus_numbers) + data["Switched Shunt"] = _extract_switched_shunts(case.bus, bus_numbers) + + # Record types MATPOWER drops entirely -- empty + for rt in ( + "Two-Terminal DC", + "VSC DC", + "Impedance Correction", + "Multi-Terminal DC", + "Multi-Section Line", + "FACTS", + "Interarea Transfer", + ): + data[rt] = [] + + return data + + +def _export_table_without_schema( + rows: list[dict[str, int | float | str]], + record_type: str, + table_name: str, + output_path: Path, +) -> TableExport: + """Export a table using field definitions from intermediate_schema.py.""" + schema = _get_schema_by_record_type(record_type) + columns = [f.name for f in schema.fields] + field_types = {f.name: f.data_type for f in schema.fields} + + output_path.parent.mkdir(parents=True, exist_ok=True) + + with open(output_path, "w", encoding="utf-8", newline="") as f: + writer = csv.DictWriter(f, fieldnames=columns, extrasaction="ignore") + writer.writeheader() + for row in rows: + formatted: dict[str, str] = {} + for col in columns: + val = row.get(col, "") + ftype = field_types.get(col, "string") + if ftype == "integer" and val != "" and val is not None: + try: + formatted[col] = str(int(float(val))) + except (ValueError, TypeError): + formatted[col] = str(val) + elif ftype == "number" and val != "" and val is not None: + formatted[col] = str(float(val)) + else: + formatted[col] = str(val) if val is not None else "" + writer.writerow(formatted) + + return TableExport( + table_name=table_name, + record_type=record_type, + file_name=output_path.name, + file_path=output_path, + record_count=len(rows), + column_count=len(columns), + schema_file=f"{table_name}.schema.json", + ) + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> None: + """CLI entry point for the export pipeline. + + Usage:: + + python -m fnm.scripts.export_intermediate_csvs \\ + --mat-path data/fnm/reference/cleaned/fnm_main_island.mat \\ + --excluded-buses data/fnm/reference/excluded_buses.json \\ + --schema-dir data/fnm/intermediate/schemas \\ + --output-dir data/fnm/intermediate/tables + """ + parser = argparse.ArgumentParser( + description="Export MATPOWER .mat case to intermediate-format CSVs." + ) + parser.add_argument( + "--mat-path", + type=Path, + required=True, + help="Path to the cleaned MATPOWER .mat case file.", + ) + parser.add_argument( + "--excluded-buses", + type=Path, + required=True, + help="Path to the excluded buses JSON file.", + ) + parser.add_argument( + "--schema-dir", + type=Path, + required=True, + help="Path to the JSON Schema directory.", + ) + parser.add_argument( + "--output-dir", + type=Path, + required=True, + help="Output directory for CSV files and manifest.", + ) + args = parser.parse_args(argv) + + result = run_export_pipeline( + mat_path=args.mat_path, + excluded_buses_path=args.excluded_buses, + schema_dir=args.schema_dir, + output_dir=args.output_dir, + ) + + if result.success: + print("Export pipeline completed successfully.") + print(f"Output directory: {result.output_dir}") + print(f"Tables: {result.manifest.total_tables}") + print(f"Records: {result.manifest.total_records}") + print(f"Non-empty types: {len(result.manifest.non_empty_record_types)}") + else: + print("Export pipeline FAILED:", file=sys.stderr) + for err in result.errors: + print(f" {err}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/data/fnm/scripts/fnm_gating.py b/data/fnm/scripts/fnm_gating.py new file mode 100644 index 00000000..f2f9d2eb --- /dev/null +++ b/data/fnm/scripts/fnm_gating.py @@ -0,0 +1,240 @@ +"""FNM_PATH gating infrastructure. + +Provides resolution and validation of the FNM_PATH environment variable against +the FNM manifest, enabling graceful test skipping and developer onboarding. +""" + +from __future__ import annotations + +import enum +import os +from dataclasses import dataclass, field +from pathlib import Path + +from fnm.scripts.manifest_io import FnmManifest, load_manifest + + +class FnmFileStatus(enum.Enum): + """Status of an individual FNM source file check.""" + + FOUND = "found" + MISSING = "missing" + + +@dataclass(frozen=True) +class FnmFileCheck: + """Result of checking a single FNM source file. + + Attributes: + relative_path: Relative path within the FNM directory. + expected_name: Base filename expected by the manifest. + status: Whether the file was found or missing. + absolute_path: Resolved absolute path, or None if not found. + """ + + relative_path: str + expected_name: str + status: FnmFileStatus + absolute_path: Path | None + + +class FnmPathStatus(enum.Enum): + """Overall status of FNM_PATH resolution.""" + + VALID = "valid" + PARTIAL = "partial" + INVALID_PATH = "invalid_path" + NOT_SET = "not_set" + MANIFEST_ERROR = "manifest_error" + + +@dataclass(frozen=True) +class FnmPathResult: + """Result of resolving and validating FNM_PATH. + + Attributes: + status: Overall status of the resolution. + fnm_path: Resolved absolute path, or None if not set. + file_checks: Per-file check results from manifest validation. + message: Human-readable summary message. + """ + + status: FnmPathStatus + fnm_path: Path | None + file_checks: list[FnmFileCheck] = field(default_factory=list) + message: str = "" + + @property + def is_usable(self) -> bool: + """Whether the FNM path is usable (VALID or PARTIAL).""" + return self.status in (FnmPathStatus.VALID, FnmPathStatus.PARTIAL) + + @property + def found_files(self) -> list[FnmFileCheck]: + """File checks with FOUND status.""" + return [f for f in self.file_checks if f.status == FnmFileStatus.FOUND] + + @property + def missing_files(self) -> list[FnmFileCheck]: + """File checks with MISSING status.""" + return [f for f in self.file_checks if f.status == FnmFileStatus.MISSING] + + @property + def skip_reason(self) -> str: + """Pytest skip message explaining why FNM data is unavailable.""" + if self.status == FnmPathStatus.NOT_SET: + return ( + "FNM_PATH environment variable is not set. " + "Set FNM_PATH to the directory containing FNM Annual S01 data files. " + "See data/fnm/README.md for setup instructions." + ) + if self.status == FnmPathStatus.INVALID_PATH: + return ( + f"FNM_PATH is set to '{self.fnm_path}' but this is not a valid directory. " + "Verify the path exists and contains the FNM data files." + ) + if self.status == FnmPathStatus.MANIFEST_ERROR: + return ( + "Could not load the FNM manifest file (data/fnm/manifest.json). " + "Ensure PRD 01 deliverables are in place." + ) + if self.status == FnmPathStatus.PARTIAL: + missing = ", ".join(f.expected_name for f in self.missing_files) + return ( + f"FNM_PATH is set but {len(self.missing_files)} required file(s) are missing: " + f"{missing}. Verify your FNM data directory is complete." + ) + return "" + + +def find_repo_root(start: Path | None = None) -> Path: + """Walk up from start to find the repository root. + + Looks for a directory containing either a ``.git`` entry or an + ``evaluation_guides/`` subdirectory. + + Args: + start: Starting directory. Defaults to the current working directory. + + Returns: + The repository root path. + + Raises: + FileNotFoundError: If no repo root can be found. + """ + current = (start or Path.cwd()).resolve() + for parent in [current, *current.parents]: + if (parent / ".git").exists() or (parent / "evaluation_guides").is_dir(): + return parent + raise FileNotFoundError( + f"Could not find repository root from {current}. " + "Expected .git or evaluation_guides/ directory." + ) + + +def load_fnm_manifest(manifest_path: Path) -> FnmManifest: + """Load the FNM manifest, delegating to manifest_io.load_manifest(). + + Args: + manifest_path: Path to the manifest JSON file. + + Returns: + The deserialized FnmManifest. + """ + return load_manifest(manifest_path) + + +def resolve_fnm_path( + *, + env_var: str = "FNM_PATH", + manifest_path: Path | None = None, +) -> FnmPathResult: + """Resolve FNM_PATH env var and validate expected files against the manifest. + + Args: + env_var: Name of the environment variable to read (default: ``FNM_PATH``). + manifest_path: Explicit path to the manifest JSON file. If None, the + function locates it relative to the repository root. + + Returns: + An FnmPathResult describing the resolution outcome. + """ + raw_value = os.environ.get(env_var) + if raw_value is None: + return FnmPathResult( + status=FnmPathStatus.NOT_SET, + fnm_path=None, + message=f"{env_var} environment variable is not set.", + ) + + fnm_path = Path(raw_value).expanduser().resolve() + if not fnm_path.is_dir(): + return FnmPathResult( + status=FnmPathStatus.INVALID_PATH, + fnm_path=fnm_path, + message=f"{env_var} points to '{fnm_path}' which is not a valid directory.", + ) + + # Locate manifest + if manifest_path is None: + try: + repo_root = find_repo_root() + manifest_path = repo_root / "data" / "fnm" / "manifest.json" + except FileNotFoundError: + return FnmPathResult( + status=FnmPathStatus.MANIFEST_ERROR, + fnm_path=fnm_path, + message="Could not locate repository root to find manifest.json.", + ) + + try: + manifest = load_fnm_manifest(manifest_path) + except (FileNotFoundError, ValueError) as exc: + return FnmPathResult( + status=FnmPathStatus.MANIFEST_ERROR, + fnm_path=fnm_path, + message=f"Failed to load manifest: {exc}", + ) + + # Check each source file + file_checks: list[FnmFileCheck] = [] + for entry in manifest.source_files: + abs_path = fnm_path / entry.file_name + if abs_path.exists(): + file_checks.append( + FnmFileCheck( + relative_path=entry.file_name, + expected_name=entry.file_name, + status=FnmFileStatus.FOUND, + absolute_path=abs_path, + ) + ) + else: + file_checks.append( + FnmFileCheck( + relative_path=entry.file_name, + expected_name=entry.file_name, + status=FnmFileStatus.MISSING, + absolute_path=None, + ) + ) + + found_count = sum(1 for fc in file_checks if fc.status == FnmFileStatus.FOUND) + missing_count = sum(1 for fc in file_checks if fc.status == FnmFileStatus.MISSING) + + if missing_count == 0: + status = FnmPathStatus.VALID + message = f"All {found_count} expected files found." + elif found_count > 0: + status = FnmPathStatus.PARTIAL + message = f"{found_count} of {len(file_checks)} files found, {missing_count} missing." + else: + status = FnmPathStatus.PARTIAL + message = f"No expected files found in '{fnm_path}'." + + return FnmPathResult( + status=status, + fnm_path=fnm_path, + file_checks=file_checks, + message=message, + ) diff --git a/data/fnm/scripts/fnm_gating_cli.py b/data/fnm/scripts/fnm_gating_cli.py new file mode 100644 index 00000000..0b076e2b --- /dev/null +++ b/data/fnm/scripts/fnm_gating_cli.py @@ -0,0 +1,78 @@ +"""CLI entry point for validating FNM_PATH configuration. + +Provides a human-readable summary of the FNM_PATH environment variable status, +including per-file found/missing markers for developer onboarding and debugging. +""" + +from __future__ import annotations + +import argparse +import sys + +from fnm.scripts.fnm_gating import FnmFileStatus, FnmPathStatus, resolve_fnm_path + + +def cli_validate_fnm_path(args: list[str] | None = None) -> int: + """CLI entry point for validating FNM_PATH. + + Resolves FNM_PATH and prints a human-readable report of the validation result. + + Args: + args: Command-line arguments. If None, reads from sys.argv. + + Returns: + Exit code: 0 if VALID, 1 otherwise. + """ + parser = argparse.ArgumentParser( + prog="validate-fnm-path", + description="Validate the FNM_PATH environment variable and check for expected files.", + ) + parser.parse_args(args) + + result = resolve_fnm_path() + + print(f"FNM_PATH Status: {result.status.value}") + print() + + if result.status == FnmPathStatus.NOT_SET: + print("FNM_PATH environment variable is not set.") + print() + print("To configure:") + print(" export FNM_PATH=/path/to/fnm/data") + print() + print("See data/fnm/README.md for setup instructions.") + return 1 + + if result.status == FnmPathStatus.INVALID_PATH: + print(f"FNM_PATH is set to '{result.fnm_path}' but this is not a valid directory.") + return 1 + + if result.status == FnmPathStatus.MANIFEST_ERROR: + print(f"Error: {result.message}") + return 1 + + print(f"FNM_PATH: {result.fnm_path}") + print() + + if result.file_checks: + print("File checks:") + for fc in result.file_checks: + marker = "[FOUND]" if fc.status == FnmFileStatus.FOUND else "[MISSING]" + print(f" {marker} {fc.expected_name}") + print() + + found_count = len(result.found_files) + missing_count = len(result.missing_files) + total = len(result.file_checks) + print(f"Summary: {found_count}/{total} files found, {missing_count} missing.") + + if result.status == FnmPathStatus.VALID: + print("Status: All expected files are present.") + return 0 + + print("Status: Some files are missing. Verify your FNM data directory is complete.") + return 1 + + +if __name__ == "__main__": + sys.exit(cli_validate_fnm_path()) diff --git a/data/fnm/scripts/fnm_gating_fixtures.py b/data/fnm/scripts/fnm_gating_fixtures.py new file mode 100644 index 00000000..d886236a --- /dev/null +++ b/data/fnm/scripts/fnm_gating_fixtures.py @@ -0,0 +1,71 @@ +"""Pytest fixtures for FNM_PATH gating. + +Provides reusable fixtures that skip tests gracefully when FNM data is unavailable. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from fnm.scripts.fnm_gating import FnmFileStatus, FnmPathResult, resolve_fnm_path + + +@pytest.fixture +def require_fnm() -> FnmPathResult: + """Skip the test if FNM data is unavailable. Returns FnmPathResult when usable.""" + result = resolve_fnm_path() + if not result.is_usable: + pytest.skip(result.skip_reason) + return result + + +@pytest.fixture +def require_fnm_raw(require_fnm: FnmPathResult) -> Path: + """Return the absolute path to the PSS/E RAW file. Skip if not found. + + Depends on ``require_fnm`` to ensure FNM_PATH is validated first. + """ + for fc in require_fnm.file_checks: + if fc.expected_name.endswith(".raw") and fc.status == FnmFileStatus.FOUND: + assert fc.absolute_path is not None + return fc.absolute_path + pytest.skip("PSS/E RAW file not found in FNM_PATH.") + # unreachable, but satisfies type checker + raise AssertionError # pragma: no cover + + +@pytest.fixture +def require_fnm_csvs(require_fnm: FnmPathResult) -> dict[str, Path]: + """Return a dict of CSV filename -> absolute path for found CSVs. + + Only includes CSV files that were actually found on disk. + Depends on ``require_fnm`` to ensure FNM_PATH is validated first. + """ + csvs: dict[str, Path] = {} + for fc in require_fnm.file_checks: + if ( + fc.expected_name.endswith(".csv") + and fc.status == FnmFileStatus.FOUND + and fc.absolute_path is not None + ): + csvs[fc.expected_name] = fc.absolute_path + return csvs + + +def get_conftest_template() -> str: + """Return conftest.py content that imports FNM gating fixtures. + + Downstream test directories can use this as a template or copy it directly + into their ``conftest.py``. + """ + return '''\ +"""Conftest providing FNM gating fixtures for this test directory.""" + +from __future__ import annotations + +from fnm.scripts.fnm_gating_fixtures import require_fnm, require_fnm_csvs, require_fnm_raw + +__all__ = ["require_fnm", "require_fnm_csvs", "require_fnm_raw"] +''' diff --git a/data/fnm/scripts/generate_schema_reference.py b/data/fnm/scripts/generate_schema_reference.py new file mode 100644 index 00000000..245fb89e --- /dev/null +++ b/data/fnm/scripts/generate_schema_reference.py @@ -0,0 +1,1699 @@ +"""Generate the Intermediate Format Schema Reference markdown document. + +This script reads the Phase 1 D7 schema definitions programmatically and +produces data/fnm/docs/intermediate-schema.md with full semantic descriptions, +worked examples, and evaluate-tool guidance for every field. + +Usage (inside devcontainer): + cd /workspace/data + uv run python -m fnm.scripts.generate_schema_reference +""" + +from __future__ import annotations + +from pathlib import Path + +from fnm.scripts.intermediate_schema import ( + FieldSpec, + TableSchema, + get_table_schemas, +) + +# --------------------------------------------------------------------------- +# Semantic descriptions, evaluate-tool guidance, and worked examples +# --------------------------------------------------------------------------- +# These are hand-authored domain knowledge keyed by (record_type, field_name). +# The generator merges them with the machine-readable schema metadata. + +_UNIT_MAP: dict[str, str] = { + "": "\u2014", + "kV": "kV", + "pu": "pu", + "deg": "deg", + "MW": "MW", + "MVAR": "MVAR", + "MVA": "MVA", + "A": "A", + "ohm": "ohm", + "%": "%", +} + + +def _unit(f: FieldSpec) -> str: + return _UNIT_MAP.get(f.unit, f.unit if f.unit else "\u2014") + + +def _default_str(f: FieldSpec) -> str: + if f.default_value is None: + return "none" + if isinstance(f.default_value, str): + if f.default_value.strip() == "": + return '`"' + " " * len(f.default_value) + '"`' if f.default_value else '`""`' + return f'`"{f.default_value}"`' + return str(f.default_value) + + +def _range_str(f: FieldSpec) -> str: + if f.valid_range is None: + return "\u2014" + lo, hi = f.valid_range + lo_s = str(lo) if lo is not None else "\u2014" + hi_s = str(hi) if hi is not None else "\u2014" + return f"{lo_s}\u2013{hi_s}" + + +# --------------------------------------------------------------------------- +# Per-field semantic descriptions and guidance (hand-authored) +# --------------------------------------------------------------------------- + +# fmt: off +_SEMANTIC: dict[str, dict[str, tuple[str, str, str]]] = { + # (record_type, field_name) -> (semantic_desc, expected_range, guidance) + "Bus": { + "I": ( + "Unique bus number identifying this node in the network topology. " + "All branch, generator, load, and shunt records reference buses by this number.", + "10000\u201399999 for large networks", + "Verify I is a positive integer preserved exactly; loss of bus number destroys topology" + ), + "NAME": ( + "Alphanumeric bus name, up to 12 characters, padded with trailing spaces. " + "Used for human-readable identification in reports and diagrams.", + "\u2014", + "Verify NAME is preserved including trailing whitespace; " + "compare after stripping only if the tool normalizes whitespace" + ), + "BASKV": ( + "Bus base voltage in kV. Defines the voltage class for this bus and is the " + "reference for all per-unit voltage calculations at this bus. A value of 0.0 " + "is the PSS/E default but is physically meaningless for real network buses.", + "69\u2013500 for transmission", + "Verify BASKV > 0 for all buses with IDE != 4 (non-isolated); " + "verify preserved to at least 1 decimal place" + ), + "IDE": ( + "Bus type code: 1=PQ (load), 2=PV (generator), 3=swing (reference), " + "4=isolated (disconnected). Determines the power flow solution method at this bus.", + "1\u20134", + "Verify IDE is one of {1, 2, 3, 4} and matches the source exactly; " + "confirm bus type code maps to the tool's equivalent enum without silent coercion" + ), + "AREA": ( + "Area number to which this bus is assigned. Areas define interchange " + "control regions in the power flow solution.", + "1\u201350 for large ISOs", + "Verify AREA references a valid area number in the area table" + ), + "ZONE": ( + "Zone number for geographic or administrative grouping. Zones provide " + "finer-grained grouping than areas, used in reporting and load allocation.", + "1\u201350", + "Verify ZONE references a valid zone number in the zone table" + ), + "OWNER": ( + "Owner number identifying the entity that owns this bus. Used for " + "ownership tracking and cost allocation.", + "1\u2013200", + "Verify OWNER references a valid owner number in the owner table" + ), + "VM": ( + "Bus voltage magnitude in per-unit on the bus base voltage (BASKV). " + "In a solved case, this represents the steady-state voltage. " + "In an unsolved case, this is the initial voltage guess.", + "0.95\u20131.05 for solved case", + "Verify VM preserved to at least 4 decimal places; " + "values outside 0.9\u20131.1 in a solved case indicate convergence issues" + ), + "VA": ( + "Bus voltage angle in degrees. The swing bus angle is the reference " + "(typically 0.0). All other angles are relative to the swing bus.", + "-180\u2013180", + "Verify VA preserved to at least 2 decimal places; " + "swing bus (IDE=3) should have VA near 0.0" + ), + "NVHI": ( + "Normal operating voltage high limit in per-unit. Used by OPF and " + "monitoring functions to flag voltage violations.", + "1.05\u20131.10", + "If field is at PSS/E default (1.1), tool may omit \u2014 do not penalize" + ), + "NVLO": ( + "Normal operating voltage low limit in per-unit. Buses with voltage " + "below this limit are flagged as voltage violations.", + "0.90\u20130.95", + "If field is at PSS/E default (0.9), tool may omit \u2014 do not penalize" + ), + "EVHI": ( + "Emergency voltage high limit in per-unit. Applied during contingency " + "analysis to allow wider voltage tolerance under emergency conditions.", + "1.05\u20131.10", + "If field is at PSS/E default (1.1), tool may omit \u2014 do not penalize" + ), + "EVLO": ( + "Emergency voltage low limit in per-unit. More relaxed than normal " + "low limit, applied during contingency analysis.", + "0.85\u20130.95", + "If field is at PSS/E default (0.9), tool may omit \u2014 do not penalize" + ), + }, + "Load": { + "I": ( + "Bus number at which this load is connected. Multiple loads can exist " + "at the same bus, distinguished by ID.", + "10000\u201399999", + "Verify I references a valid bus number in the bus table" + ), + "ID": ( + "Two-character load identifier. Together with I, forms the composite " + "primary key. Default '1 ' (one followed by space).", + "\u2014", + "Verify ID is preserved as a 2-character string including trailing space" + ), + "STATUS": ( + "Load status: 1=in-service (included in power flow), " + "0=out-of-service (excluded from power flow solution).", + "0\u20131", + "Verify STATUS is one of {0, 1} and matches the source exactly" + ), + "AREA": ( + "Area number for this load, defaults to the bus's area. " + "Used for area interchange calculations.", + "1\u201350", + "Verify AREA is a positive integer; if at default (1), tool may inherit from bus" + ), + "ZONE": ( + "Zone number for this load, defaults to the bus's zone.", + "1\u201350", + "Verify ZONE is a positive integer; if at default (1), tool may inherit from bus" + ), + "PL": ( + "Constant-power active load in MW. The primary real power demand " + "at this bus. Positive values consume power.", + "0\u20135000", + "Verify PL preserved to at least 2 decimal places; " + "sign convention: positive = consumption" + ), + "QL": ( + "Constant-power reactive load in MVAR. Positive values consume " + "reactive power (lagging power factor).", + "-500\u20131000", + "Verify QL preserved to at least 2 decimal places; " + "sign convention: positive = lagging (inductive)" + ), + "IP": ( + "Constant-current active load component in MW at 1.0 pu voltage. " + "Scales linearly with voltage magnitude.", + "\u2014", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize" + ), + "IQ": ( + "Constant-current reactive load component in MVAR at 1.0 pu voltage.", + "\u2014", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize" + ), + "YP": ( + "Constant-admittance active load component in MW at 1.0 pu voltage. " + "Scales with voltage squared.", + "\u2014", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize" + ), + "YQ": ( + "Constant-admittance reactive load component in MVAR at 1.0 pu voltage.", + "\u2014", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize" + ), + "OWNER": ( + "Owner number for this load.", + "1\u2013200", + "If field is at PSS/E default (1), tool may omit \u2014 do not penalize" + ), + "SCALE": ( + "Load scaling flag: 1=load participates in scaling, 0=fixed load. " + "Controls whether the load is adjusted during area interchange scaling.", + "0\u20131", + "If field is at PSS/E default (1), tool may omit \u2014 do not penalize" + ), + }, + "Fixed Shunt": { + "I": ( + "Bus number at which this fixed shunt is connected.", + "10000\u201399999", + "Verify I references a valid bus number in the bus table" + ), + "ID": ( + "Two-character shunt identifier. Together with I, forms the composite " + "primary key.", + "\u2014", + "Verify ID is preserved as a 2-character string including trailing space" + ), + "STATUS": ( + "Shunt status: 1=in-service, 0=out-of-service.", + "0\u20131", + "Verify STATUS is one of {0, 1} and matches the source exactly" + ), + "GL": ( + "Active component of shunt admittance to ground in MW at 1.0 pu voltage. " + "Positive GL represents real power consumption (resistive losses).", + "\u2014", + "Verify GL preserved to at least 4 decimal places; " + "most fixed shunts have GL=0 (purely reactive)" + ), + "BL": ( + "Reactive component of shunt admittance to ground in MVAR at 1.0 pu voltage. " + "Positive BL is capacitive (generates reactive power), " + "negative BL is inductive (absorbs reactive power).", + "-500\u2013500", + "Verify BL is positive for capacitive shunts, negative for inductive; " + "verify preserved to at least 2 decimal places" + ), + }, + "Generator": { + "I": ( + "Bus number at which this generator is connected.", + "10000\u201399999", + "Verify I references a valid bus number in the bus table" + ), + "ID": ( + "Two-character machine identifier. Together with I, forms the composite " + "primary key. Allows multiple generators at the same bus.", + "\u2014", + "Verify ID is preserved as a 2-character string including trailing space" + ), + "PG": ( + "Active power output of the generator in MW. Positive values " + "indicate generation. Negative values indicate a synchronous condenser " + "consuming real power.", + "50\u20131000 for large units", + "Verify PG preserved to at least 2 decimal places" + ), + "QG": ( + "Reactive power output of the generator in MVAR. Determined by the " + "power flow solution within the QB\u2013QT limits.", + "-500\u2013500", + "Verify QG preserved to at least 2 decimal places" + ), + "QT": ( + "Maximum reactive power output in MVAR. Upper limit for the " + "generator's reactive capability curve.", + "0\u20131000", + "Verify QT preserved to at least 1 decimal place; " + "default 9999.0 indicates unconstrained" + ), + "QB": ( + "Minimum reactive power output in MVAR. Lower limit for the " + "generator's reactive capability.", + "-1000\u20130", + "Verify QB preserved to at least 1 decimal place; " + "default -9999.0 indicates unconstrained" + ), + "VS": ( + "Voltage setpoint for voltage-regulating generators in per-unit. " + "The generator adjusts reactive output to maintain this voltage at " + "the regulated bus (local or remote via IREG).", + "0.95\u20131.10", + "Verify VS preserved to at least 4 decimal places" + ), + "IREG": ( + "**[preservation-critical]** Remote regulated bus number. " + "0=local voltage regulation (at bus I). Non-zero=remote bus whose " + "voltage is controlled by this generator. Critical for correct " + "voltage regulation topology in power flow.", + "0 or valid bus number", + "MUST be preserved exactly; verify IREG=0 means local regulation, " + "not missing; loss of remote regulation topology is a fidelity finding" + ), + "MBASE": ( + "Machine MVA base for per-unit impedance conversion. Generator " + "impedances ZR, ZX are on this base.", + "50\u20131500", + "Verify MBASE preserved to at least 1 decimal place" + ), + "ZR": ( + "Machine resistance in per-unit on MBASE. Part of the generator's " + "internal impedance model for short-circuit studies.", + "\u2014", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize" + ), + "ZX": ( + "Machine reactance in per-unit on MBASE. Sub-transient or transient " + "reactance used in short-circuit calculations.", + "0.1\u20130.4", + "If field is at PSS/E default (1.0), tool may omit \u2014 do not penalize" + ), + "RT": ( + "Step-up transformer resistance in per-unit on MBASE.", + "\u2014", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize" + ), + "XT": ( + "Step-up transformer reactance in per-unit on MBASE.", + "\u2014", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize" + ), + "GTAP": ( + "Step-up transformer off-nominal turns ratio in per-unit on bus base kV.", + "0.9\u20131.1", + "If field is at PSS/E default (1.0), tool may omit \u2014 do not penalize" + ), + "STAT": ( + "Generator status: 1=in-service, 0=out-of-service.", + "0\u20131", + "Verify STAT is one of {0, 1} and matches the source exactly" + ), + "RMPCT": ( + "Percent of total MVAR range allocated to remote voltage regulation.", + "0\u2013100", + "If field is at PSS/E default (100.0), tool may omit \u2014 do not penalize" + ), + "PT": ( + "Maximum active power output in MW.", + "50\u20132000", + "Verify PT preserved to at least 1 decimal place; " + "default 9999.0 indicates unconstrained" + ), + "PB": ( + "Minimum active power output in MW.", + "-100\u20130", + "Verify PB preserved to at least 1 decimal place; " + "default -9999.0 indicates unconstrained" + ), + "O1": ( + "Owner number 1.", + "1\u2013200", + "If field is at PSS/E default (1), tool may omit \u2014 do not penalize" + ), + "F1": ( + "Fraction of generator owned by owner 1.", + "0.0\u20131.0", + "If field is at PSS/E default (1.0), tool may omit \u2014 do not penalize" + ), + "O2": ("Owner number 2.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + "F2": ("Fraction owned by owner 2.", "0.0\u20131.0", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize"), + "O3": ("Owner number 3.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + "F3": ("Fraction owned by owner 3.", "0.0\u20131.0", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize"), + "O4": ("Owner number 4.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + "F4": ("Fraction owned by owner 4.", "0.0\u20131.0", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize"), + "WMOD": ( + "Wind machine reactive power control mode. 0=standard, " + "1=constant power factor, 2=constant Q, 3=constant voltage.", + "0\u20133", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize" + ), + "WPF": ( + "Wind machine power factor for WMOD=1 mode.", + "0.8\u20131.0", + "If field is at PSS/E default (1.0), tool may omit \u2014 do not penalize" + ), + }, + "Branch": { + "I": ( + "From-bus number. Together with J and CKT, forms the branch's composite " + "primary key.", + "10000\u201399999", + "Verify I references a valid bus number in the bus table" + ), + "J": ( + "To-bus number. Branch connects bus I to bus J. The sign of J does not " + "matter for topology (absolute value is used).", + "10000\u201399999", + "Verify J references a valid bus number in the bus table" + ), + "CKT": ( + "Two-character circuit identifier allowing parallel branches between " + "the same bus pair.", + "\u2014", + "Verify CKT is preserved as a 2-character string including trailing space" + ), + "R": ( + "Branch resistance in per-unit on system MVA base (SBASE) and bus " + "base voltage. For transmission lines, R is typically much smaller " + "than X (R/X ratio < 0.5).", + "0.0001\u20130.1", + "Verify R preserved to at least 5 decimal places; " + "verify R < X for transmission lines (R/X < 1.0)" + ), + "X": ( + "Branch reactance in per-unit on system MVA base. The dominant " + "impedance component for transmission lines. X must be non-zero " + "for in-service branches.", + "0.001\u20130.5", + "Verify X is non-zero for all in-service branches (ST=1); " + "verify X preserved to at least 5 decimal places" + ), + "B": ( + "Total branch charging susceptance in per-unit on system MVA base. " + "For overhead lines, B is proportional to line length and voltage. " + "For short lines, B may be 0.", + "0.0\u20135.0", + "Verify B preserved to at least 5 decimal places; " + "B=0 is valid for short lines and cables" + ), + "RATEA": ( + "Normal thermal rating in MVA (Rating A). Used for continuous " + "loading monitoring.", + "0\u20133000", + "Verify RATEA preserved to at least 1 decimal place; " + "0.0 means no limit (not monitored)" + ), + "RATEB": ( + "Emergency thermal rating in MVA (Rating B). Short-term overload limit.", + "0\u20134000", + "Verify RATEB preserved to at least 1 decimal place; " + "0.0 means no limit" + ), + "RATEC": ( + "Long-term emergency rating in MVA (Rating C).", + "0\u20135000", + "Verify RATEC preserved to at least 1 decimal place; " + "0.0 means no limit" + ), + "GI": ("Line shunt conductance at from-bus end in per-unit.", "\u2014", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize"), + "BI": ("Line shunt susceptance at from-bus end in per-unit.", "\u2014", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize"), + "GJ": ("Line shunt conductance at to-bus end in per-unit.", "\u2014", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize"), + "BJ": ("Line shunt susceptance at to-bus end in per-unit.", "\u2014", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize"), + "ST": ( + "Branch status: 1=in-service, 0=out-of-service. " + "Out-of-service branches are excluded from the admittance matrix.", + "0\u20131", + "Verify ST is one of {0, 1} and matches the source exactly" + ), + "MET": ( + "Metered end flag: 1=from-bus (I), 2=to-bus (J). Determines " + "which end is used for loss allocation.", + "1\u20132", + "If field is at PSS/E default (1), tool may omit \u2014 do not penalize" + ), + "LEN": ( + "Line length in user-selected units. Informational field, not " + "used in power flow calculations.", + "\u2014", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize" + ), + "O1": ("Owner number 1.", "1\u2013200", + "If field is at PSS/E default (1), tool may omit \u2014 do not penalize"), + "F1": ("Fraction owned by owner 1.", "0.0\u20131.0", + "If field is at PSS/E default (1.0), tool may omit \u2014 do not penalize"), + "O2": ("Owner number 2.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + "F2": ("Fraction owned by owner 2.", "0.0\u20131.0", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize"), + "O3": ("Owner number 3.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + "F3": ("Fraction owned by owner 3.", "0.0\u20131.0", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize"), + "O4": ("Owner number 4.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + "F4": ("Fraction owned by owner 4.", "0.0\u20131.0", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize"), + }, + "Transformer": { + "I": ("Winding 1 (primary) bus number.", "10000\u201399999", + "Verify I references a valid bus number in the bus table"), + "J": ("Winding 2 (secondary) bus number.", "10000\u201399999", + "Verify J references a valid bus number in the bus table"), + "K": ( + "**[preservation-critical]** Winding 3 bus number. K=0 indicates a 2-winding " + "transformer; K!=0 indicates a 3-winding transformer. This field determines " + "the topology interpretation for all subsequent winding data.", + "0 or valid bus number", + "MUST be preserved exactly; K=0 vs K!=0 changes transformer topology " + "interpretation entirely; loss is a critical fidelity finding" + ), + "CKT": ("Circuit identifier for parallel transformers.", "\u2014", + "Verify CKT is preserved as a 2-character string including trailing space"), + "CW": ( + "**[preservation-critical]** Winding data I/O code controlling how WINDV1/2/3 " + "are interpreted: 1=turns ratio in pu on bus base kV, 2=voltage in kV, " + "3=turns ratio in pu on nominal kV.", + "1\u20133", + "MUST be preserved exactly; CW determines the interpretation of all " + "winding voltage/turns-ratio fields; loss corrupts impedance calculations" + ), + "CZ": ( + "**[preservation-critical]** Impedance data I/O code: 1=pu on system base, " + "2=pu on winding MVA/kV base, 3=ohms/kV load loss.", + "1\u20133", + "MUST be preserved exactly; CZ determines per-unit base for R and X fields" + ), + "CM": ( + "**[preservation-critical]** Magnetizing admittance I/O code: " + "1=pu on system base, 2=no-load loss/exciting current.", + "1\u20132", + "MUST be preserved exactly; CM determines interpretation of MAG1/MAG2" + ), + "MAG1": ("Magnetizing conductance or no-load loss, depending on CM.", "\u2014", + "Verify MAG1 preserved to at least 5 decimal places"), + "MAG2": ("Magnetizing susceptance or exciting current, depending on CM.", "\u2014", + "Verify MAG2 preserved to at least 5 decimal places"), + "NMETR": ("Non-metered end code.", "\u2014", + "If field is at PSS/E default (2), tool may omit \u2014 do not penalize"), + "NAME": ("Transformer name, up to 12 characters.", "\u2014", + "Verify NAME is preserved including trailing whitespace"), + "STAT": ( + "Transformer status: 0=out-of-service, 1=in-service, 2=winding 2 out, " + "3=winding 3 out, 4=winding 2 and 3 out.", + "0\u20134", + "Verify STAT is one of {0, 1, 2, 3, 4} and matches the source exactly" + ), + "O1": ("Owner number 1.", "1\u2013200", + "If field is at PSS/E default (1), tool may omit \u2014 do not penalize"), + "F1": ("Fraction by owner 1.", "0.0\u20131.0", + "If field is at PSS/E default (1.0), tool may omit \u2014 do not penalize"), + "O2": ("Owner 2.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + "F2": ("Fraction by owner 2.", "0.0\u20131.0", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize"), + "O3": ("Owner 3.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + "F3": ("Fraction by owner 3.", "0.0\u20131.0", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize"), + "O4": ("Owner 4.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + "F4": ("Fraction by owner 4.", "0.0\u20131.0", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize"), + "VECGRP": ("Vector group designation (12 chars).", "\u2014", + "If field is at PSS/E default (blank), tool may omit \u2014 do not penalize"), + "R1_2": ( + "Resistance of winding 1\u20132 pair, interpretation depends on CZ.", + "0.0\u20130.1", + "Verify R1_2 preserved to at least 5 decimal places", + ), + "X1_2": ( + "Reactance of winding 1\u20132 pair, interpretation depends on CZ.", + "0.01\u20130.5", + "Verify X1_2 is non-zero for in-service transformers; " + "verify preserved to at least 5 decimal places", + ), + "SBASE1_2": ("MVA base for winding 1\u20132 impedance.", "50\u20132000", + "Verify SBASE1_2 preserved to at least 1 decimal place"), + "R2_3": ("Resistance of winding 2\u20133 pair (3W only).", "\u2014", + "Verify non-null when K != 0; if K=0, tool may omit"), + "X2_3": ("Reactance of winding 2\u20133 pair (3W only).", "\u2014", + "Verify non-null when K != 0; if K=0, tool may omit"), + "SBASE2_3": ("MVA base for winding 2\u20133 (3W only).", "\u2014", + "Verify non-null when K != 0; if K=0, tool may omit"), + "R3_1": ("Resistance of winding 3\u20131 pair (3W only).", "\u2014", + "Verify non-null when K != 0; if K=0, tool may omit"), + "X3_1": ("Reactance of winding 3\u20131 pair (3W only).", "\u2014", + "Verify non-null when K != 0; if K=0, tool may omit"), + "SBASE3_1": ("MVA base for winding 3\u20131 (3W only).", "\u2014", + "Verify non-null when K != 0; if K=0, tool may omit"), + "VMSTAR": ("Star-point bus voltage magnitude for 3W transformers.", "\u2014", + "If field is at PSS/E default (1.0), tool may omit \u2014 do not penalize"), + "ANSTAR": ("Star-point bus voltage angle for 3W transformers.", "\u2014", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize"), + "WINDV1": ( + "**[preservation-critical]** Winding 1 off-nominal turns ratio or voltage. " + "Interpretation depends on CW code.", + "0.9\u20131.1 (pu) or kV", + "MUST be preserved exactly to at least 5 decimal places; " + "loss corrupts transformer model" + ), + "NOMV1": ( + "**[preservation-critical]** Winding 1 nominal voltage in kV. " + "Used with CW=3 for turns ratio calculation. Must correspond to a " + "standard transmission voltage class.", + "69\u2013500", + "MUST be preserved exactly; verify drawn from standard kV classes" + ), + "ANG1": ( + "**[preservation-critical]** Winding 1 phase shift angle in degrees. " + "Non-zero for phase-shifting transformers.", + "-180\u2013180", + "MUST be preserved exactly to at least 2 decimal places; " + "non-zero ANG1 indicates phase-shifting transformer" + ), + "RATA1": ( + "**[preservation-critical]** Winding 1 normal rating in MVA.", + "50\u20132000", + "MUST be preserved exactly to at least 1 decimal place" + ), + "RATB1": ("Winding 1 emergency rating in MVA.", "50\u20133000", + "Verify RATB1 preserved to at least 1 decimal place"), + "RATC1": ("Winding 1 long-term emergency rating in MVA.", "50\u20134000", + "Verify RATC1 preserved to at least 1 decimal place"), + "COD1": ("Winding 1 tap control mode code.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + "CONT1": ("Winding 1 controlled bus number.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + "RMA1": ("Winding 1 upper tap or voltage limit.", "0.9\u20131.1", + "Verify RMA1 preserved to at least 4 decimal places"), + "RMI1": ("Winding 1 lower tap or voltage limit.", "0.9\u20131.1", + "Verify RMI1 preserved to at least 4 decimal places"), + "VMA1": ("Winding 1 upper voltage limit for control.", "1.0\u20131.1", + "Verify VMA1 preserved to at least 4 decimal places"), + "VMI1": ("Winding 1 lower voltage limit for control.", "0.9\u20131.0", + "Verify VMI1 preserved to at least 4 decimal places"), + "NTP1": ("Number of tap positions for winding 1.", "11\u201399", + "If field is at PSS/E default (33), tool may omit \u2014 do not penalize"), + "TAB1": ("Impedance correction table number for winding 1.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + "CR1": ("Load drop compensation resistance for winding 1.", "\u2014", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize"), + "CX1": ("Load drop compensation reactance for winding 1.", "\u2014", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize"), + "CNXA1": ("Connection angle for winding 1.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + "WINDV2": ( + "**[preservation-critical]** Winding 2 off-nominal turns ratio or voltage.", + "0.9\u20131.1 (pu) or kV", + "MUST be preserved exactly to at least 5 decimal places" + ), + "NOMV2": ( + "**[preservation-critical]** Winding 2 nominal voltage in kV.", + "69\u2013500", + "MUST be preserved exactly; verify drawn from standard kV classes" + ), + "ANG2": ("Winding 2 phase shift angle in degrees.", "-180\u2013180", + "Verify ANG2 preserved to at least 2 decimal places"), + "RATA2": ( + "**[preservation-critical]** Winding 2 normal rating in MVA.", + "50\u20132000", + "MUST be preserved exactly to at least 1 decimal place" + ), + "RATB2": ("Winding 2 emergency rating in MVA.", "\u2014", + "Verify RATB2 preserved to at least 1 decimal place"), + "RATC2": ("Winding 2 long-term emergency rating.", "\u2014", + "Verify RATC2 preserved to at least 1 decimal place"), + "COD2": ("Winding 2 tap control mode code.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + "CONT2": ("Winding 2 controlled bus number.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + "RMA2": ("Winding 2 upper tap or voltage limit.", "0.9\u20131.1", + "Verify RMA2 preserved to at least 4 decimal places"), + "RMI2": ("Winding 2 lower tap or voltage limit.", "0.9\u20131.1", + "Verify RMI2 preserved to at least 4 decimal places"), + "VMA2": ("Winding 2 upper voltage limit for control.", "1.0\u20131.1", + "Verify VMA2 preserved to at least 4 decimal places"), + "VMI2": ("Winding 2 lower voltage limit for control.", "0.9\u20131.0", + "Verify VMI2 preserved to at least 4 decimal places"), + "NTP2": ("Number of tap positions for winding 2.", "11\u201399", + "If field is at PSS/E default (33), tool may omit \u2014 do not penalize"), + "TAB2": ("Impedance correction table for winding 2.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + "CR2": ("Load drop compensation resistance for winding 2.", "\u2014", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize"), + "CX2": ("Load drop compensation reactance for winding 2.", "\u2014", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize"), + "CNXA2": ("Connection angle for winding 2.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + "WINDV3": ( + "**[preservation-critical]** Winding 3 off-nominal turns ratio or voltage. " + "Null/default for 2-winding transformers (K=0).", + "0.9\u20131.1 (pu) or kV", + "MUST be preserved exactly to at least 5 decimal places when K != 0; " + "verify winding 3 fields are non-null when K != 0" + ), + "NOMV3": ( + "**[preservation-critical]** Winding 3 nominal voltage in kV. " + "Null/default for 2W transformers.", + "69\u2013500", + "MUST be preserved exactly when K != 0; verify drawn from standard kV classes" + ), + "ANG3": ("Winding 3 phase shift angle in degrees.", "-180\u2013180", + "Verify ANG3 preserved to at least 2 decimal places when K != 0"), + "RATA3": ( + "**[preservation-critical]** Winding 3 normal rating in MVA.", + "50\u20132000", + "MUST be preserved exactly to at least 1 decimal place when K != 0" + ), + "RATB3": ("Winding 3 emergency rating.", "\u2014", + "If K=0, tool may omit \u2014 do not penalize"), + "RATC3": ("Winding 3 long-term emergency rating.", "\u2014", + "If K=0, tool may omit \u2014 do not penalize"), + "COD3": ("Winding 3 tap control mode.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + "CONT3": ("Winding 3 controlled bus.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + "RMA3": ("Winding 3 upper tap/voltage limit.", "\u2014", + "Verify RMA3 preserved to at least 4 decimal places when K != 0"), + "RMI3": ("Winding 3 lower tap/voltage limit.", "\u2014", + "Verify RMI3 preserved to at least 4 decimal places when K != 0"), + "VMA3": ("Winding 3 upper voltage limit for control.", "\u2014", + "Verify VMA3 preserved to at least 4 decimal places when K != 0"), + "VMI3": ("Winding 3 lower voltage limit for control.", "\u2014", + "Verify VMI3 preserved to at least 4 decimal places when K != 0"), + "NTP3": ("Number of tap positions for winding 3.", "\u2014", + "If field is at PSS/E default (33), tool may omit \u2014 do not penalize"), + "TAB3": ("Impedance correction table for winding 3.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + "CR3": ("Load drop compensation resistance for winding 3.", "\u2014", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize"), + "CX3": ("Load drop compensation reactance for winding 3.", "\u2014", + "If field is at PSS/E default (0.0), tool may omit \u2014 do not penalize"), + "CNXA3": ("Connection angle for winding 3.", "\u2014", + "If field is at PSS/E default (0), tool may omit \u2014 do not penalize"), + }, + "Area": { + "I": ("Unique area number identifying this interchange control area.", "1\u201350", + "Verify I is a positive integer preserved exactly"), + "ISW": ( + "**[preservation-critical]** Area slack bus number. The swing bus " + "that absorbs area interchange mismatch. 0=no area slack bus specified.", + "0 or valid bus number", + "MUST be preserved exactly; loss of area slack assignment corrupts " + "area interchange control" + ), + "PDES": ( + "**[preservation-critical]** Desired net area interchange in MW. " + "Positive=export, negative=import.", + "-5000\u20135000", + "MUST be preserved exactly to at least 2 decimal places" + ), + "PTOL": ( + "**[preservation-critical]** Area interchange tolerance in MW. " + "Convergence criterion for area interchange control.", + "1.0\u201350.0", + "MUST be preserved exactly to at least 1 decimal place" + ), + "ARNAME": ( + "Area name, up to 12 characters.", + "\u2014", + "Verify ARNAME is preserved including trailing whitespace" + ), + }, +} + +# For record types not in _SEMANTIC, generate basic descriptions from FieldSpec +def _auto_semantic(rt: str, f: FieldSpec) -> tuple[str, str, str]: + """Generate a reasonable semantic tuple from FieldSpec metadata.""" + prefix = "**[preservation-critical]** " if f.preservation_critical else "" + desc = f"{prefix}{f.description}." + exp_range = _range_str(f) if f.valid_range else "\u2014" + if f.default_value is not None and not f.preservation_critical: + guidance = ( + f"If field is at PSS/E default ({f.default_value}), " + "tool may omit \u2014 do not penalize" + ) + elif f.preservation_critical: + guidance = "MUST be preserved exactly; loss of this field is a fidelity finding" + elif f.data_type == "integer": + guidance = f"Verify {f.name} is a valid integer and matches the source" + elif f.data_type == "number": + guidance = f"Verify {f.name} preserved to at least 4 decimal places" + else: + guidance = f"Verify {f.name} is preserved as a string including any trailing whitespace" + return (desc, exp_range, guidance) +# fmt: on + + +def _get_semantic(rt: str, f: FieldSpec) -> tuple[str, str, str]: + """Look up hand-authored semantic, fall back to auto-generated.""" + rt_dict = _SEMANTIC.get(rt, {}) + if f.name in rt_dict: + desc, exp_range, guidance = rt_dict[f.name] + # Ensure preservation-critical prefix + if f.preservation_critical and "**[preservation-critical]**" not in desc: + desc = "**[preservation-critical]** " + desc + return (desc, exp_range, guidance) + return _auto_semantic(rt, f) + + +# --------------------------------------------------------------------------- +# Worked examples (hand-authored, synthetic, NDA-safe) +# --------------------------------------------------------------------------- + +_WORKED_EXAMPLES: dict[str, str] = { + "Bus": """\ +``` +I: 30100 +NAME: "MESA 230 " +BASKV: 230.0 +IDE: 1 +AREA: 5 +ZONE: 12 +OWNER: 3 +VM: 1.0142 +VA: -8.35 +NVHI: 1.1 +NVLO: 0.9 +EVHI: 1.1 +EVLO: 0.9 +```""", + "Load": """\ +``` +I: 30100 +ID: "1 " +STATUS: 1 +AREA: 5 +ZONE: 12 +PL: 245.80 +QL: 85.30 +IP: 0.0 +IQ: 0.0 +YP: 0.0 +YQ: 0.0 +OWNER: 3 +SCALE: 1 +```""", + "Fixed Shunt": """\ +``` +I: 42500 +ID: "1 " +STATUS: 1 +GL: 0.0 +BL: 150.0 +```""", + "Generator": """\ +``` +I: 50200 +ID: "1 " +PG: 350.00 +QG: 45.20 +QT: 200.0 +QB: -100.0 +VS: 1.0250 +IREG: 0 +MBASE: 400.0 +ZR: 0.0 +ZX: 1.0 +RT: 0.0 +XT: 0.0 +GTAP: 1.0 +STAT: 1 +RMPCT: 100.0 +PT: 400.0 +PB: 100.0 +O1: 5 +F1: 1.0 +O2: 0 +F2: 0.0 +O3: 0 +F3: 0.0 +O4: 0 +F4: 0.0 +WMOD: 0 +WPF: 1.0 +```""", + "Branch": """\ +``` +I: 30100 +J: 30200 +CKT: "1 " +R: 0.00320 +X: 0.03150 +B: 0.52800 +RATEA: 600.0 +RATEB: 720.0 +RATEC: 800.0 +GI: 0.0 +BI: 0.0 +GJ: 0.0 +BJ: 0.0 +ST: 1 +MET: 1 +LEN: 0.0 +O1: 3 +F1: 1.0 +O2: 0 +F2: 0.0 +O3: 0 +F3: 0.0 +O4: 0 +F4: 0.0 +```""", + "Transformer": """\ +``` +I: 30100 +J: 30500 +K: 0 +CKT: "1 " +CW: 1 +CZ: 1 +CM: 1 +MAG1: 0.0 +MAG2: 0.0 +NMETR: 2 +NAME: "XF-230/69 " +STAT: 1 +O1: 3 +F1: 1.0 +O2: 0 +F2: 0.0 +O3: 0 +F3: 0.0 +O4: 0 +F4: 0.0 +VECGRP: " " +R1_2: 0.00250 +X1_2: 0.12500 +SBASE1_2: 200.0 +R2_3: 0.0 +X2_3: 0.0 +SBASE2_3: 100.0 +R3_1: 0.0 +X3_1: 0.0 +SBASE3_1: 100.0 +VMSTAR: 1.0 +ANSTAR: 0.0 +WINDV1: 1.0125 +NOMV1: 230.0 +ANG1: 0.0 +RATA1: 200.0 +RATB1: 240.0 +RATC1: 280.0 +COD1: 0 +CONT1: 0 +RMA1: 1.1 +RMI1: 0.9 +VMA1: 1.1 +VMI1: 0.9 +NTP1: 33 +TAB1: 0 +CR1: 0.0 +CX1: 0.0 +CNXA1: 0 +WINDV2: 1.0 +NOMV2: 69.0 +ANG2: 0.0 +RATA2: 200.0 +RATB2: 0.0 +RATC2: 0.0 +COD2: 0 +CONT2: 0 +RMA2: 1.1 +RMI2: 0.9 +VMA2: 1.1 +VMI2: 0.9 +NTP2: 33 +TAB2: 0 +CR2: 0.0 +CX2: 0.0 +CNXA2: 0 +WINDV3: 1.0 +NOMV3: 0.0 +ANG3: 0.0 +RATA3: 0.0 +RATB3: 0.0 +RATC3: 0.0 +COD3: 0 +CONT3: 0 +RMA3: 1.1 +RMI3: 0.9 +VMA3: 1.1 +VMI3: 0.9 +NTP3: 33 +TAB3: 0 +CR3: 0.0 +CX3: 0.0 +CNXA3: 0 +```""", + "Area": """\ +``` +I: 5 +ISW: 50200 +PDES: 150.0 +PTOL: 10.0 +ARNAME: "SOUTH ZONE " +```""", + "Two-Terminal DC": """\ +``` +NAME: "PDCI_NORTH " +MDC: 1 +RDC: 12.5 +SETVL: 1600.0 +VSCHD: 500.0 +VCMOD: 0.0 +RCOMP: 0.0 +DELTI: 0.0 +METER: "I" +DCVMIN: 0.0 +CCCITMX: 20 +CCCACC: 1.0 +IPR: 60100 +NBR: 2 +ANMXR: 0.0 +ANMNR: 0.0 +RCR: 0.0 +XCR: 0.0 +EBASR: 0.0 +TRR: 1.0 +TAPR: 1.0 +TMXR: 1.5 +TMNR: 0.51 +STPR: 0.00625 +ICR: 0 +IFR: 0 +ITR: 0 +IDR: "1 " +XCAPR: 0.0 +IPI: 60200 +NBI: 2 +ANMXI: 0.0 +ANMNI: 0.0 +RCI: 0.0 +XCI: 0.0 +EBASI: 0.0 +TRI: 1.0 +TAPI: 1.0 +TMXI: 1.5 +TMNI: 0.51 +STPI: 0.00625 +ICI: 0 +IFI: 0 +ITI: 0 +IDI: "1 " +XCAPI: 0.0 +```""", + "VSC DC": """\ +``` +NAME: "VSC_LINK_1 " +MDC: 1 +RDC: 5.0 +O1: 1 +F1: 1.0 +O2: 0 +F2: 0.0 +O3: 0 +F3: 0.0 +O4: 0 +F4: 0.0 +IBUS1: 70100 +TYPE1: 1 +MODE1: 1 +DCSET1: 400.0 +ACSET1: 1.0 +ALOSS1: 0.0 +BLOSS1: 0.0 +MINLOSS1: 0.0 +SMAX1: 500.0 +IMAX1: 0.0 +PWF1: 1.0 +MAXQ1: 200.0 +MINQ1: -200.0 +REMOT1: 0 +RMPCT1: 100.0 +IBUS2: 70200 +TYPE2: 1 +MODE2: 1 +DCSET2: 0.0 +ACSET2: 1.0 +ALOSS2: 0.0 +BLOSS2: 0.0 +MINLOSS2: 0.0 +SMAX2: 500.0 +IMAX2: 0.0 +PWF2: 1.0 +MAXQ2: 200.0 +MINQ2: -200.0 +REMOT2: 0 +RMPCT2: 100.0 +```""", + "Impedance Correction": """\ +``` +T: 1 +T1: 0.9 +F1: 0.95 +T2: 0.95 +F2: 0.98 +T3: 1.0 +F3: 1.0 +T4: 1.05 +F4: 0.98 +T5: 1.1 +F5: 0.95 +T6: 0.0 +F6: 0.0 +T7: 0.0 +F7: 0.0 +T8: 0.0 +F8: 0.0 +T9: 0.0 +F9: 0.0 +T10: 0.0 +F10: 0.0 +T11: 0.0 +F11: 0.0 +```""", + "Multi-Terminal DC": """\ +``` +NAME: "MTDC_SYS_1 " +NCONV: 3 +NDCBS: 4 +NDCLN: 3 +MDC: 1 +VCONV: 1 +VCMOD: 0.0 +VCONVN: 0 +```""", + "Multi-Section Line": """\ +``` +I: 30100 +J: 30400 +ID: "1 " +MET: 1 +DUM1: 30150 +DUM2: 30200 +DUM3: 30250 +DUM4: 0 +DUM5: 0 +DUM6: 0 +DUM7: 0 +DUM8: 0 +DUM9: 0 +```""", + "Zone": """\ +``` +I: 12 +ZONAME: "SOUTH BAY " +```""", + "Interarea Transfer": """\ +``` +ARFROM: 5 +ARTO: 8 +TRID: "1 " +PTRAN: 200.0 +```""", + "Owner": """\ +``` +I: 3 +OWNAME: "SOCAL EDISON" +```""", + "FACTS": """\ +``` +NAME: "SVC_MESA " +I: 30100 +J: 0 +MODE: 1 +SET1: 1.0 +SET2: 0.0 +VSREF: 1.0 +REMOT: 0 +MESSION: 0.0 +LINX: 0.05 +RMPCT: 100.0 +OWNER: 3 +SET3: 0.0 +SET4: 0.0 +```""", + "Switched Shunt": """\ +``` +I: 42500 +MODSW: 1 +ADJM: 0 +STAT: 1 +VSWHI: 1.05 +VSWLO: 0.95 +SWREM: 0 +RMPCT: 100.0 +RMIDNT: "" +BINIT: 50.0 +N1: 2 +B1: 25.0 +N2: 3 +B2: 50.0 +N3: 0 +B3: 0.0 +N4: 0 +B4: 0.0 +N5: 0 +B5: 0.0 +N6: 0 +B6: 0.0 +N7: 0 +B7: 0.0 +N8: 0 +B8: 0.0 +```""", +} + +# --------------------------------------------------------------------------- +# Nullable/default behavior (hand-authored per record type) +# --------------------------------------------------------------------------- + +_NULLABLE_BEHAVIOR: dict[str, str] = { + "Bus": ( + "All bus fields are required in PSS/E v31 and have well-defined defaults. " + "BASKV=0.0 is the PSS/E default but indicates an uninitialized bus; real network " + "buses always have BASKV > 0. VM defaults to 1.0 (flat start), and VA defaults " + "to 0.0 degrees. The voltage limit fields (NVHI, NVLO, EVHI, EVLO) default to " + "standard PSS/E values and may be omitted by tools without penalty. " + "The canonical parser writes all fields including those at default values." + ), + "Load": ( + "The constant-current (IP, IQ) and constant-admittance (YP, YQ) load components " + "default to 0.0, meaning the load is modeled as constant-power only. A zero value " + "is semantically meaningful (not missing) -- it means that load component is absent. " + "OWNER defaults to 1, and SCALE defaults to 1 (load participates in scaling)." + ), + "Fixed Shunt": ( + "GL defaults to 0.0, meaning no active power loss in the shunt (purely reactive). " + "BL defaults to 0.0 but is typically non-zero for any meaningful shunt device. " + "STATUS defaults to 1 (in-service)." + ), + "Generator": ( + "QT=9999.0 and QB=-9999.0 indicate unconstrained reactive capability (PSS/E defaults). " + "IREG=0 means local voltage regulation (at bus I), not 'no regulation'. This distinction " + "is critical: zero is a meaningful value, not a null. ZR, ZX, RT, XT, GTAP are machine " + "impedance parameters that default to their PSS/E values; tools commonly omit these for " + "steady-state power flow. Owner fields O2-O4 default to 0, meaning single ownership." + ), + "Branch": ( + "R and X have no default -- they must be provided for every branch. B defaults to 0.0 " + "(no line charging), which is valid for short lines. Rating fields (RATEA, RATEB, RATEC) " + "default to 0.0, meaning no thermal limit is enforced. GI, BI, GJ, BJ are line shunt " + "elements that default to 0.0 (no shunt admittance at line ends). Owner fields O2-O4 " + "default to 0." + ), + "Transformer": ( + "K=0 indicates a 2-winding transformer; all winding-3 fields revert to defaults. " + "The CW, CZ, CM codes default to 1 but are preservation-critical because they control " + "how all impedance and turns-ratio fields are interpreted. WINDV1/WINDV2 default to 1.0 " + "(unity turns ratio). NOMV1/NOMV2 default to 0.0, meaning the bus base kV is used. " + "VMSTAR and ANSTAR are meaningful only for 3W transformers." + ), + "Area": ( + "ISW=0 means no area slack bus is designated. PDES=0.0 means no net interchange " + "target. PTOL=10.0 is the default interchange tolerance. All fields are required." + ), + "Two-Terminal DC": ( + "Many fields have PSS/E defaults that represent 'not specified' or 'not applicable'. " + "RDC has no default and must always be present. SETVL and VSCHD are operationally " + "significant and should be preserved. Converter tap limits (TMXR, TMNR, etc.) have " + "standard defaults." + ), + "VSC DC": ( + "Owner fields O2-O4 default to 0 (single ownership). Converter loss coefficients " + "(ALOSS, BLOSS, MINLOSS) default to 0.0. SMAX and IMAX default to 0.0 meaning " + "no limit. Q limits default to +/-9999.0." + ), + "Impedance Correction": ( + "T1-T11 and F1-F11 pairs define piecewise-linear correction curves. Unused pairs " + "default to 0.0. The table is terminated by the first T value of 0.0." + ), + "Multi-Terminal DC": ( + "NCONV, NDCBS, NDCLN define the structure of the multi-terminal DC system. " + "These must be non-zero for a valid record. MDC defaults to 0. " + "VCONV, VCMOD, VCONVN are control parameters with standard defaults." + ), + "Multi-Section Line": ( + "DUM1-DUM9 are intermediate bus numbers defining the multi-section line topology. " + "DUM values of 0 indicate unused slots. At least DUM1 must be non-zero for a " + "valid multi-section line grouping." + ), + "Zone": ( + "Both I and ZONAME are required. ZONAME defaults to blank (12 spaces). " + "No fields are nullable." + ), + "Interarea Transfer": ( + "All fields are required. PTRAN defaults to 0.0 (no scheduled transfer). " + "TRID defaults to '1 '." + ), + "Owner": ( + "Both I and OWNAME are required. OWNAME defaults to blank (12 spaces). " + "No fields are nullable." + ), + "FACTS": ( + "J=0 indicates a shunt FACTS device (no terminal bus). MODE defaults to 1. " + "SET1-SET4 are control setpoints with mode-dependent interpretations; they default " + "to 0.0. VSREF defaults to 1.0 pu. LINX defaults to 0.05 pu." + ), + "Switched Shunt": ( + "N1-N8 and B1-B8 define discrete switching blocks. Unused blocks have N=0 and B=0.0. " + "BINIT is the initial susceptance and should match the sum of switched-in blocks. " + "MODSW defaults to 1 (discrete mode). SWREM=0 means local voltage control. " + "RMIDNT is an optional name field that may be empty." + ), +} + +# --------------------------------------------------------------------------- +# Approximate record counts (NDA-safe order-of-magnitude) +# --------------------------------------------------------------------------- + +_APPROX_COUNTS: dict[str, str] = { + "Bus": "30000", + "Load": "15000", + "Fixed Shunt": "~500", + "Generator": "~5,000", + "Branch": "~35,000", + "Transformer": "~8,000", + "Area": "~30", + "Two-Terminal DC": "~5", + "VSC DC": "~2", + "Impedance Correction": "~200", + "Multi-Terminal DC": "~1", + "Multi-Section Line": "~800", + "Zone": "~40", + "Interarea Transfer": "~50", + "Owner": "~100", + "FACTS": "~50", + "Switched Shunt": "~3,000", +} + +# --------------------------------------------------------------------------- +# Purpose statements (richer than _TABLE_DESCRIPTIONS) +# --------------------------------------------------------------------------- + +_PURPOSES: dict[str, str] = { + "Bus": ( + "Defines every node (bus) in the transmission network. Each bus has a unique number, " + "base voltage, type code (PQ/PV/swing/isolated), and solved-state voltage. All other " + "record types reference buses by number. The bus table is the topological foundation " + "of the network model." + ), + "Load": ( + "Represents electrical demand at each bus. Loads are modeled with constant-power (PL, QL), " + "constant-current (IP, IQ), and constant-admittance (YP, YQ) components. Multiple loads " + "can exist at one bus, distinguished by their two-character ID." + ), + "Fixed Shunt": ( + "Represents fixed (non-switchable) shunt compensation devices. Fixed shunts provide " + "reactive power support (capacitive) or absorption (inductive) at a constant value " + "regardless of voltage. Distinguished from switched shunts which have discrete steps." + ), + "Generator": ( + "Represents all generating units including conventional thermal, hydro, wind, and solar " + "plants. Each generator has active/reactive output, capability limits, voltage setpoint, " + "and machine impedance data. Multiple generators at one bus use different IDs." + ), + "Branch": ( + "Represents transmission lines, cables, and series elements connecting two buses. " + "Each branch has impedance (R, X, B), thermal ratings, and status. Parallel branches " + "between the same bus pair are distinguished by circuit identifier CKT." + ), + "Transformer": ( + "Represents 2-winding and 3-winding power transformers. The PSS/E RAW format uses " + "a multi-line record (up to 5 lines) that is flattened into a single row in the " + "intermediate format. The CW/CZ/CM codes control how impedance and turns-ratio " + "data are interpreted -- these must be preserved exactly." + ), + "Area": ( + "Defines interchange control areas for the power flow solution. Each area has a " + "slack bus, desired net interchange (export/import), and tolerance. Areas are the " + "primary aggregation unit for balancing supply and demand." + ), + "Two-Terminal DC": ( + "Represents conventional line-commutated converter (LCC) HVDC links with a rectifier " + "and inverter terminal. Each record contains DC line parameters plus full converter " + "transformer and control data for both ends." + ), + "VSC DC": ( + "Represents voltage-source converter (VSC) HVDC links. More modern than LCC technology, " + "with independent P and Q control at each converter. Each record contains DC line " + "parameters and two converter specifications." + ), + "Impedance Correction": ( + "Defines piecewise-linear impedance correction tables referenced by transformers " + "(via TAB1/TAB2/TAB3). Each table maps tap ratio or phase angle to a correction " + "factor applied to the transformer impedance." + ), + "Multi-Terminal DC": ( + "Header record for multi-terminal HVDC systems with more than two converters. " + "Defines the number of converters, DC buses, and DC links in the system. " + "Detailed converter/bus/link data follows in the PSS/E RAW file." + ), + "Multi-Section Line": ( + "Groups multiple branch records into a single multi-section transmission line. " + "DUM1-DUM9 define intermediate bus numbers along the line. All sections share " + "the same from-bus (I), to-bus (J), and line identifier (ID)." + ), + "Zone": ( + "Defines geographic or administrative zones for reporting and load allocation. " + "Zones provide finer-grained grouping than areas. Each bus is assigned to exactly one zone." + ), + "Interarea Transfer": ( + "Defines scheduled power transfers between interchange areas. Each transfer specifies " + "a from-area, to-area, transfer ID, and scheduled MW amount." + ), + "Owner": ( + "Defines ownership entities referenced by buses, branches, generators, and transformers. " + "Used for cost allocation and ownership tracking across the network." + ), + "FACTS": ( + "Represents Flexible AC Transmission System devices including SVCs, STATCOMs, TCSCs, " + "and UPFCs. Each device has control mode, setpoints, and impedance parameters." + ), + "Switched Shunt": ( + "Represents switchable shunt compensation with discrete step blocks. Each device has " + "up to 8 blocks (N1-N8, B1-B8) defining the number of steps and MVAR per step. " + "Control mode determines whether switching is discrete or continuous." + ), +} + + +# --------------------------------------------------------------------------- +# Cross-reference sections +# --------------------------------------------------------------------------- + +_PU = "per-unit-conventions.md" +_FCM = "field-criticality-matrix.md" +_MG = "mapping-guide.md" +_3W = "three-winding-transformers.md" + +_CROSS_REFS: dict[str, list[str]] = { + "Bus": [ + f"See [Per-Unit Convention Reference]({_PU}#bus-voltage) for VM/VA per-unit basis.", + f"See [Field Criticality Matrix]({_FCM}) for DCPF/ACPF criticality tiers.", + f"See [Record-Type Mapping Guide]({_MG}#bus) for tool-specific bus representations.", + ], + "Load": [ + f"See [Per-Unit Convention Reference]({_PU}) for load component scaling.", + f"See [Record-Type Mapping Guide]({_MG}#load) for tool-specific load representations.", + ], + "Fixed Shunt": [ + f"See [Per-Unit Convention Reference]({_PU}#shunt-admittance) for BL sign convention.", + f"See [Record-Type Mapping Guide]({_MG}#fixed-shunt)" + " for tool-specific shunt representations.", + ], + "Generator": [ + f"See [Per-Unit Convention Reference]({_PU}#generator-impedance) for MBASE-based per-unit.", + f"See [Record-Type Mapping Guide]({_MG}#generator)" + " for tool-specific generator representations.", + ], + "Branch": [ + f"See [Per-Unit Convention Reference]({_PU}#branch-impedance) for conversion formulas.", + f"See [Field Criticality Matrix]({_FCM}) for DCPF-critical branch fields.", + f"See [Record-Type Mapping Guide]({_MG}#branch) for tool-specific branch representations.", + ], + "Transformer": [ + f"See [Per-Unit Convention Reference]({_PU}#transformer-impedance)" + " for CW/CZ/CM conversions.", + f"See [3-Winding Transformer Reference]({_3W}) for topology details.", + f"See [Field Criticality Matrix]({_FCM}) for preservation-critical transformer fields.", + f"See [Record-Type Mapping Guide]({_MG}#transformer) for tool-specific representations.", + ], + "Area": [ + f"See [Record-Type Mapping Guide]({_MG}#area) for tool-specific area representations.", + ], + "Two-Terminal DC": [ + f"See [Record-Type Mapping Guide]({_MG}#two-terminal-dc)" + " for tool-specific HVDC representations.", + ], + "VSC DC": [ + f"See [Record-Type Mapping Guide]({_MG}#vsc-dc) for tool-specific VSC representations.", + ], + "Impedance Correction": [ + f"See [Record-Type Mapping Guide]({_MG}#impedance-correction)" + " for tool-specific representations.", + ], + "Multi-Terminal DC": [ + f"See [Record-Type Mapping Guide]({_MG}#multi-terminal-dc)" + " for tool-specific representations.", + ], + "Multi-Section Line": [ + f"See [Record-Type Mapping Guide]({_MG}#multi-section-line)" + " for tool-specific representations.", + ], + "Zone": [ + f"See [Record-Type Mapping Guide]({_MG}#zone) for tool-specific zone representations.", + ], + "Interarea Transfer": [ + f"See [Record-Type Mapping Guide]({_MG}#interarea-transfer)" + " for tool-specific representations.", + ], + "Owner": [ + f"See [Record-Type Mapping Guide]({_MG}#owner) for tool-specific owner representations.", + ], + "FACTS": [ + f"See [Record-Type Mapping Guide]({_MG}#facts) for tool-specific FACTS representations.", + ], + "Switched Shunt": [ + f"See [Per-Unit Convention Reference]({_PU}#shunt-admittance) for BL sign convention.", + f"See [Record-Type Mapping Guide]({_MG}#switched-shunt) for tool-specific representations.", + ], +} + + +def generate_document(schemas: list[TableSchema]) -> str: + """Generate the full intermediate-schema.md document.""" + lines: list[str] = [] + + # --- Front matter --- + lines.append("# Intermediate Format Schema Reference") + lines.append("") + lines.append("**Version:** 1.0") + lines.append("**Phase 1 Schema:** `../intermediate/schemas/` (JSON Schema Draft 2020-12)") + lines.append("**Audience:** evaluate-tool agents, human reviewers") + lines.append("**Normative definitions:** Phase 1 D7 JSON Schema files define data types,") + lines.append(" required/optional status, and valid ranges. This document adds semantic") + lines.append(" descriptions, worked examples, and ingestion verification guidance.") + lines.append("") + + # --- Table Summary --- + lines.append("## Table Summary") + lines.append("") + lines.append("| Table | PSS/E Record Type | Records | Columns | Primary Key | Purpose |") + lines.append("| ----- | ----------------- | ------- | ------- | ----------- | ------- |") + for ts in schemas: + pk = ", ".join(ts.primary_key) + count = _APPROX_COUNTS.get(ts.record_type, "~?") + purpose = ts.description.split(".")[0] + lines.append( + f"| `{ts.table_name}` | {ts.record_type} | {count} | " + f"{len(ts.fields)} | `[{pk}]` | {purpose} |" + ) + lines.append("") + + # --- Per-table sections --- + for ts in schemas: + rt = ts.record_type + + # Header block + lines.append(f"## {rt}") + lines.append("") + lines.append(f"**Table name:** `{ts.table_name}`") + lines.append( + f"**Schema file:** [`../intermediate/schemas/{ts.table_name}.schema.json`]" + f"(../intermediate/schemas/{ts.table_name}.schema.json)" + ) + pk_str = ", ".join(ts.primary_key) + lines.append(f"**Primary key:** `[{pk_str}]`") + lines.append(f"**Purpose:** {_PURPOSES.get(rt, ts.description)}") + lines.append("") + + # Field description table + lines.append("### Fields") + lines.append("") + lines.append( + "| Field | Type | Unit | Semantic Description | Expected Range " + "| Nullable | Default | Evaluate-Tool Guidance |" + ) + lines.append( + "| ----- | ---- | ---- | -------------------- | -------------- " + "| -------- | ------- | ---------------------- |" + ) + for f in ts.fields: + sem_desc, exp_range, guidance = _get_semantic(rt, f) + nullable = "yes" if not f.required else "no" + # Escape pipes in content + sem_desc_esc = sem_desc.replace("|", "\\|") + guidance_esc = guidance.replace("|", "\\|") + exp_range_esc = exp_range.replace("|", "\\|") + lines.append( + f"| `{f.name}` | {f.data_type} | {_unit(f)} | " + f"{sem_desc_esc} | {exp_range_esc} | {nullable} | " + f"{_default_str(f)} | {guidance_esc} |" + ) + lines.append("") + + # Worked example + lines.append("### Worked Example") + lines.append("") + example = _WORKED_EXAMPLES.get(rt, "") + if example: + lines.append(example) + else: + lines.append("```") + lines.append("(No example available)") + lines.append("```") + lines.append("") + + # Nullable/default behavior + lines.append("### Nullable and Default Behavior") + lines.append("") + lines.append(_NULLABLE_BEHAVIOR.get(rt, "See field table above for default values.")) + lines.append("") + + # Cross-references + lines.append("### Cross-References") + lines.append("") + refs = _CROSS_REFS.get(rt, []) + if refs: + for ref in refs: + lines.append(f"- {ref}") + else: + lines.append( + "- See [Field Criticality Matrix](field-criticality-matrix.md) " + "for criticality tiers." + ) + lines.append("") + + # --- Appendix: Preservation-Critical Fields --- + lines.append("## Appendix: Preservation-Critical Fields") + lines.append("") + lines.append( + "Fields with `x-psse-preservation-critical: true` in the Phase 1 JSON Schema. " + "These fields carry elevated fidelity requirements and generate mandatory test cases." + ) + lines.append("") + lines.append("| Record Type | Field | Why Preservation-Critical |") + lines.append("| ----------- | ----- | ------------------------- |") + for ts in schemas: + for f in ts.fields: + if f.preservation_critical: + sem, _, _ = _get_semantic(ts.record_type, f) + # Strip the prefix for the appendix + why = sem.replace("**[preservation-critical]** ", "").split(".")[0] + lines.append(f"| {ts.record_type} | `{f.name}` | {why} |") + lines.append("") + + # --- Appendix: Present-but-Inactive Fields --- + lines.append("## Appendix: Present-but-Inactive Fields") + lines.append("") + lines.append( + "Fields with `x-psse-present-but-inactive: true` in the Phase 1 JSON Schema. " + "These fields are uniformly at their default values across the entire dataset. " + "Evaluate-tool should not penalize a tool for omitting or zeroing these fields." + ) + lines.append("") + lines.append("| Record Type | Field | Default Value | Note |") + lines.append("| ----------- | ----- | ------------- | ---- |") + for ts in schemas: + for f in ts.fields: + if f.present_but_inactive: + note = "Field present in schema but uniformly at default in FNM data" + lines.append(f"| {ts.record_type} | `{f.name}` | {_default_str(f)} | {note} |") + lines.append("") + + # --- Appendix: Schema Cross-Reference Index --- + lines.append("## Appendix: Schema Cross-Reference Index") + lines.append("") + lines.append( + "Lookup table mapping each JSON Schema file to its corresponding section in this document." + ) + lines.append("") + lines.append("| Schema File | Document Section |") + lines.append("| ----------- | ---------------- |") + for ts in schemas: + lines.append( + f"| `../intermediate/schemas/{ts.table_name}.schema.json` " + f"| [## {ts.record_type}](#{ts.record_type.lower().replace(' ', '-')}) |" + ) + lines.append("") + + return "\n".join(lines) + + +def main() -> None: + """Generate and write the intermediate schema reference document.""" + schemas = get_table_schemas() + doc = generate_document(schemas) + + output_path = Path(__file__).resolve().parent.parent / "docs" / "intermediate-schema.md" + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(doc, encoding="utf-8") + print(f"Wrote {output_path} ({len(doc)} bytes, {doc.count(chr(10))} lines)") + + +if __name__ == "__main__": + main() diff --git a/data/fnm/scripts/gridcal_parser.py b/data/fnm/scripts/gridcal_parser.py new file mode 100644 index 00000000..36dc6690 --- /dev/null +++ b/data/fnm/scripts/gridcal_parser.py @@ -0,0 +1,696 @@ +"""GridCal v31 RAW file parser with structured logging and CSV export. + +Loads a PSS/E v31 RAW file through GridCal (VeraGridEngine), captures parser +logs, counts intermediate and final element counts, and exports GridCal +collections to CSV files for downstream validation. + +Data classes and constants are importable without GridCal installed. Functions +that require GridCal perform lazy imports and raise ImportError with a helpful +message if VeraGridEngine is not available. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path + +from fnm.scripts.raw_record_counter import PSSE_V31_SECTION_NAMES + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +PSSE_TO_GRIDCAL_MAPPING: dict[str, str | None] = { + "Bus": "buses", + "Load": "loads", + "Fixed Shunt": "shunts", + "Generator": "generators", + "Branch": "lines", + "Transformer": "transformers2w", + "Area": "areas", + "Two-Terminal DC": "hvdc_lines", + "VSC DC": "vsc_devices", + "Impedance Correction": None, # merged into transformer tap tables + "Multi-Terminal DC": None, # dropped — not supported + "Multi-Section Line": None, # dropped — not supported + "Zone": "zones", + "Interarea Transfer": None, # dropped — no GridCal equivalent + "Owner": None, # dropped — metadata only + "FACTS": "facts_devices", + "Switched Shunt": "controllable_shunts", +} + +GRIDCAL_ELEMENT_COLLECTIONS: tuple[str, ...] = ( + "buses", + "loads", + "shunts", + "generators", + "lines", + "transformers2w", + "transformers3w", + "areas", + "zones", + "hvdc_lines", + "vsc_devices", + "facts_devices", + "controllable_shunts", + "batteries", + "static_generators", + "substations", + "voltage_levels", + "connectivity_nodes", + "fluid_nodes", + "fluid_paths", +) + +# --------------------------------------------------------------------------- +# Data Classes +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ParserLogEntry: + """A single log entry emitted during GridCal parsing. + + Attributes: + time: ISO-8601 timestamp of the log entry. + severity: Log level (e.g. 'INFO', 'WARNING', 'ERROR'). + message: Human-readable log message. + device: Device identifier, if applicable. + device_class: Device class/type name, if applicable. + value: Actual value that triggered the log entry, if applicable. + expected_value: Expected value for comparison, if applicable. + """ + + time: str + severity: str + message: str + device: str = "" + device_class: str = "" + value: str = "" + expected_value: str = "" + + +@dataclass +class ParserLog: + """Aggregate parser log with individual entries and summary counts. + + Attributes: + entries: List of individual log entries. + info_count: Number of INFO-level entries. + warning_count: Number of WARNING-level entries. + error_count: Number of ERROR-level entries. + """ + + entries: list[ParserLogEntry] = field(default_factory=list) + info_count: int = 0 + warning_count: int = 0 + error_count: int = 0 + + +@dataclass(frozen=True) +class PsseIntermediateCounts: + """Record counts from the PsseCircuit intermediate representation. + + Each field corresponds to one of the 17 PSS/E v31 data sections. + """ + + bus: int = 0 + load: int = 0 + fixed_shunt: int = 0 + generator: int = 0 + branch: int = 0 + transformer: int = 0 + area: int = 0 + two_terminal_dc: int = 0 + vsc_dc: int = 0 + impedance_correction: int = 0 + multi_terminal_dc: int = 0 + multi_section_line: int = 0 + zone: int = 0 + interarea_transfer: int = 0 + owner: int = 0 + facts: int = 0 + switched_shunt: int = 0 + + +@dataclass(frozen=True) +class MultiCircuitCounts: + """Element counts from the final GridCal MultiCircuit. + + Each field corresponds to a named element collection on the MultiCircuit. + """ + + buses: int = 0 + loads: int = 0 + shunts: int = 0 + generators: int = 0 + lines: int = 0 + transformers2w: int = 0 + transformers3w: int = 0 + areas: int = 0 + zones: int = 0 + hvdc_lines: int = 0 + vsc_devices: int = 0 + facts_devices: int = 0 + controllable_shunts: int = 0 + batteries: int = 0 + static_generators: int = 0 + substations: int = 0 + voltage_levels: int = 0 + connectivity_nodes: int = 0 + fluid_nodes: int = 0 + fluid_paths: int = 0 + + +@dataclass(frozen=True) +class RecordTypeMapping: + """Documents how one PSS/E section maps to a GridCal collection. + + Attributes: + psse_section: PSS/E v31 section name. + gridcal_collection: GridCal MultiCircuit property name, or None. + status: Mapping status — 'mapped', 'dropped', or 'merged'. + notes: Explanation of the mapping or reason for dropping/merging. + """ + + psse_section: str + gridcal_collection: str | None + status: str + notes: str + + +@dataclass(frozen=True) +class GridCalParserSummary: + """Complete output of a GridCal parse operation. + + Attributes: + raw_path: Path to the source RAW file. + psse_intermediate_counts: Counts from PsseCircuit. + multicircuit_counts: Counts from MultiCircuit. + parser_log: Structured parser log. + record_type_mapping: Mapping documentation for all 17 sections. + csv_files: List of exported CSV file paths. + log_file: Path to the exported log file, or None. + timestamp: ISO-8601 timestamp of the parse run. + """ + + raw_path: str + psse_intermediate_counts: PsseIntermediateCounts + multicircuit_counts: MultiCircuitCounts + parser_log: ParserLog + record_type_mapping: list[RecordTypeMapping] + csv_files: list[str] + log_file: str | None + timestamp: str + + +# --------------------------------------------------------------------------- +# Lazy GridCal Import Helper +# --------------------------------------------------------------------------- + + +def _require_gridcal(): # noqa: ANN202 + """Import and return VeraGridEngine, raising ImportError if unavailable.""" + try: + import VeraGridEngine as vge + + return vge + except ImportError: + raise ImportError( + "VeraGridEngine (GridCal) is not installed. " + "Install it with: pip install GridCal or uv add GridCal" + ) from None + + +# --------------------------------------------------------------------------- +# Functions +# --------------------------------------------------------------------------- + + +def load_raw_with_logging(raw_path: str | Path) -> tuple: + """Load a PSS/E RAW file via GridCal and return the parsed objects. + + Args: + raw_path: Path to the PSS/E v31 RAW file. + + Returns: + A tuple of (MultiCircuit, psse_circuit_or_None, Logger). + psse_circuit may be None if the GridCal API does not expose + the intermediate PsseCircuit object. + + Raises: + ImportError: If VeraGridEngine is not installed. + FileNotFoundError: If raw_path does not exist. + """ + _require_gridcal() + raw_path = Path(raw_path) + if not raw_path.exists(): + raise FileNotFoundError(f"RAW file not found: {raw_path}") + + # Use the FileOpen API for access to the logger + from VeraGridEngine.IO.file_handler import FileOpen + + fo = FileOpen(str(raw_path)) + fo.open() + grid = fo.circuit + logger = fo.logger + + # Try to get the psse_circuit intermediate, if available + psse_circuit = getattr(fo, "psse_circuit", None) + + return (grid, psse_circuit, logger) + + +def extract_logger_entries(logger: object) -> ParserLog: + """Convert a GridCal Logger object to a structured ParserLog. + + Args: + logger: A GridCal Logger instance with messages. + + Returns: + A ParserLog with entries and aggregate counts. + """ + entries: list[ParserLogEntry] = [] + info_count = 0 + warning_count = 0 + error_count = 0 + + # GridCal Logger stores messages as lists of strings + # Try common attribute patterns + messages: list[str] = [] + if hasattr(logger, "messages"): + messages = list(logger.messages) if logger.messages else [] + elif hasattr(logger, "entries"): + messages = list(logger.entries) if logger.entries else [] + + now = datetime.now(tz=timezone.utc).isoformat() + + for msg in messages: + msg_str = str(msg) + severity = "INFO" + if "error" in msg_str.lower(): + severity = "ERROR" + error_count += 1 + elif "warn" in msg_str.lower(): + severity = "WARNING" + warning_count += 1 + else: + info_count += 1 + + entries.append( + ParserLogEntry( + time=now, + severity=severity, + message=msg_str, + ) + ) + + return ParserLog( + entries=entries, + info_count=info_count, + warning_count=warning_count, + error_count=error_count, + ) + + +def count_psse_intermediate(psse_circuit: object | None) -> PsseIntermediateCounts: + """Extract record counts from a PsseCircuit intermediate object. + + Args: + psse_circuit: The GridCal PsseCircuit object, or None. + + Returns: + PsseIntermediateCounts with counts for each PSS/E section. + All zeros if psse_circuit is None. + """ + if psse_circuit is None: + return PsseIntermediateCounts() + + # Map PSS/E section names to PsseCircuit attribute names + attr_map = { + "bus": ("buses",), + "load": ("loads",), + "fixed_shunt": ("fixed_shunts",), + "generator": ("generators",), + "branch": ("branches",), + "transformer": ("transformers",), + "area": ("areas",), + "two_terminal_dc": ("two_terminal_dc",), + "vsc_dc": ("vsc_dc",), + "impedance_correction": ("impedance_corrections",), + "multi_terminal_dc": ("multi_terminal_dc",), + "multi_section_line": ("multi_section_lines",), + "zone": ("zones",), + "interarea_transfer": ("interarea_transfers",), + "owner": ("owners",), + "facts": ("facts",), + "switched_shunt": ("switched_shunts",), + } + + counts: dict[str, int] = {} + for field_name, attr_names in attr_map.items(): + count = 0 + for attr_name in attr_names: + val = getattr(psse_circuit, attr_name, None) + if val is not None: + try: + count = len(val) + except TypeError: + count = 0 + break + counts[field_name] = count + + return PsseIntermediateCounts(**counts) + + +def count_multicircuit(grid: object) -> MultiCircuitCounts: + """Extract element counts from a GridCal MultiCircuit. + + Args: + grid: A GridCal MultiCircuit instance. + + Returns: + MultiCircuitCounts with counts for each element collection. + """ + counts: dict[str, int] = {} + for collection_name in GRIDCAL_ELEMENT_COLLECTIONS: + val = getattr(grid, collection_name, None) + if val is not None: + try: + counts[collection_name] = len(val) + except TypeError: + counts[collection_name] = 0 + else: + counts[collection_name] = 0 + + return MultiCircuitCounts(**counts) + + +def export_collection_to_csv(grid: object, collection_name: str, output_dir: Path) -> Path | None: + """Export one GridCal element collection as a CSV file. + + Args: + grid: A GridCal MultiCircuit instance. + collection_name: Name of the collection property on the grid. + output_dir: Directory where the CSV file will be written. + + Returns: + Path to the created CSV file, or None if the collection is empty. + """ + elements = getattr(grid, collection_name, None) + if elements is None or len(elements) == 0: + return None + + output_dir.mkdir(parents=True, exist_ok=True) + csv_path = output_dir / f"gridcal_{collection_name}.csv" + + # Attempt to extract properties from registered_properties or __dict__ + rows: list[dict[str, object]] = [] + for elem in elements: + row: dict[str, object] = {} + if hasattr(elem, "registered_properties"): + for prop in elem.registered_properties: + prop_name = getattr(prop, "name", str(prop)) + try: + row[prop_name] = getattr(elem, prop_name, None) + except Exception: + row[prop_name] = None + else: + # Fallback: use public attributes + for attr in dir(elem): + if not attr.startswith("_") and not callable(getattr(elem, attr, None)): + try: + row[attr] = getattr(elem, attr, None) + except Exception: + pass + rows.append(row) + + if not rows: + return None + + # Gather all column names preserving order + columns: list[str] = [] + seen: set[str] = set() + for row in rows: + for key in row: + if key not in seen: + columns.append(key) + seen.add(key) + + with open(csv_path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=columns, extrasaction="ignore") + writer.writeheader() + for row in rows: + # Convert non-serializable values to strings + safe_row = {k: str(v) if v is not None else "" for k, v in row.items()} + writer.writerow(safe_row) + + return csv_path + + +def export_all_collections(grid: object, output_dir: Path) -> list[Path]: + """Export all non-empty GridCal element collections to CSV files. + + Args: + grid: A GridCal MultiCircuit instance. + output_dir: Directory where CSV files will be written. + + Returns: + List of paths to the created CSV files. + """ + csv_files: list[Path] = [] + for collection_name in GRIDCAL_ELEMENT_COLLECTIONS: + result = export_collection_to_csv(grid, collection_name, output_dir) + if result is not None: + csv_files.append(result) + return csv_files + + +# Mapping notes for each PSS/E section +_MAPPING_NOTES: dict[str, str] = { + "Bus": "Direct 1:1 mapping to GridCal Bus objects.", + "Load": "Direct 1:1 mapping to GridCal Load objects.", + "Fixed Shunt": "Mapped to GridCal Shunt objects (fixed admittance).", + "Generator": "Direct 1:1 mapping to GridCal Generator objects.", + "Branch": "Mapped to GridCal Line objects (pi-model branches).", + "Transformer": "Mapped to GridCal 2-winding transformer objects.", + "Area": "Direct 1:1 mapping to GridCal Area objects.", + "Two-Terminal DC": "Mapped to GridCal HVDC Line objects.", + "VSC DC": "Mapped to GridCal VSC device objects.", + "Impedance Correction": ( + "Merged into transformer tap-changer tables; no standalone GridCal collection." + ), + "Multi-Terminal DC": "Dropped — GridCal does not support multi-terminal DC.", + "Multi-Section Line": "Dropped — GridCal does not support multi-section lines.", + "Zone": "Direct 1:1 mapping to GridCal Zone objects.", + "Interarea Transfer": "Dropped — no GridCal equivalent for interarea transfer schedules.", + "Owner": "Dropped — ownership metadata not modeled in GridCal.", + "FACTS": "Mapped to GridCal FACTS device objects.", + "Switched Shunt": "Mapped to GridCal controllable shunt objects.", +} + + +def build_record_type_mapping() -> list[RecordTypeMapping]: + """Build 17 RecordTypeMapping entries documenting PSS/E-to-GridCal mapping. + + Returns: + List of 17 RecordTypeMapping entries, one per PSS/E v31 section. + """ + mappings: list[RecordTypeMapping] = [] + for section_name in PSSE_V31_SECTION_NAMES: + gridcal_collection = PSSE_TO_GRIDCAL_MAPPING[section_name] + + if gridcal_collection is not None: + status = "mapped" + elif section_name == "Impedance Correction": + status = "merged" + else: + status = "dropped" + + notes = _MAPPING_NOTES.get(section_name, "") + + mappings.append( + RecordTypeMapping( + psse_section=section_name, + gridcal_collection=gridcal_collection, + status=status, + notes=notes, + ) + ) + + return mappings + + +def build_summary( + raw_path: str | Path, + grid: object, + psse_circuit: object | None, + csv_files: list[str], + log_file: str | None, +) -> GridCalParserSummary: + """Assemble a complete GridCalParserSummary. + + Args: + raw_path: Path to the source RAW file. + grid: GridCal MultiCircuit instance. + psse_circuit: GridCal PsseCircuit instance, or None. + csv_files: List of exported CSV file paths (as strings). + log_file: Path to the exported log file, or None. + + Returns: + A fully populated GridCalParserSummary. + """ + return GridCalParserSummary( + raw_path=str(raw_path), + psse_intermediate_counts=count_psse_intermediate(psse_circuit), + multicircuit_counts=count_multicircuit(grid), + parser_log=ParserLog(), + record_type_mapping=build_record_type_mapping(), + csv_files=csv_files, + log_file=log_file, + timestamp=datetime.now(tz=timezone.utc).isoformat(), + ) + + +def summary_to_dict(summary: GridCalParserSummary) -> dict: + """Convert a GridCalParserSummary to a JSON-serializable dict. + + Args: + summary: The summary to convert. + + Returns: + A dict suitable for ``json.dumps()``. + """ + return { + "raw_path": summary.raw_path, + "timestamp": summary.timestamp, + "psse_intermediate_counts": { + "bus": summary.psse_intermediate_counts.bus, + "load": summary.psse_intermediate_counts.load, + "fixed_shunt": summary.psse_intermediate_counts.fixed_shunt, + "generator": summary.psse_intermediate_counts.generator, + "branch": summary.psse_intermediate_counts.branch, + "transformer": summary.psse_intermediate_counts.transformer, + "area": summary.psse_intermediate_counts.area, + "two_terminal_dc": summary.psse_intermediate_counts.two_terminal_dc, + "vsc_dc": summary.psse_intermediate_counts.vsc_dc, + "impedance_correction": summary.psse_intermediate_counts.impedance_correction, + "multi_terminal_dc": summary.psse_intermediate_counts.multi_terminal_dc, + "multi_section_line": summary.psse_intermediate_counts.multi_section_line, + "zone": summary.psse_intermediate_counts.zone, + "interarea_transfer": summary.psse_intermediate_counts.interarea_transfer, + "owner": summary.psse_intermediate_counts.owner, + "facts": summary.psse_intermediate_counts.facts, + "switched_shunt": summary.psse_intermediate_counts.switched_shunt, + }, + "multicircuit_counts": { + name: getattr(summary.multicircuit_counts, name) for name in GRIDCAL_ELEMENT_COLLECTIONS + }, + "parser_log": parser_log_to_dict(summary.parser_log), + "record_type_mapping": [ + { + "psse_section": m.psse_section, + "gridcal_collection": m.gridcal_collection, + "status": m.status, + "notes": m.notes, + } + for m in summary.record_type_mapping + ], + "csv_files": summary.csv_files, + "log_file": summary.log_file, + } + + +def parser_log_to_dict(log: ParserLog) -> dict: + """Convert a ParserLog to a JSON-serializable dict. + + Args: + log: The parser log to convert. + + Returns: + A dict suitable for ``json.dumps()``. + """ + return { + "entries": [ + { + "time": e.time, + "severity": e.severity, + "message": e.message, + "device": e.device, + "device_class": e.device_class, + "value": e.value, + "expected_value": e.expected_value, + } + for e in log.entries + ], + "info_count": log.info_count, + "warning_count": log.warning_count, + "error_count": log.error_count, + } + + +def main(argv: list[str] | None = None) -> None: + """CLI entry point for GridCal parser execution. + + Usage: + python -m fnm.scripts.gridcal_parser /path/to/file.raw [-o output_dir] + """ + _require_gridcal() + + parser = argparse.ArgumentParser( + description="Parse a PSS/E v31 RAW file via GridCal and export results." + ) + parser.add_argument("raw_file", type=str, help="Path to the PSS/E v31 RAW file") + parser.add_argument( + "-o", + "--output-dir", + type=str, + default=None, + help="Output directory for CSV exports and summary JSON", + ) + args = parser.parse_args(argv) + + raw_path = Path(args.raw_file) + output_dir = Path(args.output_dir) if args.output_dir else raw_path.parent / "gridcal_output" + + # Load and parse + grid, psse_circuit, logger = load_raw_with_logging(raw_path) + + # Extract log + parser_log = extract_logger_entries(logger) + + # Export CSVs + csv_paths = export_all_collections(grid, output_dir) + csv_files = [str(p) for p in csv_paths] + + # Write log file + log_file_path = output_dir / "parser_log.json" + output_dir.mkdir(parents=True, exist_ok=True) + log_dict = parser_log_to_dict(parser_log) + log_file_path.write_text(json.dumps(log_dict, indent=2) + "\n", encoding="utf-8") + + # Build and write summary + summary = GridCalParserSummary( + raw_path=str(raw_path), + psse_intermediate_counts=count_psse_intermediate(psse_circuit), + multicircuit_counts=count_multicircuit(grid), + parser_log=parser_log, + record_type_mapping=build_record_type_mapping(), + csv_files=csv_files, + log_file=str(log_file_path), + timestamp=datetime.now(tz=timezone.utc).isoformat(), + ) + + summary_path = output_dir / "gridcal_summary.json" + summary_dict = summary_to_dict(summary) + summary_path.write_text(json.dumps(summary_dict, indent=2) + "\n", encoding="utf-8") + + print(f"Summary written to {summary_path}", file=sys.stderr) + print(f"Exported {len(csv_files)} CSV file(s) to {output_dir}", file=sys.stderr) + print(json.dumps(summary_dict, indent=2)) diff --git a/data/fnm/scripts/intermediate_schema.py b/data/fnm/scripts/intermediate_schema.py new file mode 100644 index 00000000..b1f39caa --- /dev/null +++ b/data/fnm/scripts/intermediate_schema.py @@ -0,0 +1,1591 @@ +"""Intermediate Format Schema Specification for PSS/E v31. + +Defines JSON Schema files (one per PSS/E v31 record type table) plus a +top-level manifest schema and a human-readable markdown summary. The schema +is the central artifact of Phase 1 -- every reference solution, mapping guide, +supplemental CSV join, and per-tool FNM ingestion test flows through it. + +Field inventory is hardcoded to PSS/E v31 Program Operation Manual definitions. +""" + +from __future__ import annotations + +import argparse +import csv +import json +from collections.abc import Callable +from dataclasses import dataclass +from enum import Enum +from pathlib import Path + +from fnm.scripts.raw_record_counter import PSSE_V31_SECTION_NAMES + +# --------------------------------------------------------------------------- +# PSS/E v31 record types -- canonical ordering +# --------------------------------------------------------------------------- + +PSSE_V31_RECORD_TYPES: tuple[str, ...] = PSSE_V31_SECTION_NAMES +"""All 17 PSS/E v31 record types in section order. Only non-empty types +(as reported by D3) get a table schema in the intermediate format.""" + + +# --------------------------------------------------------------------------- +# Per-unit base classification +# --------------------------------------------------------------------------- + + +class PerUnitBase(Enum): + """Classification of a field's per-unit base reference.""" + + SYSTEM_MVA = "system_mva" + WINDING_MVA = "winding_mva" + BUS_KV = "bus_kv" + NONE = "none" + MIXED = "mixed" + + +# --------------------------------------------------------------------------- +# Field specification +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class FieldSpec: + """Schema specification for a single field in a PSS/E record type.""" + + name: str + data_type: str + description: str + required: bool + per_unit_base: PerUnitBase + unit: str + default_value: int | float | str | None + valid_range: tuple[float | None, float | None] | None + present_but_inactive: bool = False + preservation_critical: bool = False + + +# --------------------------------------------------------------------------- +# Table schema +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class TableSchema: + """Schema specification for one PSS/E record type table.""" + + record_type: str + table_name: str + description: str + fields: list[FieldSpec] + primary_key: list[str] + multi_line_record: bool = False + notes: str = "" + + +# --------------------------------------------------------------------------- +# Manifest +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ManifestEntry: + """Entry for one table in the intermediate format manifest.""" + + table_name: str + record_type: str + file_name: str + record_count: int + column_count: int + schema_file: str + + +@dataclass(frozen=True) +class IntermediateFormatManifest: + """Top-level manifest for the intermediate format.""" + + sbase: float + basfrq: float + rev: float + case_id: str + canonical_parser: str + tables: list[ManifestEntry] + total_records: int + total_tables: int + non_empty_record_types: list[str] + schema_version: str = "1.0.0" + generated_timestamp: str = "" + + +# --------------------------------------------------------------------------- +# Validation report +# --------------------------------------------------------------------------- + + +class ConformanceLevel(Enum): + """Severity level for a conformance finding.""" + + ERROR = "error" + WARNING = "warning" + INFO = "info" + + +@dataclass(frozen=True) +class ConformanceFinding: + """A single conformance check result.""" + + table_name: str + field_name: str + level: ConformanceLevel + message: str + check_id: str + + +@dataclass(frozen=True) +class ConformanceReport: + """Complete validation report for intermediate format tables.""" + + tables_checked: int + tables_expected: int + errors: list[ConformanceFinding] + warnings: list[ConformanceFinding] + info: list[ConformanceFinding] + is_conformant: bool + manifest_valid: bool + + +# --------------------------------------------------------------------------- +# PSS/E v31 Field Inventory -- hardcoded from Program Operation Manual +# --------------------------------------------------------------------------- + +_F = FieldSpec +_N = PerUnitBase.NONE +_S = PerUnitBase.SYSTEM_MVA +_B = PerUnitBase.BUS_KV +_M = PerUnitBase.MIXED +_BLANK12 = " " + + +def _bus_fields() -> list[FieldSpec]: + return [ + _F("I", "integer", "Bus number (1-999997)", True, _N, "", None, (1, 999997)), + _F("NAME", "string", "Bus name (up to 12 chars)", True, _N, "", _BLANK12, None), + _F("BASKV", "number", "Bus base voltage", True, _N, "kV", 0.0, (0.0, None)), + _F("IDE", "integer", "Bus type code (1-4)", True, _N, "", 1, (1, 4)), + _F("AREA", "integer", "Area number", True, _N, "", 1, (1, None)), + _F("ZONE", "integer", "Zone number", True, _N, "", 1, (1, None)), + _F("OWNER", "integer", "Owner number", True, _N, "", 1, (1, None)), + _F("VM", "number", "Bus voltage magnitude", True, _B, "pu", 1.0, (0.0, 2.0)), + _F("VA", "number", "Bus voltage angle", True, _N, "deg", 0.0, None), + _F("NVHI", "number", "Normal voltage high limit", False, _B, "pu", 1.1, (0.0, 2.0)), + _F("NVLO", "number", "Normal voltage low limit", False, _B, "pu", 0.9, (0.0, 2.0)), + _F("EVHI", "number", "Emergency voltage high limit", False, _B, "pu", 1.1, (0.0, 2.0)), + _F("EVLO", "number", "Emergency voltage low limit", False, _B, "pu", 0.9, (0.0, 2.0)), + ] + + +def _load_fields() -> list[FieldSpec]: + return [ + _F("I", "integer", "Bus number", True, _N, "", None, None), + _F("ID", "string", "Load identifier (2 chars)", True, _N, "", "1 ", None), + _F("STATUS", "integer", "Load status (1=in, 0=out)", True, _N, "", 1, (0, 1)), + _F("AREA", "integer", "Area number", True, _N, "", 1, None), + _F("ZONE", "integer", "Zone number", True, _N, "", 1, None), + _F("PL", "number", "Constant power load (P)", True, _N, "MW", 0.0, None), + _F("QL", "number", "Constant power load (Q)", True, _N, "MVAR", 0.0, None), + _F("IP", "number", "Constant current load (P)", False, _N, "MW", 0.0, None), + _F("IQ", "number", "Constant current load (Q)", False, _N, "MVAR", 0.0, None), + _F("YP", "number", "Constant admittance load (P)", False, _N, "MW", 0.0, None), + _F("YQ", "number", "Constant admittance load (Q)", False, _N, "MVAR", 0.0, None), + _F("OWNER", "integer", "Owner number", False, _N, "", 1, None), + _F("SCALE", "integer", "Scaling flag (1=yes, 0=no)", False, _N, "", 1, (0, 1)), + ] + + +def _fixed_shunt_fields() -> list[FieldSpec]: + return [ + _F("I", "integer", "Bus number", True, _N, "", None, None), + _F("ID", "string", "Shunt identifier (2 chars)", True, _N, "", "1 ", None), + _F("STATUS", "integer", "Status (1=in, 0=out)", True, _N, "", 1, (0, 1)), + _F("GL", "number", "Shunt conductance", True, _N, "MW", 0.0, None), + _F("BL", "number", "Shunt susceptance (+cap)", True, _N, "MVAR", 0.0, None), + ] + + +def _generator_fields() -> list[FieldSpec]: + return [ + _F("I", "integer", "Bus number", True, _N, "", None, None), + _F("ID", "string", "Machine identifier (2 chars)", True, _N, "", "1 ", None), + _F("PG", "number", "Active power output", True, _N, "MW", 0.0, None), + _F("QG", "number", "Reactive power output", True, _N, "MVAR", 0.0, None), + _F("QT", "number", "Max reactive power", True, _N, "MVAR", 9999.0, None), + _F("QB", "number", "Min reactive power", True, _N, "MVAR", -9999.0, None), + _F("VS", "number", "Regulated voltage setpoint", True, _B, "pu", 1.0, None), + _F("IREG", "integer", "Remote regulated bus (0=local)", True, _N, "", 0, None, False, True), + _F("MBASE", "number", "Machine MVA base", True, _N, "MVA", 100.0, None), + _F("ZR", "number", "Machine resistance (on MBASE)", False, _S, "pu", 0.0, None), + _F("ZX", "number", "Machine reactance (on MBASE)", False, _S, "pu", 1.0, None), + _F("RT", "number", "Step-up xfmr resistance", False, _S, "pu", 0.0, None), + _F("XT", "number", "Step-up xfmr reactance", False, _S, "pu", 0.0, None), + _F("GTAP", "number", "Step-up xfmr tap ratio", False, _B, "pu", 1.0, None), + _F("STAT", "integer", "Status (1=in, 0=out)", True, _N, "", 1, (0, 1)), + _F("RMPCT", "number", "MVAR range pct for remote reg", False, _N, "%", 100.0, (0.0, 100.0)), + _F("PT", "number", "Max active power", False, _N, "MW", 9999.0, None), + _F("PB", "number", "Min active power", False, _N, "MW", -9999.0, None), + _F("O1", "integer", "Owner 1", False, _N, "", 1, None), + _F("F1", "number", "Fraction owned by owner 1", False, _N, "", 1.0, (0.0, 1.0)), + _F("O2", "integer", "Owner 2", False, _N, "", 0, None), + _F("F2", "number", "Fraction owned by owner 2", False, _N, "", 0.0, (0.0, 1.0)), + _F("O3", "integer", "Owner 3", False, _N, "", 0, None), + _F("F3", "number", "Fraction owned by owner 3", False, _N, "", 0.0, (0.0, 1.0)), + _F("O4", "integer", "Owner 4", False, _N, "", 0, None), + _F("F4", "number", "Fraction owned by owner 4", False, _N, "", 0.0, (0.0, 1.0)), + _F("WMOD", "integer", "Wind machine Q control mode", False, _N, "", 0, None), + _F("WPF", "number", "Wind machine power factor", False, _N, "", 1.0, None), + ] + + +def _branch_fields() -> list[FieldSpec]: + return [ + _F("I", "integer", "From bus number", True, _N, "", None, None), + _F("J", "integer", "To bus number", True, _N, "", None, None), + _F("CKT", "string", "Circuit identifier (2 chars)", True, _N, "", "1 ", None), + _F("R", "number", "Branch resistance", True, _S, "pu", None, None), + _F("X", "number", "Branch reactance", True, _S, "pu", None, None), + _F("B", "number", "Branch charging susceptance", True, _S, "pu", 0.0, None), + _F("RATEA", "number", "Rating A (normal)", True, _N, "MVA", 0.0, None), + _F("RATEB", "number", "Rating B (emergency)", True, _N, "MVA", 0.0, None), + _F("RATEC", "number", "Rating C (long-term)", True, _N, "MVA", 0.0, None), + _F("GI", "number", "Shunt conductance at I", False, _S, "pu", 0.0, None), + _F("BI", "number", "Shunt susceptance at I", False, _S, "pu", 0.0, None), + _F("GJ", "number", "Shunt conductance at J", False, _S, "pu", 0.0, None), + _F("BJ", "number", "Shunt susceptance at J", False, _S, "pu", 0.0, None), + _F("ST", "integer", "Status (1=in, 0=out)", True, _N, "", 1, (0, 1)), + _F("MET", "integer", "Metered end (1=I, 2=J)", False, _N, "", 1, (1, 2)), + _F("LEN", "number", "Line length", False, _N, "", 0.0, None), + _F("O1", "integer", "Owner 1", False, _N, "", 1, None), + _F("F1", "number", "Fraction owned by owner 1", False, _N, "", 1.0, (0.0, 1.0)), + _F("O2", "integer", "Owner 2", False, _N, "", 0, None), + _F("F2", "number", "Fraction owned by owner 2", False, _N, "", 0.0, (0.0, 1.0)), + _F("O3", "integer", "Owner 3", False, _N, "", 0, None), + _F("F3", "number", "Fraction owned by owner 3", False, _N, "", 0.0, (0.0, 1.0)), + _F("O4", "integer", "Owner 4", False, _N, "", 0, None), + _F("F4", "number", "Fraction owned by owner 4", False, _N, "", 0.0, (0.0, 1.0)), + ] + + +def _transformer_fields() -> list[FieldSpec]: + """Transformer fields -- all 5 lines flattened.""" + P = True # preservation_critical # noqa: N806 + return [ + # Line 1 -- common + _F("I", "integer", "Winding 1 bus", True, _N, "", None, None), + _F("J", "integer", "Winding 2 bus", True, _N, "", None, None), + _F("K", "integer", "Winding 3 bus (0=2W)", True, _N, "", 0, None, False, P), + _F("CKT", "string", "Circuit identifier", True, _N, "", "1 ", None), + _F("CW", "integer", "Winding data I/O code", True, _N, "", 1, (1, 3), False, P), + _F("CZ", "integer", "Impedance data I/O code", True, _N, "", 1, (1, 3), False, P), + _F("CM", "integer", "Mag admittance I/O code", True, _N, "", 1, (1, 2), False, P), + _F("MAG1", "number", "Magnetizing conductance", False, _M, "", 0.0, None), + _F("MAG2", "number", "Magnetizing susceptance", False, _M, "", 0.0, None), + _F("NMETR", "integer", "Non-metered end code", False, _N, "", 2, None), + _F("NAME", "string", "Transformer name", False, _N, "", _BLANK12, None), + _F("STAT", "integer", "Status (0-4)", True, _N, "", 1, (0, 4)), + _F("O1", "integer", "Owner 1", False, _N, "", 1, None), + _F("F1", "number", "Fraction by owner 1", False, _N, "", 1.0, (0.0, 1.0)), + _F("O2", "integer", "Owner 2", False, _N, "", 0, None), + _F("F2", "number", "Fraction by owner 2", False, _N, "", 0.0, (0.0, 1.0)), + _F("O3", "integer", "Owner 3", False, _N, "", 0, None), + _F("F3", "number", "Fraction by owner 3", False, _N, "", 0.0, (0.0, 1.0)), + _F("O4", "integer", "Owner 4", False, _N, "", 0, None), + _F("F4", "number", "Fraction by owner 4", False, _N, "", 0.0, (0.0, 1.0)), + _F("VECGRP", "string", "Vector group (12 chars)", False, _N, "", _BLANK12, None), + # Line 2 -- impedance + _F("R1_2", "number", "R winding 1-2 (CZ dep)", True, _M, "pu", 0.0, None), + _F("X1_2", "number", "X winding 1-2 (CZ dep)", True, _M, "pu", None, None), + _F("SBASE1_2", "number", "MVA base winding 1-2", True, _N, "MVA", 100.0, None), + _F("R2_3", "number", "R winding 2-3 (3W only)", False, _M, "pu", 0.0, None), + _F("X2_3", "number", "X winding 2-3 (3W only)", False, _M, "pu", 0.0, None), + _F("SBASE2_3", "number", "MVA base winding 2-3", False, _N, "MVA", 100.0, None), + _F("R3_1", "number", "R winding 3-1 (3W only)", False, _M, "pu", 0.0, None), + _F("X3_1", "number", "X winding 3-1 (3W only)", False, _M, "pu", 0.0, None), + _F("SBASE3_1", "number", "MVA base winding 3-1", False, _N, "MVA", 100.0, None), + _F("VMSTAR", "number", "Star bus voltage mag", False, _B, "pu", 1.0, None), + _F("ANSTAR", "number", "Star bus voltage angle", False, _N, "deg", 0.0, None), + # Line 3 -- winding 1 + _F("WINDV1", "number", "Winding 1 turns ratio", True, _M, "", 1.0, None, False, P), + _F("NOMV1", "number", "Winding 1 nominal kV", True, _N, "kV", 0.0, None, False, P), + _F("ANG1", "number", "Winding 1 phase shift", True, _N, "deg", 0.0, None, False, P), + _F("RATA1", "number", "Winding 1 rating A", True, _N, "MVA", 0.0, None, False, P), + _F("RATB1", "number", "Winding 1 rating B", False, _N, "MVA", 0.0, None), + _F("RATC1", "number", "Winding 1 rating C", False, _N, "MVA", 0.0, None), + _F("COD1", "integer", "Winding 1 tap control", False, _N, "", 0, None), + _F("CONT1", "integer", "Winding 1 ctrl bus", False, _N, "", 0, None), + _F("RMA1", "number", "Winding 1 upper tap limit", False, _M, "", 1.1, None), + _F("RMI1", "number", "Winding 1 lower tap limit", False, _M, "", 0.9, None), + _F("VMA1", "number", "Winding 1 upper V limit", False, _M, "", 1.1, None), + _F("VMI1", "number", "Winding 1 lower V limit", False, _M, "", 0.9, None), + _F("NTP1", "integer", "Winding 1 tap positions", False, _N, "", 33, None), + _F("TAB1", "integer", "Winding 1 impcor table", False, _N, "", 0, None), + _F("CR1", "number", "Winding 1 LDC resistance", False, _S, "pu", 0.0, None), + _F("CX1", "number", "Winding 1 LDC reactance", False, _S, "pu", 0.0, None), + _F("CNXA1", "integer", "Winding 1 conn angle", False, _N, "", 0, None), + # Line 4 -- winding 2 + _F("WINDV2", "number", "Winding 2 turns ratio", True, _M, "", 1.0, None, False, P), + _F("NOMV2", "number", "Winding 2 nominal kV", True, _N, "kV", 0.0, None, False, P), + _F("ANG2", "number", "Winding 2 phase shift", False, _N, "deg", 0.0, None), + _F("RATA2", "number", "Winding 2 rating A", False, _N, "MVA", 0.0, None, False, P), + _F("RATB2", "number", "Winding 2 rating B", False, _N, "MVA", 0.0, None), + _F("RATC2", "number", "Winding 2 rating C", False, _N, "MVA", 0.0, None), + _F("COD2", "integer", "Winding 2 tap control", False, _N, "", 0, None), + _F("CONT2", "integer", "Winding 2 ctrl bus", False, _N, "", 0, None), + _F("RMA2", "number", "Winding 2 upper tap limit", False, _M, "", 1.1, None), + _F("RMI2", "number", "Winding 2 lower tap limit", False, _M, "", 0.9, None), + _F("VMA2", "number", "Winding 2 upper V limit", False, _M, "", 1.1, None), + _F("VMI2", "number", "Winding 2 lower V limit", False, _M, "", 0.9, None), + _F("NTP2", "integer", "Winding 2 tap positions", False, _N, "", 33, None), + _F("TAB2", "integer", "Winding 2 impcor table", False, _N, "", 0, None), + _F("CR2", "number", "Winding 2 LDC resistance", False, _S, "pu", 0.0, None), + _F("CX2", "number", "Winding 2 LDC reactance", False, _S, "pu", 0.0, None), + _F("CNXA2", "integer", "Winding 2 conn angle", False, _N, "", 0, None), + # Line 5 -- winding 3 (nullable for 2W) + _F("WINDV3", "number", "Winding 3 turns ratio", False, _M, "", 1.0, None, False, P), + _F("NOMV3", "number", "Winding 3 nominal kV", False, _N, "kV", 0.0, None, False, P), + _F("ANG3", "number", "Winding 3 phase shift", False, _N, "deg", 0.0, None), + _F("RATA3", "number", "Winding 3 rating A", False, _N, "MVA", 0.0, None, False, P), + _F("RATB3", "number", "Winding 3 rating B", False, _N, "MVA", 0.0, None), + _F("RATC3", "number", "Winding 3 rating C", False, _N, "MVA", 0.0, None), + _F("COD3", "integer", "Winding 3 tap control", False, _N, "", 0, None), + _F("CONT3", "integer", "Winding 3 ctrl bus", False, _N, "", 0, None), + _F("RMA3", "number", "Winding 3 upper tap limit", False, _M, "", 1.1, None), + _F("RMI3", "number", "Winding 3 lower tap limit", False, _M, "", 0.9, None), + _F("VMA3", "number", "Winding 3 upper V limit", False, _M, "", 1.1, None), + _F("VMI3", "number", "Winding 3 lower V limit", False, _M, "", 0.9, None), + _F("NTP3", "integer", "Winding 3 tap positions", False, _N, "", 33, None), + _F("TAB3", "integer", "Winding 3 impcor table", False, _N, "", 0, None), + _F("CR3", "number", "Winding 3 LDC resistance", False, _S, "pu", 0.0, None), + _F("CX3", "number", "Winding 3 LDC reactance", False, _S, "pu", 0.0, None), + _F("CNXA3", "integer", "Winding 3 conn angle", False, _N, "", 0, None), + ] + + +def _area_fields() -> list[FieldSpec]: + P = True # noqa: N806 + return [ + _F("I", "integer", "Area number", True, _N, "", None, None), + _F("ISW", "integer", "Area slack bus number", True, _N, "", 0, None, False, P), + _F("PDES", "number", "Desired net interchange", True, _N, "MW", 0.0, None, False, P), + _F("PTOL", "number", "Interchange tolerance", True, _N, "MW", 10.0, None, False, P), + _F("ARNAME", "string", "Area name (12 chars)", True, _N, "", _BLANK12, None), + ] + + +def _two_terminal_dc_fields() -> list[FieldSpec]: + return [ + _F("NAME", "string", "DC line name", True, _N, "", None, None), + _F("MDC", "integer", "Control mode (0-2)", True, _N, "", 0, (0, 2)), + _F("RDC", "number", "DC line resistance", True, _N, "ohm", None, None), + _F("SETVL", "number", "Current or power demand", True, _N, "", 0.0, None), + _F("VSCHD", "number", "Scheduled DC voltage", True, _N, "kV", 0.0, None), + _F("VCMOD", "number", "Mode switch DC voltage", False, _N, "", 0.0, None), + _F("RCOMP", "number", "Compounding resistance", False, _N, "", 0.0, None), + _F("DELTI", "number", "Inverter firing angle margin", False, _N, "deg", 0.0, None), + _F("METER", "string", "Metered end (R or I)", False, _N, "", "I", None), + _F("DCVMIN", "number", "Min DC voltage", False, _N, "pu", 0.0, None), + _F("CCCITMX", "integer", "Max converter ctrl iters", False, _N, "", 20, None), + _F("CCCACC", "number", "Converter ctrl accel factor", False, _N, "", 1.0, None), + # Rectifier + _F("IPR", "integer", "Rectifier bus", True, _N, "", None, None), + _F("NBR", "integer", "Rectifier bridges", True, _N, "", None, None), + _F("ANMXR", "number", "Max rect firing angle", False, _N, "deg", 0.0, None), + _F("ANMNR", "number", "Min rect firing angle", False, _N, "deg", 0.0, None), + _F("RCR", "number", "Rect commutating R", False, _N, "", 0.0, None), + _F("XCR", "number", "Rect commutating X", False, _N, "", 0.0, None), + _F("EBASR", "number", "Rect primary base kV", False, _N, "kV", 0.0, None), + _F("TRR", "number", "Rect xfmr ratio", False, _N, "", 1.0, None), + _F("TAPR", "number", "Rect tap setting", False, _N, "", 1.0, None), + _F("TMXR", "number", "Max rect tap", False, _N, "", 1.5, None), + _F("TMNR", "number", "Min rect tap", False, _N, "", 0.51, None), + _F("STPR", "number", "Rect tap step", False, _N, "", 0.00625, None), + _F("ICR", "integer", "Rect firing angle ctrl bus", False, _N, "", 0, None), + _F("IFR", "integer", "Rect commutating bus (from)", False, _N, "", 0, None), + _F("ITR", "integer", "Rect commutating bus (to)", False, _N, "", 0, None), + _F("IDR", "string", "Rect circuit ID", False, _N, "", "1 ", None), + _F("XCAPR", "number", "Rect capacitor reactance", False, _N, "", 0.0, None), + # Inverter + _F("IPI", "integer", "Inverter bus", True, _N, "", None, None), + _F("NBI", "integer", "Inverter bridges", True, _N, "", None, None), + _F("ANMXI", "number", "Max inv firing angle", False, _N, "deg", 0.0, None), + _F("ANMNI", "number", "Min inv firing angle", False, _N, "deg", 0.0, None), + _F("RCI", "number", "Inv commutating R", False, _N, "", 0.0, None), + _F("XCI", "number", "Inv commutating X", False, _N, "", 0.0, None), + _F("EBASI", "number", "Inv primary base kV", False, _N, "kV", 0.0, None), + _F("TRI", "number", "Inv xfmr ratio", False, _N, "", 1.0, None), + _F("TAPI", "number", "Inv tap setting", False, _N, "", 1.0, None), + _F("TMXI", "number", "Max inv tap", False, _N, "", 1.5, None), + _F("TMNI", "number", "Min inv tap", False, _N, "", 0.51, None), + _F("STPI", "number", "Inv tap step", False, _N, "", 0.00625, None), + _F("ICI", "integer", "Inv firing angle ctrl bus", False, _N, "", 0, None), + _F("IFI", "integer", "Inv commutating bus (from)", False, _N, "", 0, None), + _F("ITI", "integer", "Inv commutating bus (to)", False, _N, "", 0, None), + _F("IDI", "string", "Inv circuit ID", False, _N, "", "1 ", None), + _F("XCAPI", "number", "Inv capacitor reactance", False, _N, "", 0.0, None), + ] + + +def _vsc_dc_fields() -> list[FieldSpec]: + return [ + _F("NAME", "string", "VSC DC line name", True, _N, "", None, None), + _F("MDC", "integer", "Control mode", True, _N, "", 0, None), + _F("RDC", "number", "DC line resistance", True, _N, "ohm", None, None), + _F("O1", "integer", "Owner 1", False, _N, "", 1, None), + _F("F1", "number", "Fraction by owner 1", False, _N, "", 1.0, None), + _F("O2", "integer", "Owner 2", False, _N, "", 0, None), + _F("F2", "number", "Fraction by owner 2", False, _N, "", 0.0, None), + _F("O3", "integer", "Owner 3", False, _N, "", 0, None), + _F("F3", "number", "Fraction by owner 3", False, _N, "", 0.0, None), + _F("O4", "integer", "Owner 4", False, _N, "", 0, None), + _F("F4", "number", "Fraction by owner 4", False, _N, "", 0.0, None), + # Converter 1 + _F("IBUS1", "integer", "Converter 1 AC bus", True, _N, "", None, None), + _F("TYPE1", "integer", "Converter 1 type", False, _N, "", 1, None), + _F("MODE1", "integer", "Converter 1 mode", False, _N, "", 1, None), + _F("DCSET1", "number", "Converter 1 DC setpoint", False, _N, "", 0.0, None), + _F("ACSET1", "number", "Converter 1 AC setpoint", False, _N, "", 1.0, None), + _F("ALOSS1", "number", "Converter 1 loss A", False, _N, "", 0.0, None), + _F("BLOSS1", "number", "Converter 1 loss B", False, _N, "", 0.0, None), + _F("MINLOSS1", "number", "Converter 1 min loss", False, _N, "", 0.0, None), + _F("SMAX1", "number", "Converter 1 MVA rating", False, _N, "MVA", 0.0, None), + _F("IMAX1", "number", "Converter 1 current rating", False, _N, "A", 0.0, None), + _F("PWF1", "number", "Converter 1 power weight", False, _N, "", 1.0, None), + _F("MAXQ1", "number", "Converter 1 max Q", False, _N, "MVAR", 9999.0, None), + _F("MINQ1", "number", "Converter 1 min Q", False, _N, "MVAR", -9999.0, None), + _F("REMOT1", "integer", "Converter 1 remote bus", False, _N, "", 0, None), + _F("RMPCT1", "number", "Converter 1 MVAR pct", False, _N, "%", 100.0, None), + # Converter 2 + _F("IBUS2", "integer", "Converter 2 AC bus", True, _N, "", None, None), + _F("TYPE2", "integer", "Converter 2 type", False, _N, "", 1, None), + _F("MODE2", "integer", "Converter 2 mode", False, _N, "", 1, None), + _F("DCSET2", "number", "Converter 2 DC setpoint", False, _N, "", 0.0, None), + _F("ACSET2", "number", "Converter 2 AC setpoint", False, _N, "", 1.0, None), + _F("ALOSS2", "number", "Converter 2 loss A", False, _N, "", 0.0, None), + _F("BLOSS2", "number", "Converter 2 loss B", False, _N, "", 0.0, None), + _F("MINLOSS2", "number", "Converter 2 min loss", False, _N, "", 0.0, None), + _F("SMAX2", "number", "Converter 2 MVA rating", False, _N, "MVA", 0.0, None), + _F("IMAX2", "number", "Converter 2 current rating", False, _N, "A", 0.0, None), + _F("PWF2", "number", "Converter 2 power weight", False, _N, "", 1.0, None), + _F("MAXQ2", "number", "Converter 2 max Q", False, _N, "MVAR", 9999.0, None), + _F("MINQ2", "number", "Converter 2 min Q", False, _N, "MVAR", -9999.0, None), + _F("REMOT2", "integer", "Converter 2 remote bus", False, _N, "", 0, None), + _F("RMPCT2", "number", "Converter 2 MVAR pct", False, _N, "%", 100.0, None), + ] + + +def _impedance_correction_fields() -> list[FieldSpec]: + fields: list[FieldSpec] = [ + _F("T", "integer", "Correction table number", True, _N, "", None, None), + ] + for i in range(1, 12): + fields.append(_F(f"T{i}", "number", f"Tap ratio/angle pair {i}", False, _N, "", 0.0, None)) + fields.append( + _F(f"F{i}", "number", f"Correction factor pair {i}", False, _N, "", 0.0, None) + ) + return fields + + +def _multi_terminal_dc_fields() -> list[FieldSpec]: + return [ + _F("NAME", "string", "MT DC line name", True, _N, "", None, None), + _F("NCONV", "integer", "Number of AC converters", True, _N, "", 0, None), + _F("NDCBS", "integer", "Number of DC buses", True, _N, "", 0, None), + _F("NDCLN", "integer", "Number of DC links", True, _N, "", 0, None), + _F("MDC", "integer", "Control mode", False, _N, "", 0, None), + _F("VCONV", "integer", "DC voltage ctrl converter", False, _N, "", 0, None), + _F("VCMOD", "number", "Mode switch DC voltage", False, _N, "", 0.0, None), + _F("VCONVN", "integer", "New voltage ctrl converter", False, _N, "", 0, None), + ] + + +def _multi_section_line_fields() -> list[FieldSpec]: + P = True # noqa: N806 + fields: list[FieldSpec] = [ + _F("I", "integer", "From bus number", True, _N, "", None, None, False, P), + _F("J", "integer", "To bus number", True, _N, "", None, None, False, P), + _F("ID", "string", "Line identifier", True, _N, "", "1 ", None, False, P), + _F("MET", "integer", "Metered end flag", False, _N, "", 1, (1, 2)), + ] + for i in range(1, 10): + fields.append( + _F(f"DUM{i}", "integer", f"Intermediate bus {i}", True, _N, "", 0, None, False, P) + ) + return fields + + +def _zone_fields() -> list[FieldSpec]: + return [ + _F("I", "integer", "Zone number", True, _N, "", None, None), + _F("ZONAME", "string", "Zone name (12 chars)", True, _N, "", _BLANK12, None), + ] + + +def _interarea_transfer_fields() -> list[FieldSpec]: + return [ + _F("ARFROM", "integer", "From area number", True, _N, "", None, None), + _F("ARTO", "integer", "To area number", True, _N, "", None, None), + _F("TRID", "string", "Transfer ID (2 chars)", True, _N, "", "1 ", None), + _F("PTRAN", "number", "Transfer amount", True, _N, "MW", 0.0, None), + ] + + +def _owner_fields() -> list[FieldSpec]: + return [ + _F("I", "integer", "Owner number", True, _N, "", None, None), + _F("OWNAME", "string", "Owner name (12 chars)", True, _N, "", _BLANK12, None), + ] + + +def _facts_fields() -> list[FieldSpec]: + return [ + _F("NAME", "string", "FACTS device name", True, _N, "", None, None), + _F("I", "integer", "Sending end bus", True, _N, "", None, None), + _F("J", "integer", "Terminal bus (0=shunt)", True, _N, "", 0, None), + _F("MODE", "integer", "FACTS control mode", True, _N, "", 1, None), + _F("SET1", "number", "Control setpoint 1", False, _N, "", 0.0, None), + _F("SET2", "number", "Control setpoint 2", False, _N, "", 0.0, None), + _F("VSREF", "number", "Series voltage reference", False, _B, "pu", 1.0, None), + _F("REMOT", "integer", "Remote bus for V control", False, _N, "", 0, None), + _F("MESSION", "number", "Sending end impedance", False, _N, "", 0.0, None), + _F("LINX", "number", "Series reactance", False, _S, "pu", 0.05, None), + _F("RMPCT", "number", "MVAR pct for remote reg", False, _N, "%", 100.0, None), + _F("OWNER", "integer", "Owner number", False, _N, "", 1, None), + _F("SET3", "number", "Control setpoint 3", False, _N, "", 0.0, None), + _F("SET4", "number", "Control setpoint 4", False, _N, "", 0.0, None), + ] + + +def _switched_shunt_fields() -> list[FieldSpec]: + P = True # noqa: N806 + fields: list[FieldSpec] = [ + _F("I", "integer", "Bus number", True, _N, "", None, None), + _F("MODSW", "integer", "Control mode (0-2)", True, _N, "", 1, (0, 2), False, P), + _F("ADJM", "integer", "Adj method (0-1)", False, _N, "", 0, (0, 1)), + _F("STAT", "integer", "Status (1=in, 0=out)", True, _N, "", 1, (0, 1)), + _F("VSWHI", "number", "Ctrl voltage upper limit", True, _B, "pu", 1.0, None), + _F("VSWLO", "number", "Ctrl voltage lower limit", True, _B, "pu", 1.0, None), + _F("SWREM", "integer", "Remote bus (0=local)", True, _N, "", 0, None, False, P), + _F("RMPCT", "number", "MVAR pct for remote reg", False, _N, "%", 100.0, None), + _F("RMIDNT", "string", "Shunt name", False, _N, "", "", None), + _F("BINIT", "number", "Initial susceptance", True, _N, "MVAR", 0.0, None, False, P), + ] + for i in range(1, 9): + fields.append( + _F(f"N{i}", "integer", f"Steps in block {i}", True, _N, "", 0, None, False, P) + ) + fields.append( + _F( + f"B{i}", + "number", + f"Susceptance/step blk {i}", + True, + _N, + "MVAR", + 0.0, + None, + False, + P, + ) + ) + return fields + + +# --------------------------------------------------------------------------- +# Table name derivation +# --------------------------------------------------------------------------- + + +def _record_type_to_table_name(record_type: str) -> str: + """Convert PSS/E record type name to table file name stem. + + Args: + record_type: PSS/E v31 record type name. + + Returns: + Lowercase, underscore-separated table name. + """ + return record_type.lower().replace(" ", "_").replace("-", "_") + + +# --------------------------------------------------------------------------- +# Schema generation +# --------------------------------------------------------------------------- + +_TABLE_FIELD_BUILDERS: dict[str, Callable[[], list[FieldSpec]]] = { + "Bus": _bus_fields, + "Load": _load_fields, + "Fixed Shunt": _fixed_shunt_fields, + "Generator": _generator_fields, + "Branch": _branch_fields, + "Transformer": _transformer_fields, + "Area": _area_fields, + "Two-Terminal DC": _two_terminal_dc_fields, + "VSC DC": _vsc_dc_fields, + "Impedance Correction": _impedance_correction_fields, + "Multi-Terminal DC": _multi_terminal_dc_fields, + "Multi-Section Line": _multi_section_line_fields, + "Zone": _zone_fields, + "Interarea Transfer": _interarea_transfer_fields, + "Owner": _owner_fields, + "FACTS": _facts_fields, + "Switched Shunt": _switched_shunt_fields, +} + +_TABLE_PRIMARY_KEYS: dict[str, list[str]] = { + "Bus": ["I"], + "Load": ["I", "ID"], + "Fixed Shunt": ["I", "ID"], + "Generator": ["I", "ID"], + "Branch": ["I", "J", "CKT"], + "Transformer": ["I", "J", "K", "CKT"], + "Area": ["I"], + "Two-Terminal DC": ["NAME"], + "VSC DC": ["NAME"], + "Impedance Correction": ["T"], + "Multi-Terminal DC": ["NAME"], + "Multi-Section Line": ["I", "J", "ID"], + "Zone": ["I"], + "Interarea Transfer": ["ARFROM", "ARTO", "TRID"], + "Owner": ["I"], + "FACTS": ["NAME"], + "Switched Shunt": ["I"], +} + +_TABLE_DESCRIPTIONS: dict[str, str] = { + "Bus": ("PSS/E v31 Bus record type. Each row represents one bus in the network model."), + "Load": ("PSS/E v31 Load record type. Each row represents one load at a bus."), + "Fixed Shunt": ( + "PSS/E v31 Fixed Shunt record type. Each row represents a fixed shunt element." + ), + "Generator": ("PSS/E v31 Generator record type. Each row represents one generating unit."), + "Branch": ("PSS/E v31 Branch record type. Each row represents a transmission line or cable."), + "Transformer": ( + "PSS/E v31 Transformer record type. Multi-line records " + "flattened into one row. 2-winding (K=0) and 3-winding " + "(K!=0) share the same schema." + ), + "Area": ("PSS/E v31 Area record type. Each row defines an area for interchange control."), + "Two-Terminal DC": ("PSS/E v31 Two-Terminal DC line record type."), + "VSC DC": "PSS/E v31 VSC DC line record type.", + "Impedance Correction": ("PSS/E v31 Impedance Correction table record type."), + "Multi-Terminal DC": ("PSS/E v31 Multi-Terminal DC line header record type."), + "Multi-Section Line": ( + "PSS/E v31 Multi-Section Line grouping record type. DUM fields define intermediate buses." + ), + "Zone": "PSS/E v31 Zone record type.", + "Interarea Transfer": ("PSS/E v31 Interarea Transfer record type."), + "Owner": "PSS/E v31 Owner record type.", + "FACTS": "PSS/E v31 FACTS device record type.", + "Switched Shunt": ( + "PSS/E v31 Switched Shunt record type. N1-N8 and " + "B1-B8 define discrete switching step blocks." + ), +} + +_MULTI_LINE_TYPES = {"Transformer", "Multi-Terminal DC"} + + +def get_table_schemas() -> list[TableSchema]: + """Return table schemas for all 17 PSS/E v31 record types. + + This is the master field inventory. Only tables whose record type + appears in the non-empty sections list (from D3) will get JSON + Schema files written. + + Returns: + List of TableSchema in PSS/E v31 section order. + """ + schemas: list[TableSchema] = [] + for rt in PSSE_V31_RECORD_TYPES: + builder = _TABLE_FIELD_BUILDERS[rt] + schemas.append( + TableSchema( + record_type=rt, + table_name=_record_type_to_table_name(rt), + description=_TABLE_DESCRIPTIONS[rt], + fields=builder(), + primary_key=_TABLE_PRIMARY_KEYS[rt], + multi_line_record=rt in _MULTI_LINE_TYPES, + ) + ) + return schemas + + +def table_schema_to_json_schema(table: TableSchema) -> dict: + """Convert a TableSchema to a JSON Schema Draft 2020-12 document. + + Args: + table: The table schema to convert. + + Returns: + A dict that is a valid JSON Schema Draft 2020-12 document. + """ + properties: dict[str, dict] = {} + required_fields: list[str] = [] + + for f in table.fields: + prop: dict = { + "type": f.data_type, + "description": f.description, + "x-psse-unit": f.unit, + "x-psse-per-unit-base": f.per_unit_base.value, + "x-psse-default": f.default_value, + "x-psse-preservation-critical": f.preservation_critical, + "x-psse-present-but-inactive": f.present_but_inactive, + } + if f.valid_range is not None: + prop["x-psse-valid-range"] = list(f.valid_range) + else: + prop["x-psse-valid-range"] = None + + properties[f.name] = prop + if f.required: + required_fields.append(f.name) + + return { + "$schema": ("https://json-schema.org/draft/2020-12/schema"), + "$id": f"schemas/{table.table_name}.schema.json", + "title": table.record_type, + "description": table.description, + "type": "object", + "properties": properties, + "required": required_fields, + "additionalProperties": False, + } + + +def manifest_to_json_schema() -> dict: + """Return the JSON Schema for the intermediate format manifest. + + Returns: + A dict that is a valid JSON Schema Draft 2020-12 document. + """ + return { + "$schema": ("https://json-schema.org/draft/2020-12/schema"), + "$id": "schemas/manifest.schema.json", + "title": "Intermediate Format Manifest", + "description": ( + "Top-level manifest listing all tables in the intermediate format with metadata." + ), + "type": "object", + "properties": { + "sbase": { + "type": "number", + "description": "System MVA base", + }, + "basfrq": { + "type": "number", + "description": "System base frequency (Hz)", + }, + "rev": { + "type": "number", + "description": "PSS/E revision number", + }, + "case_id": { + "type": "string", + "description": "Case identification string", + }, + "canonical_parser": { + "type": "string", + "enum": ["matpower", "gridcal"], + "description": "Canonical parser from D6", + }, + "tables": { + "type": "array", + "items": { + "type": "object", + "properties": { + "table_name": {"type": "string"}, + "record_type": {"type": "string"}, + "file_name": {"type": "string"}, + "record_count": { + "type": "integer", + "minimum": 0, + }, + "column_count": { + "type": "integer", + "minimum": 1, + }, + "schema_file": {"type": "string"}, + }, + "required": [ + "table_name", + "record_type", + "file_name", + "record_count", + "column_count", + "schema_file", + ], + }, + }, + "total_records": { + "type": "integer", + "minimum": 0, + }, + "total_tables": { + "type": "integer", + "minimum": 1, + }, + "non_empty_record_types": { + "type": "array", + "items": {"type": "string"}, + }, + "schema_version": {"type": "string"}, + "generated_timestamp": { + "type": "string", + "format": "date-time", + }, + }, + "required": [ + "sbase", + "basfrq", + "rev", + "case_id", + "canonical_parser", + "tables", + "total_records", + "total_tables", + "non_empty_record_types", + "schema_version", + "generated_timestamp", + ], + "additionalProperties": False, + } + + +# --------------------------------------------------------------------------- +# Schema I/O +# --------------------------------------------------------------------------- + + +def write_schemas( + output_dir: Path, + non_empty_types: list[str], + present_but_inactive: dict[str, list[str]] | None = None, +) -> list[Path]: + """Write JSON Schema files for non-empty record types + manifest. + + Args: + output_dir: Root directory for schema output. + non_empty_types: PSS/E record type names with non-zero count. + present_but_inactive: Optional dict mapping record type to + list of field names uniformly at default values. + + Returns: + List of paths to written schema files. + + Raises: + ValueError: If a record type is not recognized. + """ + all_schemas = {ts.record_type: ts for ts in get_table_schemas()} + + for rt in non_empty_types: + if rt not in all_schemas: + msg = f"Unknown record type: {rt!r}" + raise ValueError(msg) + + schema_dir = output_dir / "schemas" + schema_dir.mkdir(parents=True, exist_ok=True) + + written: list[Path] = [] + + for rt in non_empty_types: + ts = all_schemas[rt] + + # Apply present_but_inactive annotations + if present_but_inactive and rt in present_but_inactive: + inactive_names = set(present_but_inactive[rt]) + updated = [] + for f in ts.fields: + if f.name in inactive_names: + updated.append( + FieldSpec( + name=f.name, + data_type=f.data_type, + description=f.description, + required=f.required, + per_unit_base=f.per_unit_base, + unit=f.unit, + default_value=f.default_value, + valid_range=f.valid_range, + present_but_inactive=True, + preservation_critical=(f.preservation_critical), + ) + ) + else: + updated.append(f) + ts = TableSchema( + record_type=ts.record_type, + table_name=ts.table_name, + description=ts.description, + fields=updated, + primary_key=ts.primary_key, + multi_line_record=ts.multi_line_record, + notes=ts.notes, + ) + + schema_dict = table_schema_to_json_schema(ts) + path = schema_dir / f"{ts.table_name}.schema.json" + path.write_text( + json.dumps(schema_dict, indent=2) + "\n", + encoding="utf-8", + ) + written.append(path) + + # Write manifest schema + manifest_schema = manifest_to_json_schema() + manifest_path = schema_dir / "manifest.schema.json" + manifest_path.write_text( + json.dumps(manifest_schema, indent=2) + "\n", + encoding="utf-8", + ) + written.append(manifest_path) + + return written + + +def load_schema(schema_path: Path) -> dict: + """Load a JSON Schema file and return the parsed dict. + + Args: + schema_path: Path to a .schema.json file. + + Returns: + The parsed JSON Schema dict. + + Raises: + FileNotFoundError: If the file does not exist. + json.JSONDecodeError: If the file is not valid JSON. + """ + if not schema_path.exists(): + raise FileNotFoundError(f"Schema file not found: {schema_path}") + return json.loads(schema_path.read_text(encoding="utf-8")) + + +# --------------------------------------------------------------------------- +# present-but-inactive detection +# --------------------------------------------------------------------------- + + +def detect_inactive_fields( + csv_dir: Path, + non_empty_types: list[str], + record_type_to_table_name: dict[str, str], +) -> dict[str, list[str]]: + """Scan CSV exports to find fields uniformly at default values. + + Args: + csv_dir: Directory containing canonical parser CSV exports. + non_empty_types: PSS/E record type names to check. + record_type_to_table_name: Mapping from PSS/E record type + name to the CSV file stem. + + Returns: + Dict mapping record type name to list of inactive fields. + """ + all_schemas = {ts.record_type: ts for ts in get_table_schemas()} + result: dict[str, list[str]] = {} + + for rt in non_empty_types: + if rt not in all_schemas: + continue + ts = all_schemas[rt] + csv_stem = record_type_to_table_name.get(rt) + if csv_stem is None: + continue + + csv_path = csv_dir / f"{csv_stem}.csv" + if not csv_path.exists(): + continue + + try: + with open(csv_path, encoding="utf-8") as f: + reader = csv.DictReader(f) + rows = list(reader) + except (OSError, csv.Error): + continue + + if not rows: + continue + + # Build field-name -> default mapping + field_defaults: dict[str, int | float | str | None] = {} + for fs in ts.fields: + if fs.default_value is not None: + field_defaults[fs.name] = fs.default_value + + inactive: list[str] = [] + for fname, default_val in field_defaults.items(): + if fname not in rows[0]: + continue + + all_default = True + for row in rows: + val = row.get(fname, "") + try: + if isinstance(default_val, int): + if int(float(val)) != default_val: + all_default = False + break + elif isinstance(default_val, float): + if abs(float(val) - default_val) > 1e-10: + all_default = False + break + else: + if str(val).strip() != str(default_val).strip(): + all_default = False + break + except (ValueError, TypeError): + all_default = False + break + + if all_default: + inactive.append(fname) + + if inactive: + result[rt] = sorted(inactive) + + return result + + +# --------------------------------------------------------------------------- +# Validation +# --------------------------------------------------------------------------- + + +def validate_tables( + csv_dir: Path, + schema_dir: Path, + manifest_path: Path, +) -> ConformanceReport: + """Validate intermediate format CSV tables against the schema. + + Args: + csv_dir: Directory containing CSV tables. + schema_dir: Directory containing JSON Schema files. + manifest_path: Path to the manifest JSON. + + Returns: + A ConformanceReport summarizing all findings. + """ + errors: list[ConformanceFinding] = [] + warnings: list[ConformanceFinding] = [] + info: list[ConformanceFinding] = [] + manifest_valid = True + + # 1. Load manifest + try: + manifest_data = json.loads(manifest_path.read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError) as e: + errors.append( + ConformanceFinding( + "__manifest__", + "", + ConformanceLevel.ERROR, + f"Cannot load manifest: {e}", + "manifest_load_error", + ) + ) + return ConformanceReport( + tables_checked=0, + tables_expected=0, + errors=errors, + warnings=warnings, + info=info, + is_conformant=False, + manifest_valid=False, + ) + + # Check manifest against its schema + manifest_schema_path = schema_dir / "manifest.schema.json" + if manifest_schema_path.exists(): + try: + ms = load_schema(manifest_schema_path) + _validate_against_schema(manifest_data, ms) + except Exception as e: + manifest_valid = False + errors.append( + ConformanceFinding( + "__manifest__", + "", + ConformanceLevel.ERROR, + f"Manifest schema validation failed: {e}", + "manifest_schema_invalid", + ) + ) + + tables_list = manifest_data.get("tables", []) + tables_expected = len(tables_list) + tables_checked = 0 + + all_schemas = {ts.table_name: ts for ts in get_table_schemas()} + + for table_entry in tables_list: + tname = table_entry.get("table_name", "") + fname = table_entry.get("file_name", "") + expected_count = table_entry.get("record_count", 0) + + # 2. Table presence + csv_path = csv_dir / fname + if not csv_path.exists(): + errors.append( + ConformanceFinding( + tname, + "", + ConformanceLevel.ERROR, + f"CSV file not found: {fname}", + "missing_table", + ) + ) + continue + + tables_checked += 1 + + # Load schema + spath = schema_dir / f"{tname}.schema.json" + if not spath.exists(): + warnings.append( + ConformanceFinding( + tname, + "", + ConformanceLevel.WARNING, + f"No schema: {tname}.schema.json", + "missing_schema", + ) + ) + continue + + schema = load_schema(spath) + required_fields = schema.get("required", []) + schema_props = schema.get("properties", {}) + + # Read CSV + try: + with open(csv_path, encoding="utf-8") as fh: + reader = csv.DictReader(fh) + csv_headers = reader.fieldnames or [] + rows = list(reader) + except (OSError, csv.Error) as e: + errors.append( + ConformanceFinding( + tname, + "", + ConformanceLevel.ERROR, + f"Cannot read CSV: {e}", + "csv_read_error", + ) + ) + continue + + # 3. Required fields + hdr_set = set(csv_headers) + for rf in required_fields: + if rf not in hdr_set: + errors.append( + ConformanceFinding( + tname, + rf, + ConformanceLevel.ERROR, + f"Required field '{rf}' missing", + "missing_required_field", + ) + ) + + # 4. Extra columns + schema_set = set(schema_props.keys()) + for col in csv_headers: + if col not in schema_set: + info.append( + ConformanceFinding( + tname, + col, + ConformanceLevel.INFO, + f"Extra column '{col}'", + "extra_column", + ) + ) + + # 5. Type spot-check (first 100 rows) + check_rows = rows[:100] + for cname, cschema in schema_props.items(): + if cname not in hdr_set: + continue + ctype = cschema.get("type", "string") + for ridx, row in enumerate(check_rows): + val = row.get(cname, "") + if val == "" or val is None: + continue + if ctype == "integer": + try: + int(float(val)) + except (ValueError, TypeError): + warnings.append( + ConformanceFinding( + tname, + cname, + ConformanceLevel.WARNING, + f"Row {ridx}: '{val}' not int", + "type_mismatch", + ) + ) + break + elif ctype == "number": + try: + float(val) + except (ValueError, TypeError): + warnings.append( + ConformanceFinding( + tname, + cname, + ConformanceLevel.WARNING, + f"Row {ridx}: '{val}' not num", + "type_mismatch", + ) + ) + break + + # 6. Record count check + actual = len(rows) + if actual != expected_count: + info.append( + ConformanceFinding( + tname, + "", + ConformanceLevel.INFO, + (f"Count: manifest={expected_count}, CSV={actual}"), + "record_count_mismatch", + ) + ) + + # 7. Preservation-critical fields + ts = all_schemas.get(tname) + if ts: + for fs in ts.fields: + if not fs.preservation_critical: + continue + if fs.name not in hdr_set: + errors.append( + ConformanceFinding( + tname, + fs.name, + ConformanceLevel.ERROR, + (f"Preservation-critical '{fs.name}' missing"), + "missing_preservation_critical", + ) + ) + elif rows: + all_null = all(row.get(fs.name, "") in ("", None) for row in rows) + if all_null: + warnings.append( + ConformanceFinding( + tname, + fs.name, + ConformanceLevel.WARNING, + (f"'{fs.name}' null in all records"), + "preservation_critical_all_null", + ) + ) + + return ConformanceReport( + tables_checked=tables_checked, + tables_expected=tables_expected, + errors=errors, + warnings=warnings, + info=info, + is_conformant=len(errors) == 0, + manifest_valid=manifest_valid, + ) + + +def _validate_against_schema(data: dict, schema: dict) -> None: + """Lightweight required-properties check. Raises on failure.""" + required = schema.get("required", []) + for req in required: + if req not in data: + msg = f"Missing required property: {req}" + raise ValueError(msg) + + +def report_to_dict(report: ConformanceReport) -> dict: + """Convert a ConformanceReport to a JSON-serializable dict. + + Args: + report: The conformance report to serialize. + + Returns: + A dict safe for json.dumps(). + """ + + def _f2d(f: ConformanceFinding) -> dict: + return { + "table_name": f.table_name, + "field_name": f.field_name, + "level": f.level.value, + "message": f.message, + "check_id": f.check_id, + } + + return { + "tables_checked": report.tables_checked, + "tables_expected": report.tables_expected, + "errors": [_f2d(f) for f in report.errors], + "warnings": [_f2d(f) for f in report.warnings], + "info": [_f2d(f) for f in report.info], + "is_conformant": report.is_conformant, + "manifest_valid": report.manifest_valid, + } + + +# --------------------------------------------------------------------------- +# Markdown generation +# --------------------------------------------------------------------------- + + +def generate_reference_markdown( + non_empty_types: list[str], + present_but_inactive: dict[str, list[str]] | None = None, +) -> str: + """Generate human-readable intermediate format reference. + + Args: + non_empty_types: PSS/E record type names to include. + present_but_inactive: Optional dict of inactive fields. + + Returns: + A complete markdown string. + """ + all_schemas = {ts.record_type: ts for ts in get_table_schemas()} + + lines: list[str] = [ + "# Intermediate Format Reference", + "", + "## Schema Version", + "", + "1.0.0", + "", + "## System Metadata", + "", + "- **SBASE**: System MVA base (from PSS/E header)", + "- **BASFRQ**: Base frequency in Hz", + "- **REV**: PSS/E revision number (31.x)", + "", + "## Per-Unit Convention Reference", + "", + "| Base | Description |", + "|------|-------------|", + "| system_mva | System MVA base (SBASE) |", + "| winding_mva | Winding-specific MVA base |", + "| bus_kv | Bus base voltage (BASKV) |", + "| none | Not a per-unit quantity |", + "| mixed | Depends on mode code (CW/CZ/CM) |", + "", + "## Record Type Tables", + "", + ] + + inactive_all: dict[str, set[str]] = {} + if present_but_inactive: + for rt, fields in present_but_inactive.items(): + inactive_all[rt] = set(fields) + + preservation_summary: list[tuple[str, str, str]] = [] + + for rt in non_empty_types: + ts = all_schemas.get(rt) + if ts is None: + continue + + lines.append(f"### {rt}") + lines.append("") + lines.append(ts.description) + lines.append("") + lines.append(f"**Table name:** `{ts.table_name}`") + lines.append(f"**Primary key:** `{ts.primary_key}`") + if ts.multi_line_record: + lines.append("**Multi-line record:** Yes") + lines.append("") + + # Field table + lines.append("| Field | Type | Unit | PU Base | Req | Description |") + lines.append("|-------|------|------|--------|-----|-------------|") + rt_inactive = inactive_all.get(rt, set()) + for f in ts.fields: + req = "Y" if f.required else "N" + desc = f.description + if f.preservation_critical: + desc = f"**[P]** {desc}" + preservation_summary.append((rt, f.name, f.description)) + if f.name in rt_inactive: + desc = f"{desc} *(inactive)*" + unit = f.unit or "---" + pub = f.per_unit_base.value + lines.append(f"| {f.name} | {f.data_type} | {unit} | {pub} | {req} | {desc} |") + + lines.append("") + if ts.notes: + lines.append(f"**Notes:** {ts.notes}") + lines.append("") + + # Appendix A: preservation + lines.append("## Appendix A: Preservation Requirements Summary") + lines.append("") + if preservation_summary: + lines.append("| Record Type | Field | Description |") + lines.append("|-------------|-------|-------------|") + for rt, fname, desc in preservation_summary: + lines.append(f"| {rt} | {fname} | {desc} |") + else: + lines.append("No preservation-critical fields.") + lines.append("") + + # Appendix B: inactive + lines.append("## Appendix B: Present-But-Inactive Fields") + lines.append("") + if inactive_all: + for rt, inactive_set in sorted(inactive_all.items()): + lines.append(f"### {rt}") + lines.append("") + for fname in sorted(inactive_set): + lines.append(f"- `{fname}`") + lines.append("") + else: + lines.append("No inactive fields detected.") + lines.append("") + + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> None: + """CLI entry point for schema generation and validation. + + Args: + argv: Command-line arguments. None reads sys.argv[1:]. + """ + parser = argparse.ArgumentParser(description="Intermediate Format Schema tools.") + sub = parser.add_subparsers(dest="command", required=True) + + # Generate + gen = sub.add_parser("generate", help="Generate schemas") + gen.add_argument( + "--raw-summary", + required=True, + help="Path to D3 raw counts JSON", + ) + gen.add_argument( + "--canonical-csvs", + default=None, + help="Canonical parser CSV directory", + ) + gen.add_argument( + "--parser-mapping", + default=None, + help="Record type mapping JSON", + ) + gen.add_argument( + "-o", + "--output", + default="data/fnm/intermediate", + help="Output directory", + ) + + # Validate + val = sub.add_parser("validate", help="Validate tables") + val.add_argument( + "--csv-dir", + required=True, + help="CSV tables directory", + ) + val.add_argument( + "--schema-dir", + required=True, + help="JSON Schema directory", + ) + val.add_argument( + "--manifest", + required=True, + help="Manifest JSON path", + ) + val.add_argument( + "-o", + "--output", + default=None, + help="Output directory", + ) + + args = parser.parse_args(argv) + + if args.command == "generate": + _cmd_generate(args) + elif args.command == "validate": + _cmd_validate(args) + + +def _cmd_generate(args: argparse.Namespace) -> None: + """Execute the 'generate' subcommand.""" + raw_path = Path(args.raw_summary) + output_dir = Path(args.output) + + raw_data = json.loads(raw_path.read_text(encoding="utf-8")) + non_empty = raw_data.get("non_empty_sections", []) + + inactive: dict[str, list[str]] | None = None + if args.canonical_csvs: + csv_dir = Path(args.canonical_csvs) + rt_to_tn: dict[str, str] = {rt: _record_type_to_table_name(rt) for rt in non_empty} + if args.parser_mapping: + mapping = json.loads(Path(args.parser_mapping).read_text(encoding="utf-8")) + rt_to_tn.update(mapping) + inactive = detect_inactive_fields(csv_dir, non_empty, rt_to_tn) + + written = write_schemas(output_dir, non_empty, inactive) + schema_dir = output_dir / "schemas" + print(f"Wrote {len(written)} schemas to {schema_dir}") + + md = generate_reference_markdown(non_empty, inactive) + md_path = output_dir / "intermediate_format_reference.md" + md_path.write_text(md, encoding="utf-8") + print(f"Wrote reference to {md_path}") + + +def _cmd_validate(args: argparse.Namespace) -> None: + """Execute the 'validate' subcommand.""" + csv_dir = Path(args.csv_dir) + schema_dir = Path(args.schema_dir) + manifest_path = Path(args.manifest) + + report = validate_tables(csv_dir, schema_dir, manifest_path) + result = report_to_dict(report) + + out = Path(args.output) if args.output else manifest_path.parent + out.mkdir(parents=True, exist_ok=True) + rpath = out / "conformance_report.json" + rpath.write_text( + json.dumps(result, indent=2) + "\n", + encoding="utf-8", + ) + print(f"Wrote conformance report to {rpath}") + + status = "CONFORMANT" if report.is_conformant else "NON-CONFORMANT" + print( + f"Result: {status} " + f"(errors={len(report.errors)}, " + f"warnings={len(report.warnings)}, " + f"info={len(report.info)})" + ) + + +if __name__ == "__main__": + main() diff --git a/data/fnm/scripts/manifest_io.py b/data/fnm/scripts/manifest_io.py new file mode 100644 index 00000000..371be0a0 --- /dev/null +++ b/data/fnm/scripts/manifest_io.py @@ -0,0 +1,336 @@ +"""Manifest I/O module for FNM source file management. + +Provides typed data structures and functions for loading, validating, and updating +the FNM manifest that enumerates all expected source files (PSS/E RAW + supplemental CSVs). +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from enum import Enum +from pathlib import Path + + +class SourceFileType(Enum): + """Type of FNM source file.""" + + PSSE_RAW = "psse_raw" + SUPPLEMENTAL_CSV = "supplemental_csv" + + +@dataclass(frozen=True) +class SourceFileEntry: + """A single expected FNM source file entry in the manifest. + + Attributes: + file_name: Base filename of the source file. + file_type: Whether this is a PSS/E RAW file or a supplemental CSV. + description: Human-readable description of the file's contents. + sha256: SHA-256 checksum of the file, or None if not yet computed. + required: Whether this file is required for a valid FNM dataset. + """ + + file_name: str + file_type: SourceFileType + description: str + sha256: str | None = None + required: bool = True + + +@dataclass(frozen=True) +class FnmManifest: + """Machine-readable manifest of all expected FNM source files. + + Attributes: + version: Manifest schema version string. + variant: FNM variant identifier (e.g. "FNM_ANNUAL_S01"). + source_files: List of expected source file entries. + notes: Optional free-text notes about this manifest. + """ + + version: str + variant: str + source_files: list[SourceFileEntry] + notes: str = "" + + +@dataclass(frozen=True) +class ManifestValidationResult: + """Result of validating a manifest against files on disk. + + Attributes: + manifest_path: Path to the manifest JSON file. + fnm_path: Path to the directory containing FNM source files. + found: List of filenames that were found on disk. + missing: List of filenames that were expected but not found. + checksum_mismatches: List of filenames with SHA-256 mismatches. + is_valid: True if all required files are present and checksums match. + """ + + manifest_path: Path + fnm_path: Path + found: list[str] + missing: list[str] + checksum_mismatches: list[str] + is_valid: bool + + +FNM_ROOT = Path("data/fnm") +FNM_SUBDIRS: list[str] = ["intermediate", "reference", "docs", "scripts"] + +GITIGNORE_BLOCKED_PATTERNS: list[str] = [ + "*.raw", + "*.RAW", + "*.csv", + "*.CSV", + "*.parquet", + "*.m", + "intermediate/**", + "reference/**", +] + +GITIGNORE_ALLOWED_PATTERNS: list[str] = [ + "!manifest.json", + "!README.md", + "!**/README.md", + "!scripts/**/*.py", + "!docs/**/*.md", + "!docs/**/*.json", + "!.gitignore", +] + + +def _source_file_entry_to_dict(entry: SourceFileEntry) -> dict: + """Convert a SourceFileEntry to a JSON-serializable dict.""" + return { + "file_name": entry.file_name, + "file_type": entry.file_type.value, + "description": entry.description, + "sha256": entry.sha256, + "required": entry.required, + } + + +def _dict_to_source_file_entry(d: dict) -> SourceFileEntry: + """Convert a dict to a SourceFileEntry.""" + return SourceFileEntry( + file_name=d["file_name"], + file_type=SourceFileType(d["file_type"]), + description=d["description"], + sha256=d.get("sha256"), + required=d.get("required", True), + ) + + +def build_default_manifest() -> FnmManifest: + """Build the default FNM manifest with all expected source files. + + Returns: + An FnmManifest with entries for the PSS/E RAW file and 7 supplemental CSVs, + all with placeholder (None) SHA-256 checksums. + """ + source_files = [ + SourceFileEntry( + file_name="FNM_ANNUAL_S01.raw", + file_type=SourceFileType.PSSE_RAW, + description="PSS/E v31 RAW file containing the full network model", + ), + SourceFileEntry( + file_name="bus_names.csv", + file_type=SourceFileType.SUPPLEMENTAL_CSV, + description="Bus name mapping with station and voltage level metadata", + ), + SourceFileEntry( + file_name="branch_ratings.csv", + file_type=SourceFileType.SUPPLEMENTAL_CSV, + description="Branch thermal rating overrides and seasonal limits", + ), + SourceFileEntry( + file_name="generator_costs.csv", + file_type=SourceFileType.SUPPLEMENTAL_CSV, + description="Generator cost curves and fuel type classifications", + ), + SourceFileEntry( + file_name="load_distribution.csv", + file_type=SourceFileType.SUPPLEMENTAL_CSV, + description="Load distribution factors by weather zone and bus", + ), + SourceFileEntry( + file_name="transformer_taps.csv", + file_type=SourceFileType.SUPPLEMENTAL_CSV, + description="Transformer tap position settings and regulation bands", + ), + SourceFileEntry( + file_name="shunt_switching.csv", + file_type=SourceFileType.SUPPLEMENTAL_CSV, + description="Switched shunt device status and control parameters", + ), + SourceFileEntry( + file_name="contingency_definitions.csv", + file_type=SourceFileType.SUPPLEMENTAL_CSV, + description="Contingency definitions for N-1 and N-2 analysis", + ), + ] + return FnmManifest( + version="1.0", + variant="FNM_ANNUAL_S01", + source_files=source_files, + notes="Default manifest for FNM ingestion. SHA-256 checksums are populated " + "after first successful parse via update_manifest_checksums().", + ) + + +def load_manifest(manifest_path: Path) -> FnmManifest: + """Load an FNM manifest from a JSON file. + + Args: + manifest_path: Path to the manifest JSON file. + + Returns: + The deserialized FnmManifest. + + Raises: + FileNotFoundError: If manifest_path does not exist. + ValueError: If the JSON is malformed or missing required fields. + """ + if not manifest_path.exists(): + raise FileNotFoundError(f"Manifest not found: {manifest_path}") + + try: + text = manifest_path.read_text(encoding="utf-8") + data = json.loads(text) + except json.JSONDecodeError as e: + raise ValueError(f"Malformed JSON in manifest: {e}") from e + + try: + source_files = [_dict_to_source_file_entry(sf) for sf in data["source_files"]] + return FnmManifest( + version=data["version"], + variant=data["variant"], + source_files=source_files, + notes=data.get("notes", ""), + ) + except (KeyError, TypeError) as e: + raise ValueError(f"Invalid manifest structure: {e}") from e + + +def save_manifest(manifest: FnmManifest, manifest_path: Path) -> None: + """Save an FNM manifest to a JSON file. + + Args: + manifest: The manifest to serialize. + manifest_path: Destination path for the JSON file. + """ + data = { + "version": manifest.version, + "variant": manifest.variant, + "source_files": [_source_file_entry_to_dict(sf) for sf in manifest.source_files], + "notes": manifest.notes, + } + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + + +def validate_manifest_against_disk( + manifest: FnmManifest, + fnm_path: Path, + *, + verify_checksums: bool = False, +) -> ManifestValidationResult: + """Validate that all manifest source files exist on disk. + + Args: + manifest: The manifest to validate. + fnm_path: Directory containing the FNM source files. + verify_checksums: If True, also verify SHA-256 checksums for files that have them. + + Returns: + A ManifestValidationResult with found/missing/mismatch details. + """ + found: list[str] = [] + missing: list[str] = [] + checksum_mismatches: list[str] = [] + + for entry in manifest.source_files: + file_path = fnm_path / entry.file_name + if file_path.exists(): + found.append(entry.file_name) + if verify_checksums and entry.sha256 is not None: + actual = compute_file_sha256(file_path) + if actual != entry.sha256: + checksum_mismatches.append(entry.file_name) + else: + if entry.required: + missing.append(entry.file_name) + + is_valid = len(missing) == 0 and len(checksum_mismatches) == 0 + + return ManifestValidationResult( + manifest_path=Path("manifest.json"), + fnm_path=fnm_path, + found=found, + missing=missing, + checksum_mismatches=checksum_mismatches, + is_valid=is_valid, + ) + + +def compute_file_sha256(file_path: Path) -> str: + """Compute the SHA-256 hex digest of a file. + + Args: + file_path: Path to the file to hash. + + Returns: + Lowercase hex string of the SHA-256 digest. + + Raises: + FileNotFoundError: If file_path does not exist. + """ + if not file_path.exists(): + raise FileNotFoundError(f"File not found: {file_path}") + + h = hashlib.sha256() + with open(file_path, "rb") as f: + for chunk in iter(lambda: f.read(8192), b""): + h.update(chunk) + return h.hexdigest() + + +def update_manifest_checksums(manifest: FnmManifest, fnm_path: Path) -> FnmManifest: + """Compute and populate SHA-256 checksums for all source files found on disk. + + Files that do not exist on disk retain their current checksum value (typically None). + + Args: + manifest: The manifest to update. + fnm_path: Directory containing the FNM source files. + + Returns: + A new FnmManifest with checksums populated for files that exist. + """ + updated_files: list[SourceFileEntry] = [] + for entry in manifest.source_files: + file_path = fnm_path / entry.file_name + if file_path.exists(): + sha = compute_file_sha256(file_path) + updated_files.append( + SourceFileEntry( + file_name=entry.file_name, + file_type=entry.file_type, + description=entry.description, + sha256=sha, + required=entry.required, + ) + ) + else: + updated_files.append(entry) + + return FnmManifest( + version=manifest.version, + variant=manifest.variant, + source_files=updated_files, + notes=manifest.notes, + ) diff --git a/data/fnm/scripts/matpower_parser.py b/data/fnm/scripts/matpower_parser.py new file mode 100644 index 00000000..4a1a5a10 --- /dev/null +++ b/data/fnm/scripts/matpower_parser.py @@ -0,0 +1,571 @@ +"""MATPOWER psse2mpc parser wrapper. + +Orchestrates Octave/MATPOWER's ``psse2mpc`` function to convert a PSS/E RAW file +into MATPOWER case struct CSV exports, then inspects the results and records +parser warnings, known limitations, and field-level record counts. +""" + +from __future__ import annotations + +import json +import re +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path + +from fnm.scripts.fnm_gating import find_repo_root + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +MPC_FIELD_COLUMNS: dict[str, int] = { + "bus": 13, + "gen": 21, + "branch": 13, + "gencost": 7, # minimum; variable-width cost models may have more + "areas": 2, + "dcline": 17, + "bus_name": 1, +} + +MPC_DROPPED_RECORD_TYPES: tuple[str, ...] = ( + "Two-Terminal DC", + "VSC DC", + "Multi-Terminal DC", + "Multi-Section Line", + "Impedance Correction", + "FACTS", +) + +MPC_LOSSY_RECORD_TYPES: tuple[str, ...] = ( + "Switched Shunt", + "Fixed Shunt", + "Transformer", +) + +# --------------------------------------------------------------------------- +# Type aliases +# --------------------------------------------------------------------------- + +SectionCountMap = dict[str, int] + +# --------------------------------------------------------------------------- +# Data classes +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ParserWarning: + """A single warning emitted by psse2mpc during conversion. + + Attributes: + line: Raw warning text from stderr. + category: Classification of the warning (e.g. ``skipped_record``, + ``phantom_bus``, ``unsupported_field``, ``conversion_warning``). + """ + + line: str + category: str + + +@dataclass(frozen=True) +class KnownLimitation: + """A known limitation of MATPOWER's PSS/E parser. + + Attributes: + record_type: PSS/E section name (e.g. ``"Two-Terminal DC"``). + behavior: Whether this record type is ``"dropped"`` or ``"lossy"``. + description: Human-readable explanation. + """ + + record_type: str + behavior: str + description: str + + +@dataclass +class MatpowerParserLog: + """Full log from a single psse2mpc invocation. + + Attributes: + raw_path: Path to the input PSS/E RAW file. + output_dir: Directory where CSV exports were written. + return_code: Octave process exit code. + stdout: Raw stdout from the Octave process. + stderr: Raw stderr from the Octave process. + baseMVA: Parsed base MVA value, or None. + version: Parsed MATPOWER case version string, or None. + field_counts_octave: Record counts reported by Octave stdout. + field_counts_csv: Record counts measured from written CSV files. + warnings: Classified parser warnings. + """ + + raw_path: str + output_dir: str + return_code: int + stdout: str + stderr: str + baseMVA: float | None = None + version: str | None = None + field_counts_octave: SectionCountMap = field(default_factory=dict) + field_counts_csv: SectionCountMap = field(default_factory=dict) + warnings: list[ParserWarning] = field(default_factory=list) + + +@dataclass +class MatpowerParserSummary: + """High-level summary combining parser log with known limitations. + + Attributes: + log: The underlying parser log. + known_limitations: List of known MATPOWER parser limitations. + success: Whether the conversion completed without error. + """ + + log: MatpowerParserLog + known_limitations: list[KnownLimitation] + success: bool + + +# --------------------------------------------------------------------------- +# Functions +# --------------------------------------------------------------------------- + + +def find_matpower_path() -> Path | None: + """Search for the MATPOWER installation directory. + + Checks (in order): + 1. ``MATPOWER_PATH`` environment variable. + 2. ``evaluations/matpower/matpower8.1`` relative to repo root. + 3. ``/workspace/evaluations/matpower/matpower8.1`` (container default). + + Returns: + Path to the MATPOWER installation, or None if not found. + """ + import os + + # 1. Environment variable + env_path = os.environ.get("MATPOWER_PATH") + if env_path: + p = Path(env_path) + if p.is_dir(): + return p + + # 2. Relative to repo root + try: + repo_root = find_repo_root() + candidate = repo_root / "evaluations" / "matpower" / "matpower8.1" + if candidate.is_dir(): + return candidate + except FileNotFoundError: + pass + + # 3. Check if we're in a worktree — look for the main checkout + try: + repo_root = find_repo_root() + # If in a worktree, check the git main working tree + git_dir = repo_root / ".git" + if git_dir.is_file(): + # .git is a file in worktrees, pointing to the main .git dir + text = git_dir.read_text().strip() + if text.startswith("gitdir:"): + main_git = Path(text.split(":", 1)[1].strip()) + # main_git is like /.git/worktrees/ + main_repo = main_git.parent.parent.parent + candidate = main_repo / "evaluations" / "matpower" / "matpower8.1" + if candidate.is_dir(): + return candidate + except (FileNotFoundError, OSError): + pass + + # 4. Container default + container_default = Path("/workspace/evaluations/matpower/matpower8.1") + if container_default.is_dir(): + return container_default + + return None + + +def build_octave_command( + raw_path: str | Path, + output_dir: str | Path, + matpower_path: str | Path | None = None, +) -> list[str]: + """Build the command list for invoking the Octave psse2mpc script. + + Args: + raw_path: Path to the PSS/E RAW file. + output_dir: Directory for CSV output. + matpower_path: Optional path to MATPOWER installation. If None, + the Octave script will use its built-in default. + + Returns: + A list of strings suitable for ``subprocess.run()``. + """ + repo_root = find_repo_root() + script_path = str(repo_root / "data" / "fnm" / "scripts" / "run_psse2mpc.m") + cmd = [ + "octave", + "--no-gui", + "--no-init-file", + script_path, + str(raw_path), + str(output_dir), + ] + if matpower_path is not None: + cmd.append(str(matpower_path)) + return cmd + + +def run_psse2mpc( + raw_path: str | Path, + output_dir: str | Path, + matpower_path: str | Path | None = None, + *, + timeout: int = 300, +) -> MatpowerParserLog: + """Run psse2mpc via Octave and return a structured parser log. + + Args: + raw_path: Path to the PSS/E RAW file. + output_dir: Directory for CSV output. + matpower_path: Optional path to MATPOWER installation. + timeout: Maximum seconds to wait for Octave (default 300). + + Returns: + A MatpowerParserLog with all parsed results. + """ + raw_path = Path(raw_path) + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + # Auto-detect MATPOWER path if not provided + if matpower_path is None: + matpower_path = find_matpower_path() + + cmd = build_octave_command(raw_path, output_dir, matpower_path) + + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + return MatpowerParserLog( + raw_path=str(raw_path), + output_dir=str(output_dir), + return_code=-1, + stdout="", + stderr="Octave process timed out", + ) + + log = MatpowerParserLog( + raw_path=str(raw_path), + output_dir=str(output_dir), + return_code=result.returncode, + stdout=result.stdout, + stderr=result.stderr, + ) + + # Parse stdout for structured data + parsed = parse_octave_stdout(result.stdout) + if "_baseMVA" in parsed: + log.baseMVA = parsed["_baseMVA"] + if "_version" in parsed: + log.version = parsed["_version"] + log.field_counts_octave = {k: v for k, v in parsed.items() if not k.startswith("_")} + + # Parse stderr for warnings + log.warnings = parse_octave_warnings(result.stderr) + + # Read CSV file counts + log.field_counts_csv = read_csv_field_counts(output_dir) + + return log + + +def parse_octave_stdout(stdout: str) -> dict: + """Parse structured output lines from the Octave script. + + Recognizes: + - ``MPC_BASEMVA:`` -> ``{"_baseMVA": }`` + - ``MPC_VERSION:`` -> ``{"_version": }`` + - ``MPC_FIELD_COUNT::`` -> ``{: }`` + + Args: + stdout: Raw stdout string from Octave. + + Returns: + Dict with parsed values. Special keys prefixed with ``_``. + """ + result: dict = {} + if not stdout or not stdout.strip(): + return result + + for line in stdout.splitlines(): + line = line.strip() + if line.startswith("MPC_BASEMVA:"): + try: + result["_baseMVA"] = float(line.split(":", 1)[1]) + except (ValueError, IndexError): + pass + elif line.startswith("MPC_VERSION:"): + try: + result["_version"] = line.split(":", 1)[1].strip() + except IndexError: + pass + elif line.startswith("MPC_FIELD_COUNT:"): + parts = line.split(":") + if len(parts) >= 3: + field_name = parts[1] + try: + count = int(parts[2]) + result[field_name] = count + except ValueError: + pass + + return result + + +def parse_octave_warnings(stderr: str) -> list[ParserWarning]: + """Classify Octave stderr lines into warning categories. + + Categories: + - ``skipped_record``: Record type was skipped/ignored during conversion. + - ``phantom_bus``: Bus referenced but not defined in the bus table. + - ``unsupported_field``: Field or data type not supported by the parser. + - ``conversion_warning``: Generic conversion warning. + + Args: + stderr: Raw stderr string from Octave. + + Returns: + List of classified ParserWarning objects. + """ + warnings: list[ParserWarning] = [] + if not stderr: + return warnings + + for line in stderr.splitlines(): + line = line.strip() + if not line: + continue + + category = _classify_warning(line) + warnings.append(ParserWarning(line=line, category=category)) + + return warnings + + +def _classify_warning(line: str) -> str: + """Classify a single warning line by keyword matching.""" + lower = line.lower() + if any(kw in lower for kw in ("skip", "ignor", "discard", "dropped")): + return "skipped_record" + if any(kw in lower for kw in ("phantom", "missing bus", "undefined bus")): + return "phantom_bus" + if any(kw in lower for kw in ("unsupported", "unrecognized", "unknown field")): + return "unsupported_field" + return "conversion_warning" + + +def read_csv_field_counts(output_dir: str | Path) -> SectionCountMap: + """Count rows in each ``mpc_*.csv`` file in the output directory. + + Rows are counted by number of non-empty lines. ``csvwrite`` from Octave + does not produce a header row, so all lines are data lines. + + Args: + output_dir: Directory containing the CSV exports. + + Returns: + Dict mapping field name (e.g. ``"bus"``) to row count. + """ + output_dir = Path(output_dir) + counts: SectionCountMap = {} + + for csv_path in sorted(output_dir.glob("mpc_*.csv")): + # Extract field name: mpc_bus.csv -> bus + match = re.match(r"mpc_(.+)\.csv$", csv_path.name) + if not match: + continue + field_name = match.group(1) + try: + text = csv_path.read_text(encoding="utf-8") + row_count = sum(1 for line in text.splitlines() if line.strip()) + counts[field_name] = row_count + except OSError: + counts[field_name] = 0 + + return counts + + +def build_known_limitations() -> list[KnownLimitation]: + """Build the list of known MATPOWER parser limitations. + + Covers all record types in ``MPC_DROPPED_RECORD_TYPES`` and + ``MPC_LOSSY_RECORD_TYPES``. + + Returns: + List of KnownLimitation objects. + """ + limitations: list[KnownLimitation] = [] + + dropped_descriptions: dict[str, str] = { + "Two-Terminal DC": ( + "Two-terminal DC line records are not converted to MATPOWER format. " + "The dcline field is populated from a different source or left empty." + ), + "VSC DC": ( + "VSC-based HVDC records are not supported by psse2mpc and are silently dropped." + ), + "Multi-Terminal DC": ( + "Multi-terminal DC network records are not converted. " + "MATPOWER has no equivalent data structure." + ), + "Multi-Section Line": ( + "Multi-section line grouping records are dropped. " + "Individual sections remain as separate branches." + ), + "Impedance Correction": ( + "Impedance correction table records are not used in the conversion. " + "Transformer impedance is taken at nominal tap only." + ), + "FACTS": ( + "FACTS device records (SVC, STATCOM, TCSC) are not converted to MATPOWER format." + ), + } + + for rt in MPC_DROPPED_RECORD_TYPES: + limitations.append( + KnownLimitation( + record_type=rt, + behavior="dropped", + description=dropped_descriptions.get(rt, f"{rt} records are dropped."), + ) + ) + + lossy_descriptions: dict[str, str] = { + "Switched Shunt": ( + "Switched shunt devices are converted to fixed shunts at their initial operating " + "point. Discrete switching steps and voltage control logic are lost." + ), + "Fixed Shunt": ( + "Fixed shunt admittance values are converted but may lose per-unit base " + "information if the bus voltage base differs from system base." + ), + "Transformer": ( + "Transformer records are converted but impedance correction tables, " + "phase-shifting angle limits, and multi-winding control modes may be " + "simplified or lost." + ), + } + + for rt in MPC_LOSSY_RECORD_TYPES: + limitations.append( + KnownLimitation( + record_type=rt, + behavior="lossy", + description=lossy_descriptions.get(rt, f"{rt} conversion is lossy."), + ) + ) + + return limitations + + +def log_to_dict(log: MatpowerParserLog) -> dict: + """Convert a MatpowerParserLog to a JSON-serializable dict. + + Args: + log: The parser log to convert. + + Returns: + A dict suitable for ``json.dumps()``. + """ + return { + "raw_path": log.raw_path, + "output_dir": log.output_dir, + "return_code": log.return_code, + "baseMVA": log.baseMVA, + "version": log.version, + "field_counts_octave": log.field_counts_octave, + "field_counts_csv": log.field_counts_csv, + "warnings": [{"line": w.line, "category": w.category} for w in log.warnings], + "stdout_length": len(log.stdout), + "stderr_length": len(log.stderr), + } + + +def summary_to_dict(summary: MatpowerParserSummary) -> dict: + """Convert a MatpowerParserSummary to a JSON-serializable dict. + + Args: + summary: The parser summary to convert. + + Returns: + A dict suitable for ``json.dumps()``. + """ + return { + "success": summary.success, + "log": log_to_dict(summary.log), + "known_limitations": [ + { + "record_type": kl.record_type, + "behavior": kl.behavior, + "description": kl.description, + } + for kl in summary.known_limitations + ], + } + + +def main(argv: list[str] | None = None) -> None: + """CLI entry point for MATPOWER parser execution. + + Usage:: + + python -m fnm.scripts.matpower_parser [--matpower-path PATH] + """ + import argparse + + parser = argparse.ArgumentParser( + description="Run MATPOWER psse2mpc on a PSS/E RAW file and export results." + ) + parser.add_argument("raw_path", type=str, help="Path to the PSS/E RAW file") + parser.add_argument("output_dir", type=str, help="Directory for CSV exports") + parser.add_argument( + "--matpower-path", + type=str, + default=None, + help="Path to MATPOWER installation directory", + ) + parser.add_argument( + "-o", + "--output", + type=str, + default=None, + help="Output JSON file path (default: print to stdout)", + ) + args = parser.parse_args(argv) + + log = run_psse2mpc(args.raw_path, args.output_dir, args.matpower_path) + limitations = build_known_limitations() + summary = MatpowerParserSummary( + log=log, + known_limitations=limitations, + success=log.return_code == 0, + ) + + result = summary_to_dict(summary) + output_text = json.dumps(result, indent=2) + "\n" + + if args.output: + Path(args.output).write_text(output_text, encoding="utf-8") + print(f"Results written to {args.output}", file=sys.stderr) + else: + print(output_text) diff --git a/data/fnm/scripts/parser_comparison.py b/data/fnm/scripts/parser_comparison.py new file mode 100644 index 00000000..a5628134 --- /dev/null +++ b/data/fnm/scripts/parser_comparison.py @@ -0,0 +1,1375 @@ +"""Parser fidelity comparison and canonical parser selection. + +Compares parser outputs from D3 (raw record counter), D4 (MATPOWER), and +D5 (GridCal) to assess which parser most faithfully preserves the original +PSS/E v31 data. Produces a structured comparison report with fidelity scores +and a selection recommendation. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import Enum +from pathlib import Path + +# --------------------------------------------------------------------------- +# Enums +# --------------------------------------------------------------------------- + + +class DiscrepancyType(str, Enum): + """Classification of a record-count discrepancy between raw and parsed data.""" + + MATCH = "MATCH" + DATA_LOSS = "DATA_LOSS" + PHANTOM_INSERTION = "PHANTOM_INSERTION" + STRUCTURAL_TRANSFORM = "STRUCTURAL_TRANSFORM" + COLLAPSED = "COLLAPSED" + RECORD_TYPE_MISSING = "RECORD_TYPE_MISSING" + + +class ParserName(str, Enum): + """Identifier for each parser under evaluation.""" + + MATPOWER = "MATPOWER" + GRIDCAL = "GRIDCAL" + + +class SelectionRationale(str, Enum): + """Reason for the canonical parser selection decision.""" + + CLEAR_WINNER = "CLEAR_WINNER" + TIER1_TIEBREAK = "TIER1_TIEBREAK" + PHANTOM_TIEBREAK = "PHANTOM_TIEBREAK" + MANUAL_REQUIRED = "MANUAL_REQUIRED" + + +# --------------------------------------------------------------------------- +# Data Classes +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class RecordCountComparison: + """Per-record-type count comparison between raw and both parsers. + + Attributes: + psse_section: PSS/E v31 section name (e.g. ``"Bus"``). + raw_count: Record count from the raw record counter (D3). + matpower_count: Record count from MATPOWER parser (D4), or None if missing. + gridcal_count: Record count from GridCal parser (D5), or None if missing. + matpower_discrepancy: Discrepancy classification for MATPOWER. + gridcal_discrepancy: Discrepancy classification for GridCal. + """ + + psse_section: str + raw_count: int + matpower_count: int | None + gridcal_count: int | None + matpower_discrepancy: DiscrepancyType + gridcal_discrepancy: DiscrepancyType + + +@dataclass(frozen=True) +class FieldCoverageEntry: + """Per-record-type field coverage comparison between parsers. + + Attributes: + psse_section: PSS/E v31 section name. + psse_fields: Expected PSS/E v31 field names for this section. + matpower_fields: Column names from MATPOWER CSV output. + gridcal_fields: Column names from GridCal CSV output. + common_fields: Fields present in both parser outputs (by mapping). + matpower_only: Fields present only in MATPOWER output. + gridcal_only: Fields present only in GridCal output. + matpower_coverage: Fraction of PSS/E fields covered by MATPOWER. + gridcal_coverage: Fraction of PSS/E fields covered by GridCal. + """ + + psse_section: str + psse_fields: list[str] + matpower_fields: list[str] + gridcal_fields: list[str] + common_fields: list[str] + matpower_only: list[str] + gridcal_only: list[str] + matpower_coverage: float + gridcal_coverage: float + + +@dataclass(frozen=True) +class DataLossEntry: + """A single data loss instance identified during comparison. + + Attributes: + psse_section: PSS/E v31 section name where the loss occurred. + parser: Which parser exhibits the loss. + loss_type: The type of discrepancy (DATA_LOSS, RECORD_TYPE_MISSING, etc.). + raw_count: Expected count from the raw file. + parser_count: Actual count from the parser, or None if the record type is missing. + delta: Difference (parser_count - raw_count), or None. + description: Human-readable explanation. + """ + + psse_section: str + parser: ParserName + loss_type: DiscrepancyType + raw_count: int + parser_count: int | None + delta: int | None + description: str + + +@dataclass(frozen=True) +class FidelityScore: + """Per-parser fidelity score with component breakdown. + + Attributes: + parser: Which parser this score belongs to. + overall: Weighted composite fidelity score in [0, 1]. + field_coverage: Average field coverage across all record types. + record_type_coverage: Fraction of PSS/E record types present in parser output. + tier1_field_coverage: Coverage of tier-1 critical fields. + record_count_accuracy: Fraction of record types with exact count match. + phantom_count: Number of PHANTOM_INSERTION discrepancies. + """ + + parser: ParserName + overall: float + field_coverage: float + record_type_coverage: float + tier1_field_coverage: float + record_count_accuracy: float + phantom_count: int + + +@dataclass(frozen=True) +class CanonicalParserSelection: + """Selection decision for the canonical parser. + + Attributes: + selected: The chosen parser. + rationale: The reason for the selection. + matpower_score: Overall fidelity score for MATPOWER. + gridcal_score: Overall fidelity score for GridCal. + score_diff: Absolute difference between the two scores. + explanation: Human-readable explanation of the decision. + """ + + selected: ParserName + rationale: SelectionRationale + matpower_score: float + gridcal_score: float + score_diff: float + explanation: str + + +@dataclass(frozen=True) +class ComparisonMetadata: + """Provenance information for the comparison report. + + Attributes: + timestamp: ISO-8601 timestamp of the comparison run. + raw_counts_path: Path to the D3 raw counts JSON. + matpower_summary_path: Path to the D4 MATPOWER summary JSON. + gridcal_summary_path: Path to the D5 GridCal summary JSON. + matpower_csv_dir: Path to the MATPOWER CSV output directory. + gridcal_csv_dir: Path to the GridCal CSV output directory. + """ + + timestamp: str + raw_counts_path: str + matpower_summary_path: str + gridcal_summary_path: str + matpower_csv_dir: str + gridcal_csv_dir: str + + +@dataclass(frozen=True) +class ParserComparisonReport: + """Complete parser fidelity comparison report. + + Attributes: + metadata: Provenance information. + record_counts: Per-record-type count comparisons. + field_coverage: Per-record-type field coverage comparisons. + data_loss_inventory: All identified data loss instances. + matpower_fidelity: Fidelity score for MATPOWER. + gridcal_fidelity: Fidelity score for GridCal. + selection: Canonical parser selection decision. + """ + + metadata: ComparisonMetadata + record_counts: list[RecordCountComparison] + field_coverage: list[FieldCoverageEntry] + data_loss_inventory: list[DataLossEntry] + matpower_fidelity: FidelityScore + gridcal_fidelity: FidelityScore + selection: CanonicalParserSelection + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +TIER1_CRITICAL_FIELDS: dict[str, list[str]] = { + "Bus": ["I", "NAME", "BASKV", "IDE", "AREA", "ZONE", "VM", "VA"], + "Load": ["I", "ID", "STATUS", "PL", "QL"], + "Generator": ["I", "ID", "PG", "QG", "QT", "QB", "VS", "MBASE"], + "Branch": ["I", "J", "CKT", "R", "X", "B", "RATEA", "RATEB", "RATEC"], + "Transformer": ["I", "J", "K", "CKT", "CW", "CZ", "CM", "R1-2", "X1-2", "WINDV1", "WINDV2"], + "Switched Shunt": ["I", "MODSW", "VSWHI", "VSWLO", "BINIT"], + "Area": ["I", "ISW", "PDES", "PTOL", "ARNAM"], + "Fixed Shunt": ["I", "ID", "STATUS", "GL", "BL"], +} + + +# --------------------------------------------------------------------------- +# PSS/E v31 Field Specification +# --------------------------------------------------------------------------- + + +def get_psse_v31_field_spec() -> dict[str, list[str]]: + """Return PSS/E v31 field names per record type. + + Returns: + Dict mapping PSS/E section name to list of field names. + """ + return { + "Bus": [ + "I", + "NAME", + "BASKV", + "IDE", + "AREA", + "ZONE", + "OWNER", + "VM", + "VA", + "GL", + "BL", + "NVHI", + "NVLO", + ], + "Load": [ + "I", + "ID", + "STATUS", + "AREA", + "ZONE", + "PL", + "QL", + "IP", + "IQ", + "YP", + "YQ", + "OWNER", + "SCALE", + ], + "Fixed Shunt": ["I", "ID", "STATUS", "GL", "BL"], + "Generator": [ + "I", + "ID", + "PG", + "QG", + "QT", + "QB", + "VS", + "IREG", + "MBASE", + "ZR", + "ZX", + "RT", + "XT", + "GTAP", + "STAT", + "RMPCT", + "PT", + "PB", + "O1", + "F1", + "O2", + "F2", + "O3", + "F3", + "O4", + "F4", + "WMOD", + "WPF", + ], + "Branch": [ + "I", + "J", + "CKT", + "R", + "X", + "B", + "RATEA", + "RATEB", + "RATEC", + "GI", + "BI", + "GJ", + "BJ", + "ST", + "LEN", + "O1", + "F1", + "O2", + "F2", + "O3", + "F3", + "O4", + "F4", + ], + "Transformer": [ + "I", + "J", + "K", + "CKT", + "CW", + "CZ", + "CM", + "MAG1", + "MAG2", + "NMETR", + "NAME", + "STAT", + "O1", + "F1", + "O2", + "F2", + "O3", + "F3", + "O4", + "F4", + "R1-2", + "X1-2", + "SBASE1-2", + "WINDV1", + "NOMV1", + "ANG1", + "RATA1", + "RATB1", + "RATC1", + "COD1", + "CONT1", + "RMA1", + "RMI1", + "VMA1", + "VMI1", + "NTP1", + "TAB1", + "CR1", + "CX1", + "WINDV2", + "NOMV2", + ], + "Area": ["I", "ISW", "PDES", "PTOL", "ARNAM"], + "Two-Terminal DC": [ + "NAME", + "MDC", + "RDC", + "SETEFX", + "VSCHD", + "VCMOD", + "RCOMP", + "DELTI", + "METER", + "DCVMIN", + "CCCITMX", + "CCCACC", + "IPR", + "NBR", + "ANMXR", + "ANMNR", + "RCR", + "XCR", + "EBASR", + "TRR", + "TAPR", + "TMXR", + "TMNR", + "STPR", + "ICR", + "IFR", + "ITR", + "IDR", + "XCAPR", + "IPI", + "NBI", + "ANMXI", + "ANMNI", + "RCI", + "XCI", + "EBASI", + "TRI", + "TAPI", + "TMXI", + "TMNI", + "STPI", + "ICI", + "IFI", + "ITI", + "IDI", + "XCAPI", + ], + "VSC DC": [ + "NAME", + "MDC", + "RDC", + "O1", + "F1", + "O2", + "F2", + "O3", + "F3", + "O4", + "F4", + "IBUS1", + "TYPE1", + "MODE1", + "DCSET1", + "ACSET1", + "ALOSS1", + "BLOSS1", + "MINLOSS1", + "SMAX1", + "IMAX1", + "PWF1", + "MAXQ1", + "MINQ1", + "REMOT1", + "RMPCT1", + ], + "Impedance Correction": ["T1", "F1", "T2", "F2", "T3", "F3"], + "Multi-Terminal DC": ["NAME", "NCONV", "NDCBS", "NDCLN"], + "Multi-Section Line": ["I", "J", "ID", "MET", "DUM1"], + "Zone": ["I", "ZONAME"], + "Interarea Transfer": ["ARFROM", "ARTO", "TRID", "PTRAN"], + "Owner": ["I", "OWNAME"], + "FACTS": [ + "NAME", + "I", + "J", + "MODE", + "PDES", + "QDES", + "VSET", + "SHMX", + "TRMX", + "VTMN", + "VTMX", + "VSMX", + "IMX", + "LINX", + "RMPCT", + "OWNER", + "SET1", + "SET2", + "VSREF", + ], + "Switched Shunt": [ + "I", + "MODSW", + "ADJM", + "STAT", + "VSWHI", + "VSWLO", + "SWREM", + "RMPCT", + "RMIDNT", + "BINIT", + "N1", + "B1", + "N2", + "B2", + "N3", + "B3", + "N4", + "B4", + "N5", + "B5", + "N6", + "B6", + "N7", + "B7", + "N8", + "B8", + ], + } + + +# --------------------------------------------------------------------------- +# Record-Type Mapping +# --------------------------------------------------------------------------- + +# PSS/E section name -> (matpower table name or None, gridcal table name or None) +_PSSE_TO_PARSER_TABLE: dict[str, tuple[str | None, str | None]] = { + "Bus": ("bus", "buses"), + "Load": (None, "loads"), # MATPOWER folds loads into bus + "Fixed Shunt": (None, "shunts"), # MATPOWER folds into bus + "Generator": ("gen", "generators"), + "Branch": ("branch", "lines"), + "Transformer": ("branch", "transformers2w"), # MATPOWER merges into branch + "Area": ("areas", "areas"), + "Two-Terminal DC": ("dcline", "hvdc_lines"), + "VSC DC": (None, "vsc_devices"), + "Impedance Correction": (None, None), + "Multi-Terminal DC": (None, None), + "Multi-Section Line": (None, None), + "Zone": (None, "zones"), + "Interarea Transfer": (None, None), + "Owner": (None, None), + "FACTS": (None, "facts_devices"), + "Switched Shunt": (None, "controllable_shunts"), +} + + +def build_record_type_mapping() -> dict[str, tuple[str | None, str | None]]: + """Return PSS/E section to parser table name mapping. + + Returns: + Dict mapping PSS/E section name to a tuple of + (matpower_table_name, gridcal_table_name). None means the parser + does not produce a separate table for that record type. + """ + return dict(_PSSE_TO_PARSER_TABLE) + + +def build_field_name_mapping() -> dict[str, dict[str, str]]: + """Return PSS/E field to parser column name mapping. + + This is a best-effort mapping of canonical PSS/E field names to the + column names used by each parser. Only covers commonly-mapped fields. + + Returns: + Dict mapping PSS/E section name to a dict of + {psse_field: parser_column_name} for each parser. The inner dict + keys are PSS/E field names, values are lowercase parser column names. + """ + # For the purpose of comparison, we map key PSS/E fields to lowercase + # column names that the parsers tend to use. This mapping is approximate. + return { + "Bus": { + "I": "bus_i", + "NAME": "name", + "BASKV": "base_kv", + "IDE": "type", + "AREA": "area", + "ZONE": "zone", + "VM": "vm", + "VA": "va", + }, + "Generator": { + "I": "bus", + "PG": "pg", + "QG": "qg", + "QT": "qmax", + "QB": "qmin", + "VS": "vs", + "MBASE": "mbase", + }, + "Branch": { + "I": "fbus", + "J": "tbus", + "R": "r", + "X": "x", + "B": "b", + "RATEA": "rate_a", + "RATEB": "rate_b", + "RATEC": "rate_c", + }, + } + + +# --------------------------------------------------------------------------- +# Load Functions +# --------------------------------------------------------------------------- + + +def load_raw_counts(path: str | Path) -> dict[str, int]: + """Load D3 raw record counter JSON and return section counts. + + Args: + path: Path to the D3 JSON output file. + + Returns: + Dict mapping PSS/E section name to record count. + """ + path = Path(path) + data = json.loads(path.read_text(encoding="utf-8")) + return dict(data.get("section_counts", {})) + + +def load_parser_summary(path: str | Path) -> dict[str, int]: + """Load D4 (MATPOWER) or D5 (GridCal) summary JSON and return record counts. + + For MATPOWER, reads ``log.field_counts_csv``. + For GridCal, reads ``multicircuit_counts``. + + Args: + path: Path to the parser summary JSON file. + + Returns: + Dict mapping table/collection name to record count. + """ + path = Path(path) + data = json.loads(path.read_text(encoding="utf-8")) + + # Try MATPOWER format first + if "log" in data and "field_counts_csv" in data.get("log", {}): + return dict(data["log"]["field_counts_csv"]) + + # Try GridCal format + if "multicircuit_counts" in data: + return {k: v for k, v in data["multicircuit_counts"].items() if v > 0} + + # Fallback: return top-level dict if it looks like counts + return {k: v for k, v in data.items() if isinstance(v, int)} + + +def load_csv_columns(csv_dir: str | Path) -> dict[str, list[str]]: + """Read CSV headers from a directory of parser output CSVs. + + Recognizes both ``mpc_*.csv`` (MATPOWER) and ``gridcal_*.csv`` (GridCal) naming. + + Args: + csv_dir: Directory containing CSV files. + + Returns: + Dict mapping table name to list of column header strings. + """ + import csv + import re + + csv_dir = Path(csv_dir) + result: dict[str, list[str]] = {} + + if not csv_dir.is_dir(): + return result + + for csv_path in sorted(csv_dir.glob("*.csv")): + # Extract table name from filename + match = re.match(r"(?:mpc_|gridcal_)(.+)\.csv$", csv_path.name) + if not match: + # Try bare name + table_name = csv_path.stem + else: + table_name = match.group(1) + + try: + with open(csv_path, encoding="utf-8") as f: + reader = csv.reader(f) + header = next(reader, None) + if header: + result[table_name] = [col.strip() for col in header] + except (OSError, StopIteration): + pass + + return result + + +# --------------------------------------------------------------------------- +# Comparison Functions +# --------------------------------------------------------------------------- + + +def compare_record_counts( + raw: dict[str, int], + matpower: dict[str, int], + gridcal: dict[str, int], + mapping: dict[str, tuple[str | None, str | None]], +) -> list[RecordCountComparison]: + """Compare record counts from raw (D3) against both parsers. + + Args: + raw: PSS/E section name to count from D3. + matpower: MATPOWER table name to count from D4. + gridcal: GridCal collection name to count from D5. + mapping: PSS/E section to (matpower_table, gridcal_table) mapping. + + Returns: + List of RecordCountComparison, one per PSS/E section in ``raw``. + """ + comparisons: list[RecordCountComparison] = [] + + for section, raw_count in raw.items(): + mp_table, gc_table = mapping.get(section, (None, None)) + + # MATPOWER count + mp_count: int | None = None + if mp_table is not None and mp_table in matpower: + mp_count = matpower[mp_table] + + # GridCal count + gc_count: int | None = None + if gc_table is not None and gc_table in gridcal: + gc_count = gridcal[gc_table] + + mp_disc = _classify_count_discrepancy(raw_count, mp_count, mp_table) + gc_disc = _classify_count_discrepancy(raw_count, gc_count, gc_table) + + comparisons.append( + RecordCountComparison( + psse_section=section, + raw_count=raw_count, + matpower_count=mp_count, + gridcal_count=gc_count, + matpower_discrepancy=mp_disc, + gridcal_discrepancy=gc_disc, + ) + ) + + return comparisons + + +def _classify_count_discrepancy( + raw_count: int, + parser_count: int | None, + table_name: str | None, +) -> DiscrepancyType: + """Classify a single count discrepancy.""" + if table_name is None: + # Parser has no table for this record type + if raw_count > 0: + return DiscrepancyType.RECORD_TYPE_MISSING + return DiscrepancyType.MATCH + + if parser_count is None: + if raw_count > 0: + return DiscrepancyType.RECORD_TYPE_MISSING + return DiscrepancyType.MATCH + + if parser_count == raw_count: + return DiscrepancyType.MATCH + elif parser_count < raw_count: + return DiscrepancyType.DATA_LOSS + else: + return DiscrepancyType.PHANTOM_INSERTION + + +def compare_field_coverage( + psse_spec: dict[str, list[str]], + matpower_columns: dict[str, list[str]], + gridcal_columns: dict[str, list[str]], + mapping: dict[str, tuple[str | None, str | None]], +) -> list[FieldCoverageEntry]: + """Compare field coverage of each parser against the PSS/E v31 specification. + + Args: + psse_spec: PSS/E section to list of expected field names. + matpower_columns: MATPOWER table to list of CSV column names. + gridcal_columns: GridCal table to list of CSV column names. + mapping: PSS/E section to (matpower_table, gridcal_table) mapping. + + Returns: + List of FieldCoverageEntry, one per PSS/E section in ``psse_spec``. + """ + entries: list[FieldCoverageEntry] = [] + + for section, psse_fields in psse_spec.items(): + mp_table, gc_table = mapping.get(section, (None, None)) + + mp_fields = matpower_columns.get(mp_table, []) if mp_table else [] + gc_fields = gridcal_columns.get(gc_table, []) if gc_table else [] + + # Compute sets (case-insensitive) + mp_set = {f.lower() for f in mp_fields} + gc_set = {f.lower() for f in gc_fields} + common = sorted(mp_set & gc_set) + mp_only = sorted(mp_set - gc_set) + gc_only = sorted(gc_set - mp_set) + + n_psse = len(psse_fields) if psse_fields else 1 + mp_coverage = len(mp_set) / n_psse if mp_set else 0.0 + gc_coverage = len(gc_set) / n_psse if gc_set else 0.0 + + entries.append( + FieldCoverageEntry( + psse_section=section, + psse_fields=list(psse_fields), + matpower_fields=list(mp_fields), + gridcal_fields=list(gc_fields), + common_fields=common, + matpower_only=mp_only, + gridcal_only=gc_only, + matpower_coverage=min(mp_coverage, 1.0), + gridcal_coverage=min(gc_coverage, 1.0), + ) + ) + + return entries + + +def build_data_loss_inventory( + counts: list[RecordCountComparison], + fields: list[FieldCoverageEntry], +) -> list[DataLossEntry]: + """Build an inventory of all data loss instances from count and field comparisons. + + Args: + counts: Record count comparisons from ``compare_record_counts``. + fields: Field coverage entries from ``compare_field_coverage``. + + Returns: + List of DataLossEntry for every non-MATCH discrepancy. + """ + inventory: list[DataLossEntry] = [] + + for c in counts: + for parser, disc, p_count in [ + (ParserName.MATPOWER, c.matpower_discrepancy, c.matpower_count), + (ParserName.GRIDCAL, c.gridcal_discrepancy, c.gridcal_count), + ]: + if disc == DiscrepancyType.MATCH: + continue + + delta = (p_count - c.raw_count) if p_count is not None else None + desc = _describe_loss(c.psse_section, parser, disc, c.raw_count, p_count) + inventory.append( + DataLossEntry( + psse_section=c.psse_section, + parser=parser, + loss_type=disc, + raw_count=c.raw_count, + parser_count=p_count, + delta=delta, + description=desc, + ) + ) + + return inventory + + +def _describe_loss( + section: str, + parser: ParserName, + disc: DiscrepancyType, + raw_count: int, + parser_count: int | None, +) -> str: + """Generate a human-readable description for a data loss entry.""" + if disc == DiscrepancyType.RECORD_TYPE_MISSING: + return f"{parser.value} has no table for {section} ({raw_count} raw records lost)." + if disc == DiscrepancyType.DATA_LOSS: + return ( + f"{parser.value} has {parser_count} records for {section} " + f"vs {raw_count} raw (lost {raw_count - (parser_count or 0)})." + ) + if disc == DiscrepancyType.PHANTOM_INSERTION: + return ( + f"{parser.value} has {parser_count} records for {section} " + f"vs {raw_count} raw (phantom +{(parser_count or 0) - raw_count})." + ) + return f"{parser.value} {section}: {disc.value}" + + +# --------------------------------------------------------------------------- +# Fidelity Scoring +# --------------------------------------------------------------------------- + +_W_FIELD_COVERAGE = 0.35 +_W_RECORD_TYPE_COVERAGE = 0.30 +_W_TIER1_FIELD_COVERAGE = 0.20 +_W_RECORD_COUNT_ACCURACY = 0.15 + + +def compute_fidelity_score( + parser: ParserName, + counts: list[RecordCountComparison], + fields: list[FieldCoverageEntry], + losses: list[DataLossEntry], + psse_spec: dict[str, list[str]], + tier1: dict[str, list[str]], +) -> FidelityScore: + """Compute a composite fidelity score for a single parser. + + Weights: 0.35 field_coverage + 0.30 record_type_coverage + + 0.20 tier1_field_coverage + 0.15 record_count_accuracy + + Args: + parser: Which parser to score. + counts: Record count comparisons. + fields: Field coverage entries. + losses: Data loss inventory. + psse_spec: PSS/E v31 field specification. + tier1: Tier-1 critical fields per section. + + Returns: + FidelityScore with all component scores. + """ + # 1. Field coverage: average coverage across sections that have parser fields + if parser == ParserName.MATPOWER: + cov_values = [f.matpower_coverage for f in fields] + else: + cov_values = [f.gridcal_coverage for f in fields] + field_cov = sum(cov_values) / len(cov_values) if cov_values else 0.0 + + # 2. Record type coverage: fraction of raw sections that have a parser table + total_raw_sections = len([c for c in counts if c.raw_count > 0]) + if total_raw_sections > 0: + if parser == ParserName.MATPOWER: + present = len( + [ + c + for c in counts + if c.raw_count > 0 + and c.matpower_discrepancy != DiscrepancyType.RECORD_TYPE_MISSING + ] + ) + else: + present = len( + [ + c + for c in counts + if c.raw_count > 0 + and c.gridcal_discrepancy != DiscrepancyType.RECORD_TYPE_MISSING + ] + ) + rt_cov = present / total_raw_sections + else: + rt_cov = 1.0 + + # 3. Tier-1 field coverage: fraction of tier-1 fields covered + tier1_total = 0 + tier1_covered = 0 + for section, t1_fields in tier1.items(): + tier1_total += len(t1_fields) + # Find the matching field coverage entry + for f in fields: + if f.psse_section == section: + if parser == ParserName.MATPOWER: + parser_fields_lower = {x.lower() for x in f.matpower_fields} + else: + parser_fields_lower = {x.lower() for x in f.gridcal_fields} + for t1f in t1_fields: + if t1f.lower() in parser_fields_lower: + tier1_covered += 1 + break + tier1_cov = tier1_covered / tier1_total if tier1_total > 0 else 0.0 + + # 4. Record count accuracy: fraction of sections with exact match + if parser == ParserName.MATPOWER: + matches = len([c for c in counts if c.matpower_discrepancy == DiscrepancyType.MATCH]) + else: + matches = len([c for c in counts if c.gridcal_discrepancy == DiscrepancyType.MATCH]) + rc_acc = matches / len(counts) if counts else 1.0 + + # Phantom count + if parser == ParserName.MATPOWER: + phantoms = len( + [c for c in counts if c.matpower_discrepancy == DiscrepancyType.PHANTOM_INSERTION] + ) + else: + phantoms = len( + [c for c in counts if c.gridcal_discrepancy == DiscrepancyType.PHANTOM_INSERTION] + ) + + overall = ( + _W_FIELD_COVERAGE * field_cov + + _W_RECORD_TYPE_COVERAGE * rt_cov + + _W_TIER1_FIELD_COVERAGE * tier1_cov + + _W_RECORD_COUNT_ACCURACY * rc_acc + ) + + return FidelityScore( + parser=parser, + overall=round(overall, 6), + field_coverage=round(field_cov, 6), + record_type_coverage=round(rt_cov, 6), + tier1_field_coverage=round(tier1_cov, 6), + record_count_accuracy=round(rc_acc, 6), + phantom_count=phantoms, + ) + + +# --------------------------------------------------------------------------- +# Selection Logic +# --------------------------------------------------------------------------- + + +def select_canonical_parser( + mp_score: FidelityScore, + gc_score: FidelityScore, +) -> CanonicalParserSelection: + """Select the canonical parser based on fidelity scores. + + Decision tree: + 1. |diff| > 0.05 -> CLEAR_WINNER + 2. tier1 diff > 0.02 -> TIER1_TIEBREAK + 3. phantom count differs -> PHANTOM_TIEBREAK (fewer phantoms wins) + 4. else -> MANUAL_REQUIRED + + Args: + mp_score: Fidelity score for MATPOWER. + gc_score: Fidelity score for GridCal. + + Returns: + CanonicalParserSelection with the decision. + """ + diff = abs(mp_score.overall - gc_score.overall) + + # 1. Clear winner + if diff > 0.05: + winner = ParserName.MATPOWER if mp_score.overall > gc_score.overall else ParserName.GRIDCAL + return CanonicalParserSelection( + selected=winner, + rationale=SelectionRationale.CLEAR_WINNER, + matpower_score=mp_score.overall, + gridcal_score=gc_score.overall, + score_diff=round(diff, 6), + explanation=( + f"{winner.value} wins with overall score " + f"{max(mp_score.overall, gc_score.overall):.4f} " + f"vs {min(mp_score.overall, gc_score.overall):.4f} " + f"(diff={diff:.4f} > 0.05)." + ), + ) + + # 2. Tier-1 tiebreak + tier1_diff = abs(mp_score.tier1_field_coverage - gc_score.tier1_field_coverage) + if tier1_diff > 0.02: + winner = ( + ParserName.MATPOWER + if mp_score.tier1_field_coverage > gc_score.tier1_field_coverage + else ParserName.GRIDCAL + ) + return CanonicalParserSelection( + selected=winner, + rationale=SelectionRationale.TIER1_TIEBREAK, + matpower_score=mp_score.overall, + gridcal_score=gc_score.overall, + score_diff=round(diff, 6), + explanation=( + f"Overall scores tied (diff={diff:.4f}). " + f"{winner.value} wins on tier-1 field coverage " + f"({max(mp_score.tier1_field_coverage, gc_score.tier1_field_coverage):.4f} " + f"vs {min(mp_score.tier1_field_coverage, gc_score.tier1_field_coverage):.4f})." + ), + ) + + # 3. Phantom tiebreak + if mp_score.phantom_count != gc_score.phantom_count: + winner = ( + ParserName.MATPOWER + if mp_score.phantom_count < gc_score.phantom_count + else ParserName.GRIDCAL + ) + return CanonicalParserSelection( + selected=winner, + rationale=SelectionRationale.PHANTOM_TIEBREAK, + matpower_score=mp_score.overall, + gridcal_score=gc_score.overall, + score_diff=round(diff, 6), + explanation=( + f"Overall and tier-1 scores tied. " + f"{winner.value} wins with fewer phantom insertions " + f"({min(mp_score.phantom_count, gc_score.phantom_count)} " + f"vs {max(mp_score.phantom_count, gc_score.phantom_count)})." + ), + ) + + # 4. Manual required + return CanonicalParserSelection( + selected=ParserName.GRIDCAL, # Default to GridCal if truly tied + rationale=SelectionRationale.MANUAL_REQUIRED, + matpower_score=mp_score.overall, + gridcal_score=gc_score.overall, + score_diff=round(diff, 6), + explanation=( + "Scores, tier-1 coverage, and phantom counts are all tied. Manual review required." + ), + ) + + +# --------------------------------------------------------------------------- +# Report Building +# --------------------------------------------------------------------------- + + +def build_comparison_report( + raw_path: str | Path, + mp_path: str | Path, + gc_path: str | Path, + mp_csvs: str | Path, + gc_csvs: str | Path, +) -> ParserComparisonReport: + """Build a complete comparison report from D3/D4/D5 output files. + + Args: + raw_path: Path to D3 raw counts JSON. + mp_path: Path to D4 MATPOWER summary JSON. + gc_path: Path to D5 GridCal summary JSON. + mp_csvs: Path to MATPOWER CSV output directory. + gc_csvs: Path to GridCal CSV output directory. + + Returns: + A fully populated ParserComparisonReport. + """ + raw_counts = load_raw_counts(raw_path) + mp_counts = load_parser_summary(mp_path) + gc_counts = load_parser_summary(gc_path) + + mp_columns = load_csv_columns(mp_csvs) + gc_columns = load_csv_columns(gc_csvs) + + mapping = build_record_type_mapping() + psse_spec = get_psse_v31_field_spec() + tier1 = TIER1_CRITICAL_FIELDS + + record_counts = compare_record_counts(raw_counts, mp_counts, gc_counts, mapping) + field_coverage = compare_field_coverage(psse_spec, mp_columns, gc_columns, mapping) + data_losses = build_data_loss_inventory(record_counts, field_coverage) + + mp_fidelity = compute_fidelity_score( + ParserName.MATPOWER, record_counts, field_coverage, data_losses, psse_spec, tier1 + ) + gc_fidelity = compute_fidelity_score( + ParserName.GRIDCAL, record_counts, field_coverage, data_losses, psse_spec, tier1 + ) + + selection = select_canonical_parser(mp_fidelity, gc_fidelity) + + metadata = ComparisonMetadata( + timestamp=datetime.now(tz=timezone.utc).isoformat(), + raw_counts_path=str(raw_path), + matpower_summary_path=str(mp_path), + gridcal_summary_path=str(gc_path), + matpower_csv_dir=str(mp_csvs), + gridcal_csv_dir=str(gc_csvs), + ) + + return ParserComparisonReport( + metadata=metadata, + record_counts=record_counts, + field_coverage=field_coverage, + data_loss_inventory=data_losses, + matpower_fidelity=mp_fidelity, + gridcal_fidelity=gc_fidelity, + selection=selection, + ) + + +# --------------------------------------------------------------------------- +# Serialization +# --------------------------------------------------------------------------- + + +def report_to_dict(report: ParserComparisonReport) -> dict: + """Convert a ParserComparisonReport to a JSON-serializable dict. + + Args: + report: The comparison report. + + Returns: + A dict suitable for ``json.dumps()``. + """ + return { + "metadata": { + "timestamp": report.metadata.timestamp, + "raw_counts_path": report.metadata.raw_counts_path, + "matpower_summary_path": report.metadata.matpower_summary_path, + "gridcal_summary_path": report.metadata.gridcal_summary_path, + "matpower_csv_dir": report.metadata.matpower_csv_dir, + "gridcal_csv_dir": report.metadata.gridcal_csv_dir, + }, + "record_counts": [ + { + "psse_section": c.psse_section, + "raw_count": c.raw_count, + "matpower_count": c.matpower_count, + "gridcal_count": c.gridcal_count, + "matpower_discrepancy": c.matpower_discrepancy.value, + "gridcal_discrepancy": c.gridcal_discrepancy.value, + } + for c in report.record_counts + ], + "field_coverage": [ + { + "psse_section": f.psse_section, + "psse_fields": f.psse_fields, + "matpower_fields": f.matpower_fields, + "gridcal_fields": f.gridcal_fields, + "common_fields": f.common_fields, + "matpower_only": f.matpower_only, + "gridcal_only": f.gridcal_only, + "matpower_coverage": f.matpower_coverage, + "gridcal_coverage": f.gridcal_coverage, + } + for f in report.field_coverage + ], + "data_loss_inventory": [ + { + "psse_section": d.psse_section, + "parser": d.parser.value, + "loss_type": d.loss_type.value, + "raw_count": d.raw_count, + "parser_count": d.parser_count, + "delta": d.delta, + "description": d.description, + } + for d in report.data_loss_inventory + ], + "matpower_fidelity": _fidelity_to_dict(report.matpower_fidelity), + "gridcal_fidelity": _fidelity_to_dict(report.gridcal_fidelity), + "selection": { + "selected": report.selection.selected.value, + "rationale": report.selection.rationale.value, + "matpower_score": report.selection.matpower_score, + "gridcal_score": report.selection.gridcal_score, + "score_diff": report.selection.score_diff, + "explanation": report.selection.explanation, + }, + } + + +def _fidelity_to_dict(score: FidelityScore) -> dict: + """Convert a FidelityScore to a JSON-serializable dict.""" + return { + "parser": score.parser.value, + "overall": score.overall, + "field_coverage": score.field_coverage, + "record_type_coverage": score.record_type_coverage, + "tier1_field_coverage": score.tier1_field_coverage, + "record_count_accuracy": score.record_count_accuracy, + "phantom_count": score.phantom_count, + } + + +def report_to_markdown(report: ParserComparisonReport) -> str: + """Render a ParserComparisonReport as a Markdown document. + + Args: + report: The comparison report. + + Returns: + A Markdown string. + """ + lines: list[str] = [] + lines.append("# Parser Fidelity Comparison Report") + lines.append("") + lines.append(f"**Generated:** {report.metadata.timestamp}") + lines.append("") + + # Selection summary + sel = report.selection + lines.append("## Canonical Parser Selection") + lines.append("") + lines.append(f"- **Selected:** {sel.selected.value}") + lines.append(f"- **Rationale:** {sel.rationale.value}") + lines.append(f"- **Score diff:** {sel.score_diff:.4f}") + lines.append(f"- **Explanation:** {sel.explanation}") + lines.append("") + + # Fidelity scores + lines.append("## Fidelity Scores") + lines.append("") + lines.append("| Component | MATPOWER | GridCal |") + lines.append("|-----------|----------|---------|") + mp = report.matpower_fidelity + gc = report.gridcal_fidelity + lines.append(f"| Overall | {mp.overall:.4f} | {gc.overall:.4f} |") + lines.append(f"| Field Coverage | {mp.field_coverage:.4f} | {gc.field_coverage:.4f} |") + lines.append( + f"| Record Type Coverage | {mp.record_type_coverage:.4f} | {gc.record_type_coverage:.4f} |" + ) + lines.append( + f"| Tier-1 Field Coverage | {mp.tier1_field_coverage:.4f} | {gc.tier1_field_coverage:.4f} |" + ) + lines.append( + f"| Record Count Accuracy | {mp.record_count_accuracy:.4f} " + f"| {gc.record_count_accuracy:.4f} |" + ) + lines.append(f"| Phantom Insertions | {mp.phantom_count} | {gc.phantom_count} |") + lines.append("") + + # Record counts + lines.append("## Record Count Comparison") + lines.append("") + lines.append("| Section | Raw | MATPOWER | GridCal | MP Status | GC Status |") + lines.append("|---------|-----|----------|---------|-----------|-----------|") + for c in report.record_counts: + mp_val = str(c.matpower_count) if c.matpower_count is not None else "-" + gc_val = str(c.gridcal_count) if c.gridcal_count is not None else "-" + lines.append( + f"| {c.psse_section} | {c.raw_count} | {mp_val} | {gc_val} " + f"| {c.matpower_discrepancy.value} | {c.gridcal_discrepancy.value} |" + ) + lines.append("") + + # Data loss inventory + if report.data_loss_inventory: + lines.append("## Data Loss Inventory") + lines.append("") + for d in report.data_loss_inventory: + lines.append(f"- **{d.psse_section}** ({d.parser.value}): {d.description}") + lines.append("") + + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> None: + """CLI entry point for parser comparison. + + Usage:: + + python -m fnm.scripts.parser_comparison \\ + --raw-counts d3_counts.json \\ + --matpower-summary d4_summary.json \\ + --gridcal-summary d5_summary.json \\ + --matpower-csvs /path/to/matpower/csvs \\ + --gridcal-csvs /path/to/gridcal/csvs \\ + [-o output.json] [--markdown output.md] + """ + parser = argparse.ArgumentParser( + description="Compare MATPOWER and GridCal parser fidelity against raw PSS/E counts." + ) + parser.add_argument("--raw-counts", required=True, help="Path to D3 raw counts JSON") + parser.add_argument( + "--matpower-summary", required=True, help="Path to D4 MATPOWER summary JSON" + ) + parser.add_argument("--gridcal-summary", required=True, help="Path to D5 GridCal summary JSON") + parser.add_argument( + "--matpower-csvs", required=True, help="Path to MATPOWER CSV output directory" + ) + parser.add_argument( + "--gridcal-csvs", required=True, help="Path to GridCal CSV output directory" + ) + parser.add_argument( + "-o", "--output", default=None, help="Output JSON file path (default: stdout)" + ) + parser.add_argument("--markdown", default=None, help="Output Markdown file path") + + args = parser.parse_args(argv) + + report = build_comparison_report( + raw_path=args.raw_counts, + mp_path=args.matpower_summary, + gc_path=args.gridcal_summary, + mp_csvs=args.matpower_csvs, + gc_csvs=args.gridcal_csvs, + ) + + result_dict = report_to_dict(report) + json_text = json.dumps(result_dict, indent=2) + "\n" + + if args.output: + Path(args.output).write_text(json_text, encoding="utf-8") + print(f"JSON report written to {args.output}", file=sys.stderr) + else: + print(json_text) + + if args.markdown: + md_text = report_to_markdown(report) + Path(args.markdown).write_text(md_text, encoding="utf-8") + print(f"Markdown report written to {args.markdown}", file=sys.stderr) diff --git a/data/fnm/scripts/pass_conditions.py b/data/fnm/scripts/pass_conditions.py new file mode 100644 index 00000000..d2e8d690 --- /dev/null +++ b/data/fnm/scripts/pass_conditions.py @@ -0,0 +1,1759 @@ +"""Pass Condition Definitions for ACPF and DCPF FNM Verification. + +Produces the formal pass condition specification as both machine-readable JSON +(``data/fnm/reference/pass_conditions.json``) and human-readable markdown +(``data/fnm/reference/pass_conditions.md``). The JSON is consumed at runtime +by evaluate-tool agents when comparing a tool's FNM power flow results against +the Phase 3 reference solutions. + +The module is stateless and deterministic: given the threshold constants defined +here, it generates identical JSON and markdown output on every run. There is no +FNM data dependency at generation time. + +Uses only Python stdlib (no numpy/scipy). +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from collections import Counter +from collections.abc import Sequence +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path + +# --------------------------------------------------------------------------- +# Outlier cause classification +# --------------------------------------------------------------------------- + + +class OutlierCause(Enum): + """Probable cause for a bus or branch exceeding the pass tolerance.""" + + SWITCHED_SHUNT = "switched_shunt" + """Bus has a switched shunt device.""" + + Q_LIMIT = "q_limit" + """Bus has a generator that hit a reactive power limit.""" + + SLACK_DISTRIBUTION = "slack_distribution" + """Bus is the slack bus or electrically close to it.""" + + TAP_POSITION = "tap_position" + """Bus is the regulated bus of a tap-changing transformer.""" + + ISLAND_BOUNDARY = "island_boundary" + """Bus is at the boundary of a weakly connected subnetwork.""" + + UNCLASSIFIED = "unclassified" + """Bus exceeds tolerance but matches no classification rule.""" + + +OUTLIER_PRIORITY: list[OutlierCause] = [ + OutlierCause.SWITCHED_SHUNT, + OutlierCause.Q_LIMIT, + OutlierCause.SLACK_DISTRIBUTION, + OutlierCause.TAP_POSITION, + OutlierCause.ISLAND_BOUNDARY, + OutlierCause.UNCLASSIFIED, +] + + +# --------------------------------------------------------------------------- +# ACPF pass condition parameters +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ACPFAggregateThresholds: + """Aggregate pass thresholds for ACPF verification.""" + + min_passing_fraction: float = 0.95 + vm_tolerance_pu: float = 0.005 + va_tolerance_deg: float = 0.5 + + +@dataclass(frozen=True) +class ACPFHardFailThresholds: + """Hard-fail thresholds for ACPF verification.""" + + max_failing_fraction: float = 0.20 + vm_max_deviation_pu: float = 0.1 + va_max_deviation_deg: float = 10.0 + + +# --------------------------------------------------------------------------- +# DCPF pass condition parameters +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class DCPFAggregateThresholds: + """Aggregate pass thresholds for DCPF verification.""" + + min_bus_passing_fraction: float = 0.95 + va_tolerance_deg: float = 1.0 + min_branch_passing_fraction: float = 0.90 + p_tolerance_pct: float = 10.0 + p_base_floor_mw: float = 1.0 + + +@dataclass(frozen=True) +class DCPFHardFailThresholds: + """Hard-fail thresholds for DCPF verification.""" + + max_bus_failing_fraction: float = 0.20 + max_branch_failing_fraction: float = 0.20 + p_max_deviation_pct: float = 50.0 + + +# --------------------------------------------------------------------------- +# Outlier classification rules +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class OutlierRule: + """A single outlier classification rule.""" + + cause: OutlierCause + description: str + required_data: list[str] + match_condition: str + applies_to: str = "acpf" + + +@dataclass(frozen=True) +class OutlierClassificationConfig: + """Complete outlier classification configuration.""" + + rules: list[OutlierRule] + max_classified_fraction: float = 0.10 + max_unclassified_fraction: float = 0.02 + + +# --------------------------------------------------------------------------- +# Voltage-level informational breakdown +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class VoltageLevelTier: + """Informational voltage-level tier for detailed results breakdown.""" + + label: str + min_kv: float + max_kv: float + + +# --------------------------------------------------------------------------- +# Default constants +# --------------------------------------------------------------------------- + + +DEFAULT_OUTLIER_RULES: list[OutlierRule] = [ + OutlierRule( + cause=OutlierCause.SWITCHED_SHUNT, + description=( + "Bus has a switched shunt device in the intermediate format. " + "Discrete switching step differences between solvers produce " + "VM deviations of 0.002-0.010 p.u. that are legitimate " + "solver-variation artifacts." + ), + required_data=["bus", "switched_shunt"], + match_condition="has_switched_shunt(bus)", + applies_to="acpf", + ), + OutlierRule( + cause=OutlierCause.Q_LIMIT, + description=( + "Bus has a generator at or near a reactive power limit " + "(|Q - Qmax| < 1.0 MVAr or |Q - Qmin| < 1.0 MVAr in the ACPF " + "reference). Different Q-limit enforcement sequences across " + "solvers produce different voltage setpoints at PV-to-PQ " + "transitioned buses." + ), + required_data=["bus", "generator"], + match_condition="generator_at_q_limit(bus, tolerance_mvar=1.0)", + applies_to="acpf", + ), + OutlierRule( + cause=OutlierCause.SLACK_DISTRIBUTION, + description=( + "Bus is the slack bus (type=3) or within 2 branches of the " + "slack bus in the network graph. Slack bus power absorption " + "differs between solvers, causing VA deviations that propagate " + "to electrically nearby buses." + ), + required_data=["bus", "branch"], + match_condition="is_slack_or_neighbor(bus, max_hops=2)", + applies_to="both", + ), + OutlierRule( + cause=OutlierCause.TAP_POSITION, + description=( + "Bus is the regulated bus (CONT field) of an in-service " + "tap-changing transformer. Different tap optimization " + "algorithms produce different tap positions, causing VM " + "deviations at the regulated bus." + ), + required_data=["bus", "transformer"], + match_condition="is_tap_regulated_bus(bus)", + applies_to="acpf", + ), + OutlierRule( + cause=OutlierCause.ISLAND_BOUNDARY, + description=( + "Bus is at the boundary of a weakly connected subnetwork " + "(network degree <= 2 and base_kv < 69 kV). Low-voltage " + "radial boundary buses are highly sensitive to upstream " + "modeling differences." + ), + required_data=["bus", "branch"], + match_condition="is_island_boundary(bus, max_degree=2, max_kv=69.0)", + applies_to="both", + ), +] + + +DEFAULT_VOLTAGE_TIERS: list[VoltageLevelTier] = [ + VoltageLevelTier( + label="transmission_230kv_plus", + min_kv=230.0, + max_kv=float("inf"), + ), + VoltageLevelTier( + label="subtransmission_69_to_229kv", + min_kv=69.0, + max_kv=230.0, + ), + VoltageLevelTier( + label="distribution_below_69kv", + min_kv=0.0, + max_kv=69.0, + ), +] + + +# --------------------------------------------------------------------------- +# Top-level pass condition specification +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class PassConditionSpec: + """Complete pass condition specification for ACPF and DCPF verification.""" + + version: str = "1.0.0" + + acpf_aggregate: ACPFAggregateThresholds = field(default_factory=ACPFAggregateThresholds) + acpf_hard_fail: ACPFHardFailThresholds = field(default_factory=ACPFHardFailThresholds) + dcpf_aggregate: DCPFAggregateThresholds = field(default_factory=DCPFAggregateThresholds) + dcpf_hard_fail: DCPFHardFailThresholds = field(default_factory=DCPFHardFailThresholds) + outlier_classification: OutlierClassificationConfig = field( + default_factory=lambda: OutlierClassificationConfig(rules=DEFAULT_OUTLIER_RULES) + ) + voltage_level_tiers: list[VoltageLevelTier] = field( + default_factory=lambda: list(DEFAULT_VOLTAGE_TIERS) + ) + bus_exclusion_registry_path: str = "data/fnm/reference/excluded_buses.json" + acpf_reference_dir: str = "data/fnm/reference/acpf/" + dcpf_reference_dir: str = "data/fnm/reference/dcpf/" + + +# --------------------------------------------------------------------------- +# Verdict structures +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class MetricResult: + """Result of a single metric evaluation.""" + + metric_name: str + passed: bool + value: float + threshold: float + detail: str + + +@dataclass(frozen=True) +class HardFailResult: + """Result of a hard-fail check.""" + + check_name: str + triggered: bool + value: float + threshold: float + detail: str + + +@dataclass(frozen=True) +class OutlierSummary: + """Summary of outlier classification results.""" + + total_outliers: int + classified_count: int + unclassified_count: int + by_cause: dict[str, int] + classified_fraction: float + unclassified_fraction: float + classified_warning: bool + unclassified_warning: bool + + +@dataclass(frozen=True) +class VoltageLevelBreakdown: + """Per-voltage-level-tier metric breakdown (informational).""" + + tier_label: str + bus_count: int + passing_count: int + passing_fraction: float + mean_vm_deviation_pu: float | None + mean_va_deviation_deg: float + + +@dataclass(frozen=True) +class VerificationVerdict: + """Complete verification verdict for one analysis type (ACPF or DCPF).""" + + analysis_type: str + overall_pass: bool + hard_fail: bool + aggregate_metrics: list[MetricResult] + hard_fail_checks: list[HardFailResult] + outlier_summary: OutlierSummary | None + voltage_level_breakdown: list[VoltageLevelBreakdown] + total_non_excluded_buses: int + total_in_service_branches: int + + +# --------------------------------------------------------------------------- +# Specification generation +# --------------------------------------------------------------------------- + + +def build_pass_condition_spec() -> PassConditionSpec: + """Build the complete pass condition specification with default thresholds. + + Returns: + A fully populated PassConditionSpec. + """ + return PassConditionSpec() + + +def spec_to_dict(spec: PassConditionSpec) -> dict: + """Convert a PassConditionSpec to a JSON-serializable dict. + + Recursively converts all dataclass fields. Enum values are serialized + as their string values. ``float('inf')`` is serialized as ``null``. + + Args: + spec: The specification to serialize. + + Returns: + A dict safe for ``json.dumps()``. + """ + # Build the JSON structure matching the PRD schema + outlier_rules_json = [] + for priority, rule in enumerate(spec.outlier_classification.rules, start=1): + outlier_rules_json.append( + { + "priority": priority, + "cause": rule.cause.value, + "description": rule.description, + "match_condition": rule.match_condition, + "required_data": rule.required_data, + "applies_to": rule.applies_to, + } + ) + + tiers_json = [] + for tier in spec.voltage_level_tiers: + tiers_json.append( + { + "label": tier.label, + "min_kv": tier.min_kv, + "max_kv_exclusive": None if math.isinf(tier.max_kv) else tier.max_kv, + } + ) + + return { + "$schema_version": spec.version, + "$description": ( + "Pass condition definitions for ACPF and DCPF FNM verification. " + "Machine-readable specification consumed by evaluate-tool agents " + "at runtime." + ), + "bus_exclusion": { + "registry_path": spec.bus_exclusion_registry_path, + "description": ( + "Path to the D1 bus exclusion registry. All buses listed in " + "this file are excluded from metric denominators. Exclusion " + "reasons: ide_4_isolated, vm_zero_deenergized, " + "disconnected_island." + ), + "usage": ( + "Load excluded_buses[].bus_number to build the exclusion set. " + "Metric denominators = total_buses - len(exclusion_set)." + ), + }, + "acpf": { + "reference_dir": spec.acpf_reference_dir, + "reference_files": { + "buses": "buses_acpf.csv", + "branches": "branches_acpf.csv", + "generators": "generators_acpf.csv", + "summary": "summary_acpf.json", + }, + "aggregate": { + "description": ( + "Primary pass gate. A bus passes if BOTH VM and VA " + "deviations are within tolerance. The fraction of passing " + "buses must exceed the minimum threshold." + ), + "min_passing_fraction": spec.acpf_aggregate.min_passing_fraction, + "vm_tolerance_pu": spec.acpf_aggregate.vm_tolerance_pu, + "va_tolerance_deg": spec.acpf_aggregate.va_tolerance_deg, + "bus_pass_condition": ( + f"|VM_tool - VM_ref| < {spec.acpf_aggregate.vm_tolerance_pu} " + f"AND |VA_tool - VA_ref| < {spec.acpf_aggregate.va_tolerance_deg}" + ), + "metric": ( + f"count(passing_buses) / count(non_excluded_buses) >= " + f"{spec.acpf_aggregate.min_passing_fraction}" + ), + }, + "hard_fail": { + "description": ( + "Any single condition triggers unconditional test failure, " + "regardless of aggregate statistics." + ), + "conditions": [ + { + "name": "excessive_failing_fraction", + "description": ( + f"More than {spec.acpf_hard_fail.max_failing_fraction * 100:.0f}% " + "of non-excluded buses fail the aggregate tolerance." + ), + "condition": ( + f"count(failing_buses) / count(non_excluded_buses) > " + f"{spec.acpf_hard_fail.max_failing_fraction}" + ), + "threshold": spec.acpf_hard_fail.max_failing_fraction, + }, + { + "name": "extreme_vm_deviation", + "description": ( + f"Any single bus has VM deviation exceeding " + f"{spec.acpf_hard_fail.vm_max_deviation_pu} p.u. " + "Indicates fundamental voltage error, not solver variation." + ), + "condition": ( + f"max(|VM_tool - VM_ref|) > {spec.acpf_hard_fail.vm_max_deviation_pu}" + ), + "threshold_pu": spec.acpf_hard_fail.vm_max_deviation_pu, + }, + { + "name": "extreme_va_deviation", + "description": ( + f"Any single bus has VA deviation exceeding " + f"{spec.acpf_hard_fail.va_max_deviation_deg} degrees. " + "Indicates topology or connectivity error." + ), + "condition": ( + f"max(|VA_tool - VA_ref|) > {spec.acpf_hard_fail.va_max_deviation_deg}" + ), + "threshold_deg": spec.acpf_hard_fail.va_max_deviation_deg, + }, + ], + }, + "outlier_classification": { + "description": ( + "Buses that fail the aggregate tolerance are classified by " + "probable cause. Classification does not change pass/fail " + "-- it explains why outliers exist and whether they indicate " + "ingestion error vs. expected solver variation." + ), + "evaluation_order": ( + "Rules are evaluated in the order listed. First matching " + "rule assigns the primary cause. A bus may match multiple " + "rules; only the first (highest priority) is assigned." + ), + "rules": outlier_rules_json, + "warning_thresholds": { + "max_classified_fraction": ( + spec.outlier_classification.max_classified_fraction + ), + "max_classified_description": ( + f"If classified outliers (all causes except unclassified) " + f"exceed " + f"{spec.outlier_classification.max_classified_fraction * 100:.0f}% " + f"of non-excluded buses, emit a warning." + ), + "max_unclassified_fraction": ( + spec.outlier_classification.max_unclassified_fraction + ), + "max_unclassified_description": ( + f"If unclassified outliers exceed " + f"{spec.outlier_classification.max_unclassified_fraction * 100:.0f}% " + f"of non-excluded buses, emit a warning." + ), + }, + }, + }, + "dcpf": { + "reference_dir": spec.dcpf_reference_dir, + "reference_files": { + "buses": "buses_dcpf.csv", + "branches": "branches_dcpf.csv", + "summary": "summary_dcpf.json", + }, + "aggregate": { + "description": ( + "Primary pass gate. Two independent metrics: bus angles and branch flows." + ), + "bus_angle": { + "description": ( + "Fraction of non-excluded buses with VA deviation within tolerance." + ), + "min_passing_fraction": (spec.dcpf_aggregate.min_bus_passing_fraction), + "va_tolerance_deg": spec.dcpf_aggregate.va_tolerance_deg, + "bus_pass_condition": ( + f"|VA_tool - VA_ref| < {spec.dcpf_aggregate.va_tolerance_deg}" + ), + "metric": ( + f"count(passing_buses) / count(non_excluded_buses) >= " + f"{spec.dcpf_aggregate.min_bus_passing_fraction}" + ), + }, + "branch_flow": { + "description": ( + "Fraction of in-service branches with P deviation within tolerance." + ), + "min_passing_fraction": (spec.dcpf_aggregate.min_branch_passing_fraction), + "p_tolerance_pct": spec.dcpf_aggregate.p_tolerance_pct, + "p_base_floor_mw": spec.dcpf_aggregate.p_base_floor_mw, + "deviation_formula": ( + f"|P_tool - P_ref| / max(|P_ref|, " + f"{spec.dcpf_aggregate.p_base_floor_mw}) * 100" + ), + "branch_pass_condition": ( + f"deviation_pct < {spec.dcpf_aggregate.p_tolerance_pct}" + ), + "metric": ( + f"count(passing_branches) / count(in_service_branches) >= " + f"{spec.dcpf_aggregate.min_branch_passing_fraction}" + ), + }, + }, + "hard_fail": { + "description": ("Any single condition triggers unconditional test failure."), + "conditions": [ + { + "name": "excessive_bus_failing_fraction", + "description": ( + f"More than " + f"{spec.dcpf_hard_fail.max_bus_failing_fraction * 100:.0f}% " + "of non-excluded buses fail the VA tolerance." + ), + "condition": ( + f"count(failing_buses) / count(non_excluded_buses) > " + f"{spec.dcpf_hard_fail.max_bus_failing_fraction}" + ), + "threshold": spec.dcpf_hard_fail.max_bus_failing_fraction, + }, + { + "name": "excessive_branch_failing_fraction", + "description": ( + f"More than " + f"{spec.dcpf_hard_fail.max_branch_failing_fraction * 100:.0f}% " + "of in-service branches fail the P tolerance." + ), + "condition": ( + f"count(failing_branches) / count(in_service_branches) > " + f"{spec.dcpf_hard_fail.max_branch_failing_fraction}" + ), + "threshold": spec.dcpf_hard_fail.max_branch_failing_fraction, + }, + { + "name": "extreme_branch_flow_deviation", + "description": ( + f"Any single branch has P deviation exceeding " + f"{spec.dcpf_hard_fail.p_max_deviation_pct}%. " + "Indicates topology or impedance error." + ), + "condition": ( + f"max(deviation_pct) > {spec.dcpf_hard_fail.p_max_deviation_pct}" + ), + "threshold_pct": spec.dcpf_hard_fail.p_max_deviation_pct, + }, + ], + }, + }, + "voltage_level_tiers": { + "description": ( + "Informational voltage-level breakdown in verification results. " + "Not a pass/fail gate -- the primary pass condition uses a " + "single tolerance for all buses. This breakdown helps diagnose " + "systematic voltage-level-correlated errors." + ), + "tiers": tiers_json, + }, + } + + +def write_json(spec: PassConditionSpec, output_path: Path) -> None: + """Write the pass condition specification as a JSON file. + + Args: + spec: The specification to write. + output_path: Path to the output JSON file. + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + data = spec_to_dict(spec) + output_path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + + +def write_markdown(spec: PassConditionSpec, output_path: Path) -> None: + """Write the pass condition specification as a human-readable markdown file. + + Args: + spec: The specification to render. + output_path: Path to the output markdown file. + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + + lines: list[str] = [] + _a = lines.append + + _a("# Pass Condition Definitions") + _a("") + _a(f"**Schema version:** {spec.version}") + _a("") + _a("This document defines the acceptance criteria for verifying a tool's") + _a("FNM power flow results against the Phase 3 reference solutions.") + _a("") + + # -- Summary table ------------------------------------------------------- + _a("## Summary of Thresholds") + _a("") + _a("| Analysis | Metric | Threshold | Unit |") + _a("|----------|--------|-----------|------|") + _a( + f"| ACPF | Min passing fraction | " + f"{spec.acpf_aggregate.min_passing_fraction * 100:.0f}% | — |" + ) + _a(f"| ACPF | VM tolerance | {spec.acpf_aggregate.vm_tolerance_pu} | p.u. |") + _a(f"| ACPF | VA tolerance | {spec.acpf_aggregate.va_tolerance_deg} | degrees |") + _a( + f"| DCPF | Min bus passing fraction | " + f"{spec.dcpf_aggregate.min_bus_passing_fraction * 100:.0f}% | — |" + ) + _a(f"| DCPF | VA tolerance | {spec.dcpf_aggregate.va_tolerance_deg} | degrees |") + _a( + f"| DCPF | Min branch passing fraction | " + f"{spec.dcpf_aggregate.min_branch_passing_fraction * 100:.0f}% | — |" + ) + _a(f"| DCPF | P tolerance | {spec.dcpf_aggregate.p_tolerance_pct} | % |") + _a(f"| DCPF | P base floor | {spec.dcpf_aggregate.p_base_floor_mw} | MW |") + _a("") + + # -- ACPF ---------------------------------------------------------------- + _a("## ACPF Pass Conditions") + _a("") + _a("### Aggregate") + _a("") + _a( + f"A bus passes if **both** |VM_tool - VM_ref| < " + f"{spec.acpf_aggregate.vm_tolerance_pu} p.u. **and** " + f"|VA_tool - VA_ref| < {spec.acpf_aggregate.va_tolerance_deg} degrees." + ) + _a( + f"At least {spec.acpf_aggregate.min_passing_fraction * 100:.0f}% of " + f"non-excluded buses must pass for the ACPF gate to succeed." + ) + _a("") + _a("### Hard-Fail Conditions") + _a("") + _a( + "Any single hard-fail condition triggers unconditional test failure, " + "regardless of aggregate statistics." + ) + _a("") + _a( + f"1. **Excessive failing fraction:** More than " + f"{spec.acpf_hard_fail.max_failing_fraction * 100:.0f}% of " + f"non-excluded buses fail the aggregate tolerance." + ) + _a( + f"2. **Extreme VM deviation:** Any bus has VM deviation > " + f"{spec.acpf_hard_fail.vm_max_deviation_pu} p.u." + ) + _a( + f"3. **Extreme VA deviation:** Any bus has VA deviation > " + f"{spec.acpf_hard_fail.va_max_deviation_deg} degrees." + ) + _a("") + + # -- DCPF ---------------------------------------------------------------- + _a("## DCPF Pass Conditions") + _a("") + _a("### Aggregate") + _a("") + _a("Two independent metrics must both pass:") + _a("") + _a( + f"1. **Bus angles:** >= " + f"{spec.dcpf_aggregate.min_bus_passing_fraction * 100:.0f}% of " + f"non-excluded buses must have |VA_tool - VA_ref| < " + f"{spec.dcpf_aggregate.va_tolerance_deg} degree(s)." + ) + _a( + f"2. **Branch flows:** >= " + f"{spec.dcpf_aggregate.min_branch_passing_fraction * 100:.0f}% of " + f"in-service branches must have branch flow deviation < " + f"{spec.dcpf_aggregate.p_tolerance_pct}%." + ) + _a("") + _a("### Branch Flow Deviation Formula") + _a("") + _a("```") + _a("deviation_pct = |P_tool - P_ref| / P_base * 100") + _a("") + _a(f"where P_base = max(|P_ref|, {spec.dcpf_aggregate.p_base_floor_mw})") + _a("```") + _a("") + _a("**Worked example:** P_ref = 200 MW, P_tool = 210 MW.") + _a(f"P_base = max(200, {spec.dcpf_aggregate.p_base_floor_mw}) = 200.") + _a("deviation_pct = |210 - 200| / 200 * 100 = 5.0%. This branch passes") + _a(f"(5.0 < {spec.dcpf_aggregate.p_tolerance_pct}).") + _a("") + _a("### Hard-Fail Conditions") + _a("") + _a( + f"1. **Excessive bus failing fraction:** More than " + f"{spec.dcpf_hard_fail.max_bus_failing_fraction * 100:.0f}% of " + f"non-excluded buses fail VA tolerance." + ) + _a( + f"2. **Excessive branch failing fraction:** More than " + f"{spec.dcpf_hard_fail.max_branch_failing_fraction * 100:.0f}% of " + f"in-service branches fail P tolerance." + ) + _a( + f"3. **Extreme branch flow deviation:** Any branch has P deviation > " + f"{spec.dcpf_hard_fail.p_max_deviation_pct}%." + ) + _a("") + + # -- Outlier classification ---------------------------------------------- + _a("## Outlier Classification Rules") + _a("") + _a( + "Buses that fail the aggregate tolerance are classified by probable " + "cause. Classification does not change pass/fail -- it explains why " + "outliers exist." + ) + _a("") + _a("Rules are evaluated in priority order; first match wins:") + _a("") + for i, rule in enumerate(spec.outlier_classification.rules, start=1): + _a(f"### Rule {i}: `{rule.cause.value}`") + _a("") + _a(f"- **Applies to:** {rule.applies_to}") + _a(f"- **Condition:** `{rule.match_condition}`") + _a(f"- **Required data:** {', '.join(rule.required_data)}") + _a(f"- **Description:** {rule.description}") + _a("") + + _a("### Warning Thresholds") + _a("") + _a( + f"- **Max classified fraction:** " + f"{spec.outlier_classification.max_classified_fraction * 100:.0f}% " + f"of non-excluded buses." + ) + _a( + f"- **Max unclassified fraction:** " + f"{spec.outlier_classification.max_unclassified_fraction * 100:.0f}% " + f"of non-excluded buses." + ) + _a("") + + # -- Voltage level tiers ------------------------------------------------- + _a("## Voltage Level Tiers") + _a("") + _a("Informational voltage level breakdown in verification results. Not a pass/fail gate.") + _a("") + _a("| Tier | Min kV (incl.) | Max kV (excl.) |") + _a("|------|---------------|----------------|") + for tier in spec.voltage_level_tiers: + max_kv_str = "∞" if math.isinf(tier.max_kv) else f"{tier.max_kv}" + _a(f"| {tier.label} | {tier.min_kv} | {max_kv_str} |") + _a("") + + # -- Cross-references --------------------------------------------------- + _a("## Cross-References") + _a("") + _a(f"- **Bus exclusion registry (D1):** `{spec.bus_exclusion_registry_path}`") + _a(f"- **ACPF reference (D2):** `{spec.acpf_reference_dir}`") + _a(f"- **DCPF reference (D3):** `{spec.dcpf_reference_dir}`") + _a("- **DCPF-vs-ACPF characterization (D4):** Validates DCPF thresholds") + _a("") + + output_path.write_text("\n".join(lines), encoding="utf-8") + + +# --------------------------------------------------------------------------- +# Specification loading +# --------------------------------------------------------------------------- + + +def load_spec(json_path: Path) -> PassConditionSpec: + """Load a pass condition specification from a JSON file. + + Args: + json_path: Path to the pass conditions JSON file. + + Returns: + A PassConditionSpec populated from the JSON. + + Raises: + FileNotFoundError: If the JSON file does not exist. + ValueError: If required fields are missing or values are invalid. + KeyError: If the JSON structure does not match the expected schema. + """ + if not json_path.exists(): + raise FileNotFoundError(f"Pass conditions JSON not found: {json_path}") + + data = json.loads(json_path.read_text(encoding="utf-8")) + + # Version check + version = data.get("$schema_version", "") + if version != "1.0.0": + raise ValueError(f"Schema version mismatch: expected '1.0.0', got '{version}'") + + # Parse ACPF aggregate + acpf_agg = data["acpf"]["aggregate"] + acpf_aggregate = ACPFAggregateThresholds( + min_passing_fraction=float(acpf_agg["min_passing_fraction"]), + vm_tolerance_pu=float(acpf_agg["vm_tolerance_pu"]), + va_tolerance_deg=float(acpf_agg["va_tolerance_deg"]), + ) + + # Parse ACPF hard-fail + acpf_hf_conditions = data["acpf"]["hard_fail"]["conditions"] + acpf_hf_map = {c["name"]: c for c in acpf_hf_conditions} + acpf_hard_fail = ACPFHardFailThresholds( + max_failing_fraction=float(acpf_hf_map["excessive_failing_fraction"]["threshold"]), + vm_max_deviation_pu=float(acpf_hf_map["extreme_vm_deviation"]["threshold_pu"]), + va_max_deviation_deg=float(acpf_hf_map["extreme_va_deviation"]["threshold_deg"]), + ) + + # Parse DCPF aggregate + dcpf_agg = data["dcpf"]["aggregate"] + dcpf_aggregate = DCPFAggregateThresholds( + min_bus_passing_fraction=float(dcpf_agg["bus_angle"]["min_passing_fraction"]), + va_tolerance_deg=float(dcpf_agg["bus_angle"]["va_tolerance_deg"]), + min_branch_passing_fraction=float(dcpf_agg["branch_flow"]["min_passing_fraction"]), + p_tolerance_pct=float(dcpf_agg["branch_flow"]["p_tolerance_pct"]), + p_base_floor_mw=float(dcpf_agg["branch_flow"]["p_base_floor_mw"]), + ) + + # Parse DCPF hard-fail + dcpf_hf_conditions = data["dcpf"]["hard_fail"]["conditions"] + dcpf_hf_map = {c["name"]: c for c in dcpf_hf_conditions} + dcpf_hard_fail = DCPFHardFailThresholds( + max_bus_failing_fraction=float(dcpf_hf_map["excessive_bus_failing_fraction"]["threshold"]), + max_branch_failing_fraction=float( + dcpf_hf_map["excessive_branch_failing_fraction"]["threshold"] + ), + p_max_deviation_pct=float(dcpf_hf_map["extreme_branch_flow_deviation"]["threshold_pct"]), + ) + + # Parse outlier rules + outlier_data = data["acpf"]["outlier_classification"] + rules: list[OutlierRule] = [] + for r in outlier_data["rules"]: + rules.append( + OutlierRule( + cause=OutlierCause(r["cause"]), + description=r["description"], + required_data=r["required_data"], + match_condition=r["match_condition"], + applies_to=r.get("applies_to", "acpf"), + ) + ) + warning_thresholds = outlier_data["warning_thresholds"] + outlier_config = OutlierClassificationConfig( + rules=rules, + max_classified_fraction=float(warning_thresholds["max_classified_fraction"]), + max_unclassified_fraction=float(warning_thresholds["max_unclassified_fraction"]), + ) + + # Parse voltage tiers + tiers_data = data["voltage_level_tiers"]["tiers"] + voltage_tiers: list[VoltageLevelTier] = [] + for t in tiers_data: + max_kv = t.get("max_kv_exclusive") + voltage_tiers.append( + VoltageLevelTier( + label=t["label"], + min_kv=float(t["min_kv"]), + max_kv=float("inf") if max_kv is None else float(max_kv), + ) + ) + + # Validate plausible ranges + for frac_name, frac_val in [ + ("acpf.min_passing_fraction", acpf_aggregate.min_passing_fraction), + ("acpf.max_failing_fraction", acpf_hard_fail.max_failing_fraction), + ("dcpf.min_bus_passing_fraction", dcpf_aggregate.min_bus_passing_fraction), + ("dcpf.min_branch_passing_fraction", dcpf_aggregate.min_branch_passing_fraction), + ("dcpf.max_bus_failing_fraction", dcpf_hard_fail.max_bus_failing_fraction), + ("dcpf.max_branch_failing_fraction", dcpf_hard_fail.max_branch_failing_fraction), + ]: + if not (0.0 <= frac_val <= 1.0): + raise ValueError(f"{frac_name} must be in [0, 1], got {frac_val}") + + for tol_name, tol_val in [ + ("acpf.vm_tolerance_pu", acpf_aggregate.vm_tolerance_pu), + ("acpf.va_tolerance_deg", acpf_aggregate.va_tolerance_deg), + ("dcpf.va_tolerance_deg", dcpf_aggregate.va_tolerance_deg), + ("dcpf.p_tolerance_pct", dcpf_aggregate.p_tolerance_pct), + ("dcpf.p_base_floor_mw", dcpf_aggregate.p_base_floor_mw), + ]: + if tol_val <= 0: + raise ValueError(f"{tol_name} must be > 0, got {tol_val}") + + return PassConditionSpec( + version=version, + acpf_aggregate=acpf_aggregate, + acpf_hard_fail=acpf_hard_fail, + dcpf_aggregate=dcpf_aggregate, + dcpf_hard_fail=dcpf_hard_fail, + outlier_classification=outlier_config, + voltage_level_tiers=voltage_tiers, + bus_exclusion_registry_path=data["bus_exclusion"]["registry_path"], + acpf_reference_dir=data["acpf"]["reference_dir"], + dcpf_reference_dir=data["dcpf"]["reference_dir"], + ) + + +# --------------------------------------------------------------------------- +# Voltage-level tier classification helper +# --------------------------------------------------------------------------- + + +def _classify_bus_tier( + base_kv: float, + tiers: Sequence[VoltageLevelTier], +) -> str | None: + """Return the tier label for a bus based on its base_kv.""" + for tier in tiers: + if tier.min_kv <= base_kv < tier.max_kv: + return tier.label + return None + + +def _compute_voltage_level_breakdown( + bus_deviations: list[dict], + tiers: Sequence[VoltageLevelTier], + bus_base_kv: dict[int, float], + has_vm: bool, + vm_tol: float, + va_tol: float, +) -> list[VoltageLevelBreakdown]: + """Compute per-voltage-level-tier metric breakdown. + + Args: + bus_deviations: List of dicts with 'bus', 'vm_dev' (or None), 'va_dev', 'passed'. + tiers: Voltage level tiers. + bus_base_kv: Bus number to base kV mapping. + has_vm: True for ACPF (includes VM), False for DCPF. + vm_tol: VM tolerance (used only for description, not re-evaluation). + va_tol: VA tolerance. + + Returns: + List of VoltageLevelBreakdown. + """ + # Group buses by tier + tier_data: dict[str, list[dict]] = {tier.label: [] for tier in tiers} + + for bd in bus_deviations: + bus_num = bd["bus"] + kv = bus_base_kv.get(bus_num, 0.0) + tier_label = _classify_bus_tier(kv, tiers) + if tier_label is not None and tier_label in tier_data: + tier_data[tier_label].append(bd) + + result: list[VoltageLevelBreakdown] = [] + for tier in tiers: + entries = tier_data[tier.label] + bus_count = len(entries) + if bus_count == 0: + result.append( + VoltageLevelBreakdown( + tier_label=tier.label, + bus_count=0, + passing_count=0, + passing_fraction=0.0, + mean_vm_deviation_pu=None, + mean_va_deviation_deg=0.0, + ) + ) + continue + + passing_count = sum(1 for e in entries if e["passed"]) + passing_fraction = passing_count / bus_count + + mean_va = sum(e["va_dev"] for e in entries) / bus_count + + mean_vm: float | None = None + if has_vm: + mean_vm = sum(e.get("vm_dev", 0.0) for e in entries) / bus_count + + result.append( + VoltageLevelBreakdown( + tier_label=tier.label, + bus_count=bus_count, + passing_count=passing_count, + passing_fraction=passing_fraction, + mean_vm_deviation_pu=mean_vm, + mean_va_deviation_deg=mean_va, + ) + ) + + return result + + +# --------------------------------------------------------------------------- +# ACPF verification evaluation +# --------------------------------------------------------------------------- + + +def evaluate_acpf( + spec: PassConditionSpec, + tool_buses: list[dict], + ref_buses: list[dict], + excluded_bus_numbers: set[int], + bus_base_kv: dict[int, float], + classify_outliers: bool = True, + intermediate_data: dict[str, list[dict]] | None = None, +) -> VerificationVerdict: + """Evaluate a tool's ACPF results against the reference. + + Args: + spec: The pass condition specification. + tool_buses: Tool's ACPF bus results. Each dict has keys: + ``bus`` (int), ``VM`` (float), ``VA`` (float). + ref_buses: Reference ACPF bus results (same schema). + excluded_bus_numbers: Set of excluded bus numbers from D1. + bus_base_kv: Mapping from bus number to base kV. + classify_outliers: If True and intermediate_data is provided, + classify outlier buses by cause. + intermediate_data: Optional dict mapping table names to row lists + for outlier classification. + + Returns: + A VerificationVerdict for ACPF. + """ + agg = spec.acpf_aggregate + hf = spec.acpf_hard_fail + + # Filter excluded buses + ref_filtered = [b for b in ref_buses if b["bus"] not in excluded_bus_numbers] + tool_map = {b["bus"]: b for b in tool_buses if b["bus"] not in excluded_bus_numbers} + + total_non_excluded = len(ref_filtered) + + # Handle pathological case: all buses excluded + if total_non_excluded == 0: + return VerificationVerdict( + analysis_type="acpf", + overall_pass=False, + hard_fail=True, + aggregate_metrics=[ + MetricResult( + metric_name="acpf_vm_va_aggregate", + passed=False, + value=0.0, + threshold=agg.min_passing_fraction, + detail="No non-excluded buses to evaluate.", + ) + ], + hard_fail_checks=[], + outlier_summary=None, + voltage_level_breakdown=[], + total_non_excluded_buses=0, + total_in_service_branches=0, + ) + + # Compute per-bus deviations + bus_deviations: list[dict] = [] + max_vm_dev = 0.0 + max_va_dev = 0.0 + max_vm_bus = -1 + max_va_bus = -1 + passing_count = 0 + missing_count = 0 + + for ref_bus in ref_filtered: + bus_num = ref_bus["bus"] + tool_bus = tool_map.get(bus_num) + + if tool_bus is None: + # Missing bus counts as failing with infinite deviation + missing_count += 1 + bus_deviations.append( + { + "bus": bus_num, + "vm_dev": float("inf"), + "va_dev": float("inf"), + "passed": False, + } + ) + # Update max deviations with a large value + if float("inf") > max_vm_dev: + max_vm_dev = float("inf") + max_vm_bus = bus_num + if float("inf") > max_va_dev: + max_va_dev = float("inf") + max_va_bus = bus_num + continue + + vm_dev = abs(tool_bus["VM"] - ref_bus["VM"]) + va_dev = abs(tool_bus["VA"] - ref_bus["VA"]) + passed = vm_dev < agg.vm_tolerance_pu and va_dev < agg.va_tolerance_deg + + if passed: + passing_count += 1 + + bus_deviations.append( + { + "bus": bus_num, + "vm_dev": vm_dev, + "va_dev": va_dev, + "passed": passed, + } + ) + + if vm_dev > max_vm_dev: + max_vm_dev = vm_dev + max_vm_bus = bus_num + if va_dev > max_va_dev: + max_va_dev = va_dev + max_va_bus = bus_num + + passing_fraction = passing_count / total_non_excluded + failing_count = total_non_excluded - passing_count + failing_fraction = failing_count / total_non_excluded + + # Aggregate metric + agg_passed = passing_fraction >= agg.min_passing_fraction + aggregate_metrics = [ + MetricResult( + metric_name="acpf_vm_va_aggregate", + passed=agg_passed, + value=passing_fraction, + threshold=agg.min_passing_fraction, + detail=( + f"{passing_count}/{total_non_excluded} buses pass " + f"(VM<{agg.vm_tolerance_pu} AND VA<{agg.va_tolerance_deg}). " + f"Fraction: {passing_fraction:.4f}, " + f"required: {agg.min_passing_fraction}." + ), + ) + ] + + # Hard-fail checks + hf_fraction = HardFailResult( + check_name="excessive_failing_fraction", + triggered=failing_fraction > hf.max_failing_fraction, + value=failing_fraction, + threshold=hf.max_failing_fraction, + detail=( + f"Failing fraction: {failing_fraction:.4f} (threshold: {hf.max_failing_fraction})." + ), + ) + hf_vm = HardFailResult( + check_name="extreme_vm_deviation", + triggered=max_vm_dev > hf.vm_max_deviation_pu, + value=max_vm_dev, + threshold=hf.vm_max_deviation_pu, + detail=( + f"Max VM deviation: {max_vm_dev:.6f} p.u. at bus {max_vm_bus} " + f"(threshold: {hf.vm_max_deviation_pu})." + ), + ) + hf_va = HardFailResult( + check_name="extreme_va_deviation", + triggered=max_va_dev > hf.va_max_deviation_deg, + value=max_va_dev, + threshold=hf.va_max_deviation_deg, + detail=( + f"Max VA deviation: {max_va_dev:.6f} deg at bus {max_va_bus} " + f"(threshold: {hf.va_max_deviation_deg})." + ), + ) + hard_fail_checks = [hf_fraction, hf_vm, hf_va] + any_hard_fail = any(c.triggered for c in hard_fail_checks) + + # Outlier classification + outlier_summary: OutlierSummary | None = None + if classify_outliers and intermediate_data is not None: + failing_buses = [bd for bd in bus_deviations if not bd["passed"]] + cause_counts: Counter[str] = Counter() + for fb in failing_buses: + cause = classify_outlier_bus( + bus_number=fb["bus"], + rules=spec.outlier_classification.rules, + intermediate_data=intermediate_data, + bus_base_kv=bus_base_kv, + ) + cause_counts[cause.value] += 1 + + total_outliers = len(failing_buses) + unclassified_count = cause_counts.get(OutlierCause.UNCLASSIFIED.value, 0) + classified_count = total_outliers - unclassified_count + classified_fraction = ( + classified_count / total_non_excluded if total_non_excluded > 0 else 0.0 + ) + unclassified_fraction = ( + unclassified_count / total_non_excluded if total_non_excluded > 0 else 0.0 + ) + + outlier_summary = OutlierSummary( + total_outliers=total_outliers, + classified_count=classified_count, + unclassified_count=unclassified_count, + by_cause=dict(cause_counts), + classified_fraction=classified_fraction, + unclassified_fraction=unclassified_fraction, + classified_warning=( + classified_fraction > spec.outlier_classification.max_classified_fraction + ), + unclassified_warning=( + unclassified_fraction > spec.outlier_classification.max_unclassified_fraction + ), + ) + + # Voltage-level breakdown + vl_breakdown = _compute_voltage_level_breakdown( + bus_deviations=bus_deviations, + tiers=spec.voltage_level_tiers, + bus_base_kv=bus_base_kv, + has_vm=True, + vm_tol=agg.vm_tolerance_pu, + va_tol=agg.va_tolerance_deg, + ) + + overall_pass = agg_passed and not any_hard_fail + + return VerificationVerdict( + analysis_type="acpf", + overall_pass=overall_pass, + hard_fail=any_hard_fail, + aggregate_metrics=aggregate_metrics, + hard_fail_checks=hard_fail_checks, + outlier_summary=outlier_summary, + voltage_level_breakdown=vl_breakdown, + total_non_excluded_buses=total_non_excluded, + total_in_service_branches=0, + ) + + +# --------------------------------------------------------------------------- +# DCPF verification evaluation +# --------------------------------------------------------------------------- + + +def _compute_branch_deviation_pct( + p_tool: float, + p_ref: float, + p_base_floor_mw: float, +) -> float: + """Compute the DCPF branch flow deviation percentage. + + Args: + p_tool: Tool's branch MW flow. + p_ref: Reference branch MW flow. + p_base_floor_mw: Floor for the denominator. + + Returns: + Deviation percentage. + """ + p_base = max(abs(p_ref), p_base_floor_mw) + return abs(p_tool - p_ref) / p_base * 100.0 + + +def evaluate_dcpf( + spec: PassConditionSpec, + tool_buses: list[dict], + ref_buses: list[dict], + tool_branches: list[dict], + ref_branches: list[dict], + excluded_bus_numbers: set[int], + bus_base_kv: dict[int, float], +) -> VerificationVerdict: + """Evaluate a tool's DCPF results against the reference. + + Args: + spec: The pass condition specification. + tool_buses: Tool's DCPF bus results. Each dict has keys: + ``bus`` (int), ``VA`` (float). + ref_buses: Reference DCPF bus results (same schema). + tool_branches: Tool's DCPF branch results. Each dict has keys: + ``from_bus`` (int), ``to_bus`` (int), ``ckt`` (str), + ``P_flow_MW`` (float). + ref_branches: Reference DCPF branch results (same schema). + excluded_bus_numbers: Set of excluded bus numbers from D1. + bus_base_kv: Mapping from bus number to base kV. + + Returns: + A VerificationVerdict for DCPF. + """ + agg = spec.dcpf_aggregate + hf = spec.dcpf_hard_fail + + # Filter excluded buses + ref_filtered = [b for b in ref_buses if b["bus"] not in excluded_bus_numbers] + tool_map = {b["bus"]: b for b in tool_buses if b["bus"] not in excluded_bus_numbers} + + total_non_excluded = len(ref_filtered) + + # Handle all-excluded case + if total_non_excluded == 0: + return VerificationVerdict( + analysis_type="dcpf", + overall_pass=False, + hard_fail=True, + aggregate_metrics=[], + hard_fail_checks=[], + outlier_summary=None, + voltage_level_breakdown=[], + total_non_excluded_buses=0, + total_in_service_branches=len(ref_branches), + ) + + # -- Bus VA deviations --------------------------------------------------- + bus_deviations: list[dict] = [] + bus_passing_count = 0 + + for ref_bus in ref_filtered: + bus_num = ref_bus["bus"] + tool_bus = tool_map.get(bus_num) + + if tool_bus is None: + bus_deviations.append( + { + "bus": bus_num, + "va_dev": float("inf"), + "passed": False, + } + ) + continue + + va_dev = abs(tool_bus["VA"] - ref_bus["VA"]) + passed = va_dev < agg.va_tolerance_deg + + if passed: + bus_passing_count += 1 + + bus_deviations.append( + { + "bus": bus_num, + "va_dev": va_dev, + "passed": passed, + } + ) + + bus_passing_fraction = bus_passing_count / total_non_excluded + bus_failing_fraction = 1.0 - bus_passing_fraction + + # -- Branch P deviations ------------------------------------------------- + # Build tool branch lookup with both key orders + def _branch_key(from_bus: int, to_bus: int, ckt: str) -> tuple[int, int, str]: + return (from_bus, to_bus, ckt) + + tool_branch_map: dict[tuple[int, int, str], dict] = {} + for tb in tool_branches: + key = _branch_key(tb["from_bus"], tb["to_bus"], tb["ckt"]) + tool_branch_map[key] = tb + + total_ref_branches = len(ref_branches) + branch_passing_count = 0 + max_p_dev_pct = 0.0 + max_p_dev_branch: str = "" + + for rb in ref_branches: + ref_key = _branch_key(rb["from_bus"], rb["to_bus"], rb["ckt"]) + rev_key = _branch_key(rb["to_bus"], rb["from_bus"], rb["ckt"]) + + tool_br = tool_branch_map.get(ref_key) + negate = False + if tool_br is None: + tool_br = tool_branch_map.get(rev_key) + negate = True + + if tool_br is None: + # Missing branch counts as failing + if 100.0 > max_p_dev_pct: + max_p_dev_pct = 100.0 + max_p_dev_branch = f"({rb['from_bus']}-{rb['to_bus']}-{rb['ckt']})" + continue + + p_tool = -tool_br["P_flow_MW"] if negate else tool_br["P_flow_MW"] + p_ref = rb["P_flow_MW"] + dev_pct = _compute_branch_deviation_pct(p_tool, p_ref, agg.p_base_floor_mw) + + if dev_pct < agg.p_tolerance_pct: + branch_passing_count += 1 + + if dev_pct > max_p_dev_pct: + max_p_dev_pct = dev_pct + max_p_dev_branch = f"({rb['from_bus']}-{rb['to_bus']}-{rb['ckt']})" + + if total_ref_branches > 0: + branch_passing_fraction = branch_passing_count / total_ref_branches + branch_failing_fraction = 1.0 - branch_passing_fraction + else: + branch_passing_fraction = 1.0 + branch_failing_fraction = 0.0 + + # -- Aggregate metrics --------------------------------------------------- + bus_metric_passed = bus_passing_fraction >= agg.min_bus_passing_fraction + branch_metric_passed = branch_passing_fraction >= agg.min_branch_passing_fraction + + aggregate_metrics = [ + MetricResult( + metric_name="dcpf_bus_va_aggregate", + passed=bus_metric_passed, + value=bus_passing_fraction, + threshold=agg.min_bus_passing_fraction, + detail=( + f"{bus_passing_count}/{total_non_excluded} buses pass " + f"(VA<{agg.va_tolerance_deg}). " + f"Fraction: {bus_passing_fraction:.4f}." + ), + ), + MetricResult( + metric_name="dcpf_branch_p_aggregate", + passed=branch_metric_passed, + value=branch_passing_fraction, + threshold=agg.min_branch_passing_fraction, + detail=( + f"{branch_passing_count}/{total_ref_branches} branches pass " + f"(P<{agg.p_tolerance_pct}%). " + f"Fraction: {branch_passing_fraction:.4f}." + ), + ), + ] + + # -- Hard-fail checks ---------------------------------------------------- + hf_bus = HardFailResult( + check_name="excessive_bus_failing_fraction", + triggered=bus_failing_fraction > hf.max_bus_failing_fraction, + value=bus_failing_fraction, + threshold=hf.max_bus_failing_fraction, + detail=( + f"Bus failing fraction: {bus_failing_fraction:.4f} " + f"(threshold: {hf.max_bus_failing_fraction})." + ), + ) + hf_branch = HardFailResult( + check_name="excessive_branch_failing_fraction", + triggered=branch_failing_fraction > hf.max_branch_failing_fraction, + value=branch_failing_fraction, + threshold=hf.max_branch_failing_fraction, + detail=( + f"Branch failing fraction: {branch_failing_fraction:.4f} " + f"(threshold: {hf.max_branch_failing_fraction})." + ), + ) + hf_max_p = HardFailResult( + check_name="extreme_branch_flow_deviation", + triggered=max_p_dev_pct > hf.p_max_deviation_pct, + value=max_p_dev_pct, + threshold=hf.p_max_deviation_pct, + detail=( + f"Max branch P deviation: {max_p_dev_pct:.2f}% at branch " + f"{max_p_dev_branch} (threshold: {hf.p_max_deviation_pct}%)." + ), + ) + hard_fail_checks = [hf_bus, hf_branch, hf_max_p] + any_hard_fail = any(c.triggered for c in hard_fail_checks) + + # Voltage-level breakdown + vl_breakdown = _compute_voltage_level_breakdown( + bus_deviations=bus_deviations, + tiers=spec.voltage_level_tiers, + bus_base_kv=bus_base_kv, + has_vm=False, + vm_tol=0.0, + va_tol=agg.va_tolerance_deg, + ) + + overall_pass = bus_metric_passed and branch_metric_passed and not any_hard_fail + + return VerificationVerdict( + analysis_type="dcpf", + overall_pass=overall_pass, + hard_fail=any_hard_fail, + aggregate_metrics=aggregate_metrics, + hard_fail_checks=hard_fail_checks, + outlier_summary=None, + voltage_level_breakdown=vl_breakdown, + total_non_excluded_buses=total_non_excluded, + total_in_service_branches=total_ref_branches, + ) + + +# --------------------------------------------------------------------------- +# Outlier classification helpers +# --------------------------------------------------------------------------- + + +def _has_switched_shunt( + bus_number: int, + intermediate_data: dict[str, list[dict]], +) -> bool: + """Check if bus has a switched shunt device.""" + shunts = intermediate_data.get("switched_shunt", []) + for s in shunts: + # Look for bus number in common field names + s_bus = s.get("I") or s.get("bus") or s.get("bus_number") + if s_bus is not None and int(s_bus) == bus_number: + return True + return False + + +def _generator_at_q_limit( + bus_number: int, + intermediate_data: dict[str, list[dict]], + ref_generators: list[dict] | None, + tolerance_mvar: float = 1.0, +) -> bool: + """Check if any generator at this bus is at a Q limit.""" + if ref_generators is None: + return False + + # Build set of generators at this bus from intermediate data + gen_rows = intermediate_data.get("generator", []) + bus_has_gen = False + for g in gen_rows: + g_bus = g.get("I") or g.get("bus") or g.get("bus_number") + if g_bus is not None and int(g_bus) == bus_number: + bus_has_gen = True + break + + if not bus_has_gen: + return False + + # Check reference generator Q vs limits + for rg in ref_generators: + rg_bus = rg.get("bus") or rg.get("I") + if rg_bus is not None and int(rg_bus) == bus_number: + qg = float(rg.get("QG", rg.get("Q", 0.0))) + qmax = float(rg.get("QMAX", rg.get("Qmax", float("inf")))) + qmin = float(rg.get("QMIN", rg.get("Qmin", float("-inf")))) + if abs(qg - qmax) < tolerance_mvar or abs(qg - qmin) < tolerance_mvar: + return True + + return False + + +def _is_slack_or_neighbor( + bus_number: int, + network_adjacency: dict[int, set[int]] | None, + slack_bus: int | None, + max_hops: int = 2, +) -> bool: + """Check if bus is the slack bus or within max_hops of it.""" + if slack_bus is None or network_adjacency is None: + return False + + if bus_number == slack_bus: + return True + + # BFS from slack up to max_hops + visited: set[int] = {slack_bus} + frontier: set[int] = {slack_bus} + for _ in range(max_hops): + next_frontier: set[int] = set() + for node in frontier: + for neighbor in network_adjacency.get(node, set()): + if neighbor not in visited: + visited.add(neighbor) + next_frontier.add(neighbor) + frontier = next_frontier + + return bus_number in visited + + +def _is_tap_regulated_bus( + bus_number: int, + intermediate_data: dict[str, list[dict]], +) -> bool: + """Check if bus is regulated by a tap-changing transformer (CONT field).""" + transformers = intermediate_data.get("transformer", []) + for t in transformers: + cont = t.get("CONT") or t.get("cont") + stat = t.get("STAT") or t.get("stat") or t.get("status") + if cont is not None and int(cont) == bus_number: + if stat is None or int(stat) == 1: + return True + return False + + +def _is_island_boundary( + bus_number: int, + network_adjacency: dict[int, set[int]] | None, + bus_base_kv: dict[int, float] | None, + max_degree: int = 2, + max_kv: float = 69.0, +) -> bool: + """Check if bus is at an island boundary (low degree + low voltage).""" + if network_adjacency is None or bus_base_kv is None: + return False + + degree = len(network_adjacency.get(bus_number, set())) + kv = bus_base_kv.get(bus_number, 0.0) + + return degree <= max_degree and kv < max_kv + + +def classify_outlier_bus( + bus_number: int, + rules: list[OutlierRule], + intermediate_data: dict[str, list[dict]], + ref_generators: list[dict] | None = None, + network_adjacency: dict[int, set[int]] | None = None, + slack_bus: int | None = None, + bus_base_kv: dict[int, float] | None = None, +) -> OutlierCause: + """Classify a single outlier bus by evaluating rules in priority order. + + Args: + bus_number: The bus to classify. + rules: Ordered list of classification rules. + intermediate_data: Dict mapping table names to row lists. + ref_generators: Reference ACPF generator results (for Q-limit check). + network_adjacency: Network adjacency list. + slack_bus: Slack bus number. + bus_base_kv: Bus number to base kV mapping. + + Returns: + The OutlierCause assigned to this bus. + """ + for rule in rules: + matched = False + + if rule.cause == OutlierCause.SWITCHED_SHUNT: + matched = _has_switched_shunt(bus_number, intermediate_data) + elif rule.cause == OutlierCause.Q_LIMIT: + matched = _generator_at_q_limit(bus_number, intermediate_data, ref_generators) + elif rule.cause == OutlierCause.SLACK_DISTRIBUTION: + matched = _is_slack_or_neighbor(bus_number, network_adjacency, slack_bus) + elif rule.cause == OutlierCause.TAP_POSITION: + matched = _is_tap_regulated_bus(bus_number, intermediate_data) + elif rule.cause == OutlierCause.ISLAND_BOUNDARY: + matched = _is_island_boundary(bus_number, network_adjacency, bus_base_kv) + + if matched: + return rule.cause + + return OutlierCause.UNCLASSIFIED + + +# --------------------------------------------------------------------------- +# Output orchestration +# --------------------------------------------------------------------------- + + +def generate_pass_conditions( + output_dir: Path | None = None, +) -> tuple[Path, Path]: + """Top-level function to generate both JSON and markdown pass condition files. + + Args: + output_dir: Directory for output files. Defaults to + ``data/fnm/reference/``. + + Returns: + Tuple of (json_path, markdown_path). + """ + if output_dir is None: + output_dir = Path("data/fnm/reference") + + output_dir.mkdir(parents=True, exist_ok=True) + spec = build_pass_condition_spec() + + json_path = output_dir / "pass_conditions.json" + md_path = output_dir / "pass_conditions.md" + + write_json(spec, json_path) + write_markdown(spec, md_path) + + return json_path, md_path + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> None: + """CLI entry point for pass condition generation. + + Args: + argv: Command-line arguments. If None, reads from sys.argv[1:]. + """ + parser = argparse.ArgumentParser( + description="Generate pass condition definitions for FNM verification." + ) + parser.add_argument( + "-o", + "--output-dir", + type=Path, + default=None, + help="Output directory (default: data/fnm/reference/).", + ) + args = parser.parse_args(argv if argv is not None else sys.argv[1:]) + + json_path, md_path = generate_pass_conditions(output_dir=args.output_dir) + + spec = build_pass_condition_spec() + acpf = spec.acpf_aggregate + dcpf = spec.dcpf_aggregate + acpf_hf = spec.acpf_hard_fail + dcpf_hf = spec.dcpf_hard_fail + + print( + f"ACPF: min_passing={acpf.min_passing_fraction * 100:.0f}%, " + f"VM<{acpf.vm_tolerance_pu} p.u., VA<{acpf.va_tolerance_deg} deg" + ) + print( + f"DCPF: min_bus_passing={dcpf.min_bus_passing_fraction * 100:.0f}% " + f"VA<{dcpf.va_tolerance_deg} deg, " + f"min_branch_passing={dcpf.min_branch_passing_fraction * 100:.0f}% " + f"P<{dcpf.p_tolerance_pct}%" + ) + print( + f"Hard-fail thresholds: ACPF VM>{acpf_hf.vm_max_deviation_pu}/" + f"VA>{acpf_hf.va_max_deviation_deg}, " + f"DCPF P>{dcpf_hf.p_max_deviation_pct}%" + ) + print(f"Outlier rules: {len(spec.outlier_classification.rules)} classification rules") + print(f"Voltage tiers: {len(spec.voltage_level_tiers)} informational tiers") + print(f"JSON: {json_path}") + print(f"Markdown: {md_path}") + + +if __name__ == "__main__": + main() diff --git a/data/fnm/scripts/raw_record_counter.py b/data/fnm/scripts/raw_record_counter.py new file mode 100644 index 00000000..8d1e1afc --- /dev/null +++ b/data/fnm/scripts/raw_record_counter.py @@ -0,0 +1,290 @@ +"""Parser-independent PSS/E v31 RAW file record counter. + +Reads a PSS/E v31 RAW file and counts data records per section by pure text/line +parsing. Understands the v31 file structure: 3-line header followed by 17 record +sections each terminated by a ``0`` sentinel line. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path + +PSSE_V31_SECTION_NAMES: tuple[str, ...] = ( + "Bus", + "Load", + "Fixed Shunt", + "Generator", + "Branch", + "Transformer", + "Area", + "Two-Terminal DC", + "VSC DC", + "Impedance Correction", + "Multi-Terminal DC", + "Multi-Section Line", + "Zone", + "Interarea Transfer", + "Owner", + "FACTS", + "Switched Shunt", +) + +_HVDC_FACTS_KEYS = ("Two-Terminal DC", "VSC DC", "Multi-Terminal DC", "FACTS") + + +@dataclass(frozen=True) +class HeaderInfo: + """Parsed PSS/E v31 3-line header.""" + + ic: int + sbase: float + rev: float + xfrrat: float + nxfrat: float + basfrq: float + case_id: str + case_id2: str + + +@dataclass(frozen=True) +class RecordCountSummary: + """Summary of record counts across all 17 PSS/E v31 sections.""" + + header: HeaderInfo + section_counts: dict[str, int] + total_data_lines: int + non_empty_sections: list[str] + total_sections: int + hvdc_facts_present: dict[str, bool] + + +def _is_sentinel(line: str) -> bool: + """Check if a line is a section-terminating sentinel (first token is '0').""" + stripped = line.strip() + if not stripped: + return False + first_token = stripped.split()[0] + # Also handle comma-separated: "0," or "0 ," + return first_token.rstrip(",") == "0" + + +def parse_header(lines: list[str]) -> HeaderInfo: + """Parse 3-line PSS/E v31 header. + + Args: + lines: The first 3 lines of the RAW file. + + Returns: + Parsed HeaderInfo. + + Raises: + ValueError: If the header is malformed or not v31. + """ + if len(lines) < 3: + raise ValueError(f"Expected at least 3 header lines, got {len(lines)}") + + # Line 1: IC, SBASE, REV, XFRRAT, NXFRAT, BASFRQ / comment + line1 = lines[0].split("/")[0].strip() + # Try comma-separated first, fall back to space-separated + parts = [p.strip() for p in line1.split(",") if p.strip()] + if len(parts) == 1: + # Space-separated format (e.g., production RAW files) + parts = line1.split() + + try: + ic = int(float(parts[0])) + sbase = float(parts[1]) + rev = float(parts[2]) if len(parts) > 2 else 0.0 + xfrrat = float(parts[3]) if len(parts) > 3 else 0.0 + nxfrat = float(parts[4]) if len(parts) > 4 else 0.0 + basfrq = float(parts[5]) if len(parts) > 5 else 0.0 + except (ValueError, IndexError) as exc: + raise ValueError(f"Malformed header line 1: {lines[0]!r}") from exc + + if rev < 31.0 or rev >= 32.0: + raise ValueError(f"Expected PSS/E v31 (REV=31.x), got REV={rev}") + + case_id = lines[1].strip() + case_id2 = lines[2].strip() + + return HeaderInfo( + ic=ic, + sbase=sbase, + rev=rev, + xfrrat=xfrrat, + nxfrat=nxfrat, + basfrq=basfrq, + case_id=case_id, + case_id2=case_id2, + ) + + +def count_section_records(line_iter: Iterator[str], section_index: int) -> int: + """Count records in one PSS/E section. + + Handles special cases: + - Transformer (section_index=5): 4 lines per 2-winding, 5 lines per 3-winding. + - Multi-Terminal DC (section_index=10): Variable-length records. + + Args: + line_iter: Iterator over remaining lines in the file. + section_index: 0-based index of the current section. + + Returns: + Number of records in this section. + """ + count = 0 + + if section_index == 5: + # Transformer section: multi-line records + for line in line_iter: + if _is_sentinel(line): + break + # This is line 1 of a transformer record + # Determine 2W vs 3W by checking K (3rd bus number, field index 2) + parts = line.split(",") if "," in line else line.split() + try: + k = int(parts[2].strip()) + except (ValueError, IndexError): + k = 0 + # Line 1 already consumed. Read lines 2, 3, 4. + next(line_iter) # line 2 + next(line_iter) # line 3 + next(line_iter) # line 4 + if k != 0: + next(line_iter) # line 5 for 3-winding + count += 1 + + elif section_index == 10: + # Multi-Terminal DC section: variable-length records + for line in line_iter: + if _is_sentinel(line): + break + # First line has NCONV, NDCBS, NDCLN, ... + parts = line.split(",") if "," in line else line.split() + try: + nconv = int(parts[0].strip()) + ndcbs = int(parts[1].strip()) + ndcln = int(parts[2].strip()) + except (ValueError, IndexError): + nconv = ndcbs = ndcln = 0 + # Read nconv converter lines, ndcbs DC bus lines, ndcln DC link lines + for _ in range(nconv): + next(line_iter) + for _ in range(ndcbs): + next(line_iter) + for _ in range(ndcln): + next(line_iter) + count += 1 + + else: + # Standard single-line-per-record section + for line in line_iter: + if _is_sentinel(line): + break + count += 1 + + return count + + +def count_raw_records(raw_path: str | Path) -> RecordCountSummary: + """Read a PSS/E v31 RAW file and count all records. Streaming single-pass. + + Args: + raw_path: Path to the RAW file. + + Returns: + RecordCountSummary with counts for all 17 sections. + + Raises: + ValueError: If the header is malformed or not v31. + FileNotFoundError: If the file does not exist. + """ + raw_path = Path(raw_path) + if not raw_path.exists(): + raise FileNotFoundError(f"RAW file not found: {raw_path}") + + with open(raw_path, encoding="utf-8", errors="replace") as f: + all_lines = f.readlines() + + # Parse 3-line header + header = parse_header(all_lines[:3]) + + # Count records in each of 17 sections + line_iter = iter(all_lines[3:]) + section_counts: dict[str, int] = {} + + for idx, section_name in enumerate(PSSE_V31_SECTION_NAMES): + section_counts[section_name] = count_section_records(line_iter, idx) + + total_data_lines = sum(section_counts.values()) + non_empty_sections = [name for name, cnt in section_counts.items() if cnt > 0] + hvdc_facts_present = {key: section_counts.get(key, 0) > 0 for key in _HVDC_FACTS_KEYS} + + return RecordCountSummary( + header=header, + section_counts=section_counts, + total_data_lines=total_data_lines, + non_empty_sections=non_empty_sections, + total_sections=len(PSSE_V31_SECTION_NAMES), + hvdc_facts_present=hvdc_facts_present, + ) + + +def summary_to_dict(summary: RecordCountSummary) -> dict: + """Convert a RecordCountSummary to a JSON-serializable dict. + + Args: + summary: The summary to convert. + + Returns: + A dict suitable for ``json.dumps()``. + """ + return { + "header": { + "ic": summary.header.ic, + "sbase": summary.header.sbase, + "rev": summary.header.rev, + "xfrrat": summary.header.xfrrat, + "nxfrat": summary.header.nxfrat, + "basfrq": summary.header.basfrq, + "case_id": summary.header.case_id, + "case_id2": summary.header.case_id2, + }, + "section_counts": summary.section_counts, + "total_data_lines": summary.total_data_lines, + "non_empty_sections": summary.non_empty_sections, + "total_sections": summary.total_sections, + "hvdc_facts_present": summary.hvdc_facts_present, + } + + +def main(argv: list[str] | None = None) -> None: + """CLI entry point: python -m raw_record_counter /path/to/file.raw [-o output.json]""" + parser = argparse.ArgumentParser( + description="Count records per section in a PSS/E v31 RAW file." + ) + parser.add_argument("raw_file", type=str, help="Path to the PSS/E v31 RAW file") + parser.add_argument( + "-o", + "--output", + type=str, + default=None, + help="Output JSON file path (default: print to stdout)", + ) + args = parser.parse_args(argv) + + summary = count_raw_records(args.raw_file) + result = summary_to_dict(summary) + output_text = json.dumps(result, indent=2) + "\n" + + if args.output: + Path(args.output).write_text(output_text, encoding="utf-8") + print(f"Results written to {args.output}", file=sys.stderr) + else: + print(output_text) diff --git a/data/fnm/scripts/run_psse2mpc.m b/data/fnm/scripts/run_psse2mpc.m new file mode 100644 index 00000000..ac7cd421 --- /dev/null +++ b/data/fnm/scripts/run_psse2mpc.m @@ -0,0 +1,111 @@ +% run_psse2mpc.m -- Convert a PSS/E RAW file to MATPOWER case struct and export CSVs. +% +% Usage: +% octave --no-gui --no-init-file run_psse2mpc.m [] +% +% Arguments: +% raw_path -- Path to the PSS/E RAW file. +% output_dir -- Directory to write CSV exports and .mat file. +% matpower_path -- (optional) Path to MATPOWER installation directory. +% Default: evaluations/matpower/matpower8.1 relative to repo root. +% +% Outputs: +% Structured stdout lines: +% MPC_BASEMVA: +% MPC_VERSION: +% MPC_FIELD_COUNT:: +% CSV files: mpc_bus.csv, mpc_gen.csv, mpc_branch.csv, etc. +% MAT file: mpc_case.mat + +args = argv(); +if length(args) < 2 + fprintf(2, 'Usage: run_psse2mpc.m []\n'); + exit(1); +end + +raw_path = args{1}; +output_dir = args{2}; + +% Determine MATPOWER path +if length(args) >= 3 + matpower_path = args{3}; +else + % Default: relative to this script's location + % Script is at /data/fnm/scripts/run_psse2mpc.m + % MATPOWER is at /evaluations/matpower/matpower8.1 + script_dir = fileparts(mfilename('fullpath')); + repo_root = fullfile(script_dir, '..', '..', '..'); + matpower_path = fullfile(repo_root, 'evaluations', 'matpower', 'matpower8.1'); +end + +% Add MATPOWER to path +if ~exist(matpower_path, 'dir') + fprintf(2, 'ERROR: MATPOWER path not found: %s\n', matpower_path); + exit(1); +end +addpath(genpath(matpower_path)); + +% Ensure output directory exists +if ~exist(output_dir, 'dir') + mkdir(output_dir); +end + +% Verify input file exists +if ~exist(raw_path, 'file') + fprintf(2, 'ERROR: RAW file not found: %s\n', raw_path); + exit(1); +end + +% Run psse2mpc conversion +try + [mpc, warnings] = psse2mpc(raw_path, '', 0); +catch err + fprintf(2, 'ERROR: psse2mpc failed: %s\n', err.message); + exit(1); +end + +% Print structured output +if isfield(mpc, 'baseMVA') + fprintf('MPC_BASEMVA:%g\n', mpc.baseMVA); +end + +if isfield(mpc, 'version') + fprintf('MPC_VERSION:%s\n', mpc.version); +end + +% Export numeric fields as CSV +numeric_fields = {'bus', 'gen', 'branch', 'gencost', 'areas', 'dcline'}; + +for i = 1:length(numeric_fields) + fname = numeric_fields{i}; + if isfield(mpc, fname) && ~isempty(mpc.(fname)) + csv_path = fullfile(output_dir, ['mpc_' fname '.csv']); + csvwrite(csv_path, mpc.(fname)); + row_count = size(mpc.(fname), 1); + fprintf('MPC_FIELD_COUNT:%s:%d\n', fname, row_count); + end +end + +% Handle bus_name (cell array of strings) separately +if isfield(mpc, 'bus_name') && ~isempty(mpc.bus_name) + csv_path = fullfile(output_dir, 'mpc_bus_name.csv'); + fid = fopen(csv_path, 'w'); + for i = 1:length(mpc.bus_name) + fprintf(fid, '%s\n', strtrim(mpc.bus_name{i})); + end + fclose(fid); + fprintf('MPC_FIELD_COUNT:bus_name:%d\n', length(mpc.bus_name)); +end + +% Save the full mpc struct as .mat +mat_path = fullfile(output_dir, 'mpc_case.mat'); +save('-v7', mat_path, 'mpc'); + +% Print any warnings from psse2mpc +if ~isempty(warnings) + for i = 1:length(warnings) + fprintf(2, 'PSSE2MPC_WARNING: %s\n', warnings{i}); + end +end + +fprintf('CONVERSION_COMPLETE\n'); diff --git a/data/fnm/scripts/solve_main_island.m b/data/fnm/scripts/solve_main_island.m new file mode 100644 index 00000000..ad1ff0fe --- /dev/null +++ b/data/fnm/scripts/solve_main_island.m @@ -0,0 +1,280 @@ +% solve_main_island.m -- Extract the main island and solve DCPF + ACPF. +% +% The FNM has 4 islands: main (27862 buses) + 3 tiny (63, 9, 3 buses). +% The tiny islands cause multi-slack issues that confuse solvers. +% Extract the main island, set a single slack, and solve. + +script_dir = fileparts(mfilename('fullpath')); +fnm_dir = fullfile(script_dir, '..'); +repo_root = fullfile(fnm_dir, '..', '..'); +matpower_path = fullfile(repo_root, 'evaluations', 'matpower', 'matpower8.1'); +addpath(genpath(matpower_path)); + +mat_path = fullfile(fnm_dir, 'reference', 'matpower_parse', 'mpc_case.mat'); +load(mat_path, 'mpc'); +fprintf('Loaded: %d buses, %d branches, %d gens\n', ... + size(mpc.bus, 1), size(mpc.branch, 1), size(mpc.gen, 1)); + +% ---- Data fixes ---- +zero_x = (mpc.branch(:, 4) == 0); +if any(zero_x) + mpc.branch(zero_x, 4) = 0.0001; + fprintf('Fixed %d zero-X branches\n', sum(zero_x)); +end +if size(mpc.branch, 2) >= 6 + zero_rate = (mpc.branch(:, 6) == 0); + if any(zero_rate) + mpc.branch(zero_rate, 6) = 9999; + fprintf('Fixed %d zero-RATE_A branches\n', sum(zero_rate)); + end +end + +% ---- Extract main island using MATPOWER's extract_islands ---- +fprintf('\nExtracting islands...\n'); +mpci = extract_islands(mpc); + +% mpci is a cell array of mpc structs, one per island +fprintf('Found %d islands:\n', length(mpci)); +for i = 1:length(mpci) + fprintf(' Island %d: %d buses, %d branches, %d gens\n', ... + i, size(mpci{i}.bus, 1), size(mpci{i}.branch, 1), size(mpci{i}.gen, 1)); +end + +% Find the largest island +island_sizes = cellfun(@(x) size(x.bus, 1), mpci); +[~, main_idx] = max(island_sizes); +mpc_main = mpci{main_idx}; +fprintf('\nUsing island %d (%d buses) as main island\n', ... + main_idx, size(mpc_main.bus, 1)); + +% Ensure exactly one slack bus +n_slack = sum(mpc_main.bus(:, 2) == 3); +fprintf('Slack buses in main island: %d\n', n_slack); +if n_slack == 0 + % Find the largest generator and make its bus the slack + [~, max_gen_idx] = max(mpc_main.gen(:, 9)); % col 9 = Pmax + slack_bus_num = mpc_main.gen(max_gen_idx, 1); + bus_idx = find(mpc_main.bus(:, 1) == slack_bus_num); + mpc_main.bus(bus_idx, 2) = 3; + fprintf('Assigned slack to bus %d (largest Pmax = %.1f MW)\n', ... + slack_bus_num, mpc_main.gen(max_gen_idx, 9)); +elseif n_slack > 1 + slack_idx = find(mpc_main.bus(:, 2) == 3); + for i = 2:length(slack_idx) + mpc_main.bus(slack_idx(i), 2) = 2; + end + fprintf('Reduced to 1 slack (bus %d)\n', mpc_main.bus(slack_idx(1), 1)); +end + +% ---- DCPF on main island ---- +fprintf('\n=== DCPF on Main Island ===\n'); +mpopt_dc = mpoption('verbose', 1, 'out.all', 0); +tic; +results_dc = rundcpf(mpc_main, mpopt_dc); +dcpf_time = toc; + +if results_dc.success + fprintf('DCPF CONVERGED in %.2f seconds\n', dcpf_time); +else + fprintf('DCPF FAILED\n'); +end + +% ---- ACPF on main island ---- +fprintf('\n=== ACPF on Main Island ===\n'); + +% Warm start: use DCPF angles +mpc_ac = mpc_main; +if results_dc.success + mpc_ac.bus(:, 9) = results_dc.bus(:, 9); + fprintf('Using DCPF angles as warm start\n'); +end + +% Try each solver variant +algorithms = {'NR-IC', 'NR-SP', 'NR-SH', 'NR-IH', 'NR', 'FDXB', 'FDBX'}; +results_ac = struct('success', 0); +acpf_time = 0; +winning_alg = ''; + +for a = 1:length(algorithms) + alg = algorithms{a}; + fprintf('\n--- Trying %s ---\n', alg); + + mpopt = mpoption('verbose', 1, 'out.all', 0); + mpopt = mpoption(mpopt, 'pf.alg', alg); + mpopt = mpoption(mpopt, 'pf.tol', 1e-8); + if strcmp(alg, 'FDXB') || strcmp(alg, 'FDBX') + mpopt = mpoption(mpopt, 'pf.fd.max_it', 1000); + else + mpopt = mpoption(mpopt, 'pf.nr.max_it', 200); + end + mpopt = mpoption(mpopt, 'pf.enforce_q_lims', 0); + + tic; + results_ac = runpf(mpc_ac, mpopt); + acpf_time = toc; + + if results_ac.success + fprintf('%s CONVERGED in %.2f sec, %d iterations\n', ... + alg, acpf_time, results_ac.iterations); + winning_alg = alg; + break + else + fprintf('%s failed\n', alg); + end +end + +% ---- Write reference outputs ---- +if results_dc.success + dcpf_dir = fullfile(fnm_dir, 'reference', 'dcpf'); + if ~exist(dcpf_dir, 'dir') + mkdir(dcpf_dir); + end + + bus_dc = results_dc.bus; + br_dc = results_dc.branch; + + fid = fopen(fullfile(dcpf_dir, 'buses_dcpf.csv'), 'w'); + fprintf(fid, 'bus_number,va_deg,pd_mw,base_kv,bus_type\n'); + for i = 1:size(bus_dc, 1) + if bus_dc(i, 2) == 4 + continue + end + fprintf(fid, '%d,%.8f,%.4f,%.2f,%d\n', ... + bus_dc(i, 1), bus_dc(i, 9), bus_dc(i, 3), bus_dc(i, 10), bus_dc(i, 2)); + end + fclose(fid); + + fid = fopen(fullfile(dcpf_dir, 'branches_dcpf.csv'), 'w'); + fprintf(fid, 'from_bus,to_bus,pf_mw,status\n'); + for i = 1:size(br_dc, 1) + if br_dc(i, 11) == 0 + continue + end + fprintf(fid, '%d,%d,%.8f,%d\n', ... + br_dc(i, 1), br_dc(i, 2), br_dc(i, 14), br_dc(i, 11)); + end + fclose(fid); + + slack_idx = find(bus_dc(:, 2) == 3); + active_buses = bus_dc(bus_dc(:, 2) ~= 4, :); + active_gens = results_dc.gen(results_dc.gen(:, 8) > 0, :); + + fid = fopen(fullfile(dcpf_dir, 'summary_dcpf.json'), 'w'); + fprintf(fid, '{\n'); + fprintf(fid, ' "success": %d,\n', results_dc.success); + fprintf(fid, ' "wall_clock_seconds": %.4f,\n', dcpf_time); + fprintf(fid, ' "total_gen_mw": %.4f,\n', sum(active_gens(:, 2))); + fprintf(fid, ' "total_load_mw": %.4f,\n', sum(active_buses(:, 3))); + fprintf(fid, ' "slack_bus": %d,\n', bus_dc(slack_idx(1), 1)); + fprintf(fid, ' "slack_angle": %.8f,\n', bus_dc(slack_idx(1), 9)); + fprintf(fid, ' "n_buses": %d,\n', size(active_buses, 1)); + fprintf(fid, ' "n_branches": %d,\n', sum(br_dc(:, 11) ~= 0)); + fprintf(fid, ' "n_gens": %d,\n', size(active_gens, 1)); + fprintf(fid, ' "main_island_only": true\n'); + fprintf(fid, '}\n'); + fclose(fid); + fprintf('DCPF reference written\n'); +end + +if results_ac.success + acpf_dir = fullfile(fnm_dir, 'reference', 'acpf'); + if ~exist(acpf_dir, 'dir') + mkdir(acpf_dir); + end + + bus_ac = results_ac.bus; + br_ac = results_ac.branch; + gen_ac = results_ac.gen; + active_gens = gen_ac(gen_ac(:, 8) > 0, :); + active_buses = bus_ac(bus_ac(:, 2) ~= 4, :); + n_isolated = sum(bus_ac(:, 2) == 4); + + fid = fopen(fullfile(acpf_dir, 'buses_acpf.csv'), 'w'); + fprintf(fid, 'bus_number,vm_pu,va_deg,pd_mw,qd_mvar,bus_type\n'); + for i = 1:size(bus_ac, 1) + if bus_ac(i, 2) == 4 + continue + end + fprintf(fid, '%d,%.8f,%.8f,%.4f,%.4f,%d\n', ... + bus_ac(i, 1), bus_ac(i, 8), bus_ac(i, 9), ... + bus_ac(i, 3), bus_ac(i, 4), bus_ac(i, 2)); + end + fclose(fid); + + fid = fopen(fullfile(acpf_dir, 'branches_acpf.csv'), 'w'); + fprintf(fid, 'from_bus,to_bus,pf_mw,qf_mvar,pt_mw,qt_mvar,status\n'); + for i = 1:size(br_ac, 1) + if br_ac(i, 11) == 0 + continue + end + fprintf(fid, '%d,%d,%.8f,%.8f,%.8f,%.8f,%d\n', ... + br_ac(i, 1), br_ac(i, 2), br_ac(i, 14), br_ac(i, 15), ... + br_ac(i, 16), br_ac(i, 17), br_ac(i, 11)); + end + fclose(fid); + + fid = fopen(fullfile(acpf_dir, 'generators_acpf.csv'), 'w'); + fprintf(fid, 'bus_number,pg_mw,qg_mvar,status,vm_setpoint\n'); + for i = 1:size(gen_ac, 1) + if gen_ac(i, 8) <= 0 + continue + end + fprintf(fid, '%d,%.8f,%.8f,%d,%.8f\n', ... + gen_ac(i, 1), gen_ac(i, 2), gen_ac(i, 3), ... + gen_ac(i, 8), gen_ac(i, 6)); + end + fclose(fid); + + fid = fopen(fullfile(acpf_dir, 'summary_acpf.json'), 'w'); + fprintf(fid, '{\n'); + fprintf(fid, ' "success": %d,\n', results_ac.success); + fprintf(fid, ' "wall_clock_seconds": %.4f,\n', acpf_time); + fprintf(fid, ' "iterations": %d,\n', results_ac.iterations); + fprintf(fid, ' "algorithm": "%s",\n', winning_alg); + fprintf(fid, ' "total_gen_mw": %.4f,\n', sum(active_gens(:, 2))); + fprintf(fid, ' "total_gen_mvar": %.4f,\n', sum(active_gens(:, 3))); + fprintf(fid, ' "total_load_mw": %.4f,\n', sum(active_buses(:, 3))); + fprintf(fid, ' "total_load_mvar": %.4f,\n', sum(active_buses(:, 4))); + fprintf(fid, ' "losses_mw": %.4f,\n', sum(active_gens(:, 2)) - sum(active_buses(:, 3))); + fprintf(fid, ' "n_buses": %d,\n', size(active_buses, 1)); + fprintf(fid, ' "n_branches": %d,\n', sum(br_ac(:, 11) ~= 0)); + fprintf(fid, ' "n_gens": %d,\n', size(active_gens, 1)); + fprintf(fid, ' "solver": "MATPOWER 8.1",\n'); + fprintf(fid, ' "tolerance": 1e-8,\n'); + fprintf(fid, ' "q_limits_enforced": false,\n'); + fprintf(fid, ' "isolated_buses_removed": %d,\n', n_isolated); + fprintf(fid, ' "main_island_only": true,\n'); + fprintf(fid, ' "initial_conditions": "dcpf_warm_start_flat_vm"\n'); + fprintf(fid, '}\n'); + fclose(fid); + fprintf('ACPF reference written\n'); +else + fprintf('\n=== ALL ACPF ATTEMPTS FAILED ===\n'); + acpf_dir = fullfile(fnm_dir, 'reference', 'acpf'); + if ~exist(acpf_dir, 'dir') + mkdir(acpf_dir); + end + fid = fopen(fullfile(acpf_dir, 'summary_acpf.json'), 'w'); + fprintf(fid, '{\n'); + fprintf(fid, ' "success": 0,\n'); + n_main = size(mpc_main.bus, 1); + fprintf(fid, ... + ' "failure_reason": "All MATPOWER variants diverged (%d buses)",\n', ... + n_main); + fprintf(fid, ' "attempts": ['); + for a = 1:length(algorithms) + fprintf(fid, '"%s"', algorithms{a}); + if a < length(algorithms) + fprintf(fid, ', '); + end + end + fprintf(fid, '],\n'); + fprintf(fid, ' "n_buses": %d,\n', sum(mpc_main.bus(:, 2) ~= 4)); + fprintf(fid, ' "all_vg_flat": true,\n'); + fprintf(fid, ' "all_vm_flat": true,\n'); + fprintf(fid, ' "main_island_only": true\n'); + fprintf(fid, '}\n'); + fclose(fid); +end + +fprintf('\n=== DONE ===\n'); diff --git a/data/fnm/scripts/solve_references.m b/data/fnm/scripts/solve_references.m new file mode 100644 index 00000000..3847fbda --- /dev/null +++ b/data/fnm/scripts/solve_references.m @@ -0,0 +1,554 @@ +% solve_references.m -- Clean FNM data, export cleaned case, compute references. +% +% Usage: +% octave --no-gui --no-window-system solve_references.m +% +% Inputs: +% - MATPOWER-parsed case: data/fnm/reference/matpower_parse/mpc_case.mat +% +% Outputs: +% - data/fnm/reference/cleaned/fnm_main_island.m (MATPOWER case file) +% - data/fnm/reference/cleaned/fnm_main_island.mat (binary .mat for scipy) +% - data/fnm/reference/cleaned/summary_cleaning.json (committed manifest) +% - data/fnm/reference/dcpf/ (buses_dcpf.csv, branches_dcpf.csv, summary_dcpf.json) +% - data/fnm/reference/acpf/ (buses, branches, generators, summary) + +% Determine paths +script_dir = fileparts(mfilename('fullpath')); +fnm_dir = fullfile(script_dir, '..'); +repo_root = fullfile(fnm_dir, '..', '..'); +matpower_path = fullfile(repo_root, 'evaluations', 'matpower', 'matpower8.1'); + +% Add MATPOWER +addpath(genpath(matpower_path)); + +% Load the pre-parsed case +mat_path = fullfile(fnm_dir, 'reference', 'matpower_parse', 'mpc_case.mat'); +if ~exist(mat_path, 'file') + error('mpc_case.mat not found at %s', mat_path); +end +load(mat_path, 'mpc'); +fprintf('Loaded mpc: %d buses, %d branches, %d generators\n', ... + size(mpc.bus, 1), size(mpc.branch, 1), size(mpc.gen, 1)); + +% ========================================================================= +% Stage 1: Data Cleaning (per protocol v6) +% ========================================================================= +fprintf('\n=== Stage 1: Data Cleaning ===\n'); + +cleaning_log = struct(); + +% --- Fix 1: Negative reactance -> absolute value (series capacitors) --- +neg_x = (mpc.branch(:, 4) < 0); +n_neg_x = sum(neg_x); +if n_neg_x > 0 + mpc.branch(neg_x, 4) = abs(mpc.branch(neg_x, 4)); + fprintf('Fix 1: Coerced %d negative-X branches to |X| (series capacitors)\n', n_neg_x); +end +cleaning_log.negative_x_coerced = n_neg_x; + +% --- Fix 2: Zero reactance -> small value --- +zero_x = (mpc.branch(:, 4) == 0); +n_zero_x = sum(zero_x); +if n_zero_x > 0 + mpc.branch(zero_x, 4) = 0.0001; + fprintf('Fix 2: Set %d zero-X branches to X=0.0001 pu\n', n_zero_x); +end +cleaning_log.zero_x_fixed = n_zero_x; + +% --- Fix 3: Zero resistance -> small value --- +zero_r = (mpc.branch(:, 3) == 0); +n_zero_r = sum(zero_r); +if n_zero_r > 0 + mpc.branch(zero_r, 3) = 0.0001; + fprintf('Fix 3: Set %d zero-R branches to R=0.0001 pu\n', n_zero_r); +end +cleaning_log.zero_r_fixed = n_zero_r; + +% --- Fix 4: Zero thermal rating -> unlimited --- +if size(mpc.branch, 2) >= 6 + zero_rate = (mpc.branch(:, 6) == 0); + n_zero_rate = sum(zero_rate); + if n_zero_rate > 0 + mpc.branch(zero_rate, 6) = 9999; + fprintf('Fix 4: Set %d zero-RATE_A branches to 9999 MVA\n', n_zero_rate); + end +else + n_zero_rate = 0; +end +cleaning_log.zero_rate_a_fixed = n_zero_rate; + +% ========================================================================= +% Stage 2: Island Extraction +% ========================================================================= +fprintf('\n=== Stage 2: Island Extraction ===\n'); + +mpci = extract_islands(mpc); +n_islands = length(mpci); +fprintf('Found %d islands:\n', n_islands); +island_info = {}; +for i = 1:n_islands + nb = size(mpci{i}.bus, 1); + nbr = size(mpci{i}.branch, 1); + ng = size(mpci{i}.gen, 1); + fprintf(' Island %d: %d buses, %d branches, %d gens\n', i, nb, nbr, ng); + island_info{i} = struct('buses', nb, 'branches', nbr, 'gens', ng); +end + +% Select the largest island +island_sizes = cellfun(@(x) size(x.bus, 1), mpci); +[~, main_idx] = max(island_sizes); +mpc_main = mpci{main_idx}; +fprintf('Selected island %d (%d buses) as main island\n', ... + main_idx, size(mpc_main.bus, 1)); + +cleaning_log.islands_total = n_islands; +cleaning_log.main_island_buses = size(mpc_main.bus, 1); +cleaning_log.main_island_branches = size(mpc_main.branch, 1); +cleaning_log.main_island_gens = size(mpc_main.gen, 1); +cleaning_log.excluded_buses = size(mpc.bus, 1) - size(mpc_main.bus, 1); + +% ========================================================================= +% Stage 3: Multi-Slack Reduction +% ========================================================================= +fprintf('\n=== Stage 3: Slack Bus Reduction ===\n'); + +slack_idx = find(mpc_main.bus(:, 2) == 3); +n_slack = length(slack_idx); +fprintf('Slack buses in main island: %d\n', n_slack); + +if n_slack == 0 + % Promote the bus with the largest generator to slack + [~, max_gen_idx] = max(mpc_main.gen(:, 9)); % col 9 = Pmax + slack_bus_num = mpc_main.gen(max_gen_idx, 1); + bus_idx = find(mpc_main.bus(:, 1) == slack_bus_num); + mpc_main.bus(bus_idx, 2) = 3; + fprintf('Assigned slack to bus %d (largest Pmax = %.1f MW)\n', ... + slack_bus_num, mpc_main.gen(max_gen_idx, 9)); + cleaning_log.slack_action = 'promoted_largest_gen'; + cleaning_log.slack_bus = slack_bus_num; +elseif n_slack > 1 + kept_slack = mpc_main.bus(slack_idx(1), 1); + demoted = []; + for i = 2:length(slack_idx) + demoted(end + 1) = mpc_main.bus(slack_idx(i), 1); + mpc_main.bus(slack_idx(i), 2) = 2; % demote to PV + end + fprintf('Kept bus %d as slack, demoted %d others to PV: [%s]\n', ... + kept_slack, length(demoted), num2str(demoted)); + cleaning_log.slack_action = 'demoted_extras'; + cleaning_log.slack_bus = kept_slack; + cleaning_log.slack_demoted = demoted; +else + cleaning_log.slack_action = 'none_needed'; + cleaning_log.slack_bus = mpc_main.bus(slack_idx(1), 1); +end + +% ========================================================================= +% Stage 4: Export Cleaned Case +% ========================================================================= +fprintf('\n=== Stage 4: Export Cleaned Case ===\n'); + +cleaned_dir = fullfile(fnm_dir, 'reference', 'cleaned'); +if ~exist(cleaned_dir, 'dir') + mkdir(cleaned_dir); +end + +% Ensure the case name is set +mpc_main.casename = 'fnm_main_island'; + +% Save as MATPOWER .m case file (readable by MATPOWER, PowerModels.jl, etc.) +case_m_path = fullfile(cleaned_dir, 'fnm_main_island.m'); +savecase(case_m_path, mpc_main); +fprintf('Saved cleaned case: %s\n', case_m_path); + +% Save as .mat binary (readable by scipy.io.loadmat) +case_mat_path = fullfile(cleaned_dir, 'fnm_main_island.mat'); +mpc = mpc_main; % savecase uses 'mpc' variable name +save(case_mat_path, 'mpc'); +fprintf('Saved cleaned case: %s\n', case_mat_path); + +% Count in-service elements in cleaned case +n_buses_clean = size(mpc_main.bus, 1); +n_branches_clean = size(mpc_main.branch, 1); +n_branches_active = sum(mpc_main.branch(:, 11) ~= 0); +n_gens_clean = size(mpc_main.gen, 1); +n_gens_active = sum(mpc_main.gen(:, 8) > 0); +n_loads = sum(mpc_main.bus(:, 3) ~= 0 | mpc_main.bus(:, 4) ~= 0); + +fprintf('Cleaned case: %d buses, %d branches (%d active), %d gens (%d active), %d loads\n', ... + n_buses_clean, n_branches_clean, n_branches_active, n_gens_clean, n_gens_active, n_loads); + +% Write cleaning manifest (JSON - this file gets committed) +manifest_path = fullfile(cleaned_dir, 'summary_cleaning.json'); +fid = fopen(manifest_path, 'w'); +fprintf(fid, '{\n'); +fprintf(fid, ' "source": "data/fnm/reference/matpower_parse/mpc_case.mat",\n'); +src_buses = size(mpci{1}.bus, 1) + cleaning_log.excluded_buses; +fprintf(fid, ' "source_buses": %d,\n', src_buses); +fprintf(fid, ' "cleaning_steps": [\n'); +fprintf(fid, ' {\n'); +fprintf(fid, ' "step": 1,\n'); +fprintf(fid, ' "name": "negative_x_to_abs",\n'); +desc = 'Coerce negative reactance to |X| (series capacitors)'; +fprintf(fid, ' "description": "%s",\n', desc); +fprintf(fid, ' "affected_branches": %d\n', cleaning_log.negative_x_coerced); +fprintf(fid, ' },\n'); +fprintf(fid, ' {\n'); +fprintf(fid, ' "step": 2,\n'); +fprintf(fid, ' "name": "zero_x_to_small",\n'); +desc = 'Set zero reactance to 0.0001 pu (singular admittance)'; +fprintf(fid, ' "description": "%s",\n', desc); +fprintf(fid, ' "affected_branches": %d\n', cleaning_log.zero_x_fixed); +fprintf(fid, ' },\n'); +fprintf(fid, ' {\n'); +fprintf(fid, ' "step": 3,\n'); +fprintf(fid, ' "name": "zero_r_to_small",\n'); +desc = 'Set zero resistance to 0.0001 pu (NR Jacobian)'; +fprintf(fid, ' "description": "%s",\n', desc); +fprintf(fid, ' "affected_branches": %d\n', cleaning_log.zero_r_fixed); +fprintf(fid, ' },\n'); +fprintf(fid, ' {\n'); +fprintf(fid, ' "step": 4,\n'); +fprintf(fid, ' "name": "zero_rate_a_to_unlimited",\n'); +desc = 'Set zero thermal rating to 9999 MVA (unlimited)'; +fprintf(fid, ' "description": "%s",\n', desc); +fprintf(fid, ' "affected_branches": %d\n', cleaning_log.zero_rate_a_fixed); +fprintf(fid, ' },\n'); +fprintf(fid, ' {\n'); +fprintf(fid, ' "step": 5,\n'); +fprintf(fid, ' "name": "island_extraction",\n'); +desc = 'Extract largest connected island'; +fprintf(fid, ' "description": "%s",\n', desc); +fprintf(fid, ' "islands_total": %d,\n', cleaning_log.islands_total); +fprintf(fid, ' "main_island_buses": %d,\n', cleaning_log.main_island_buses); +fprintf(fid, ' "excluded_buses": %d\n', cleaning_log.excluded_buses); +fprintf(fid, ' },\n'); +fprintf(fid, ' {\n'); +fprintf(fid, ' "step": 6,\n'); +fprintf(fid, ' "name": "single_slack_bus",\n'); +desc = 'Ensure exactly one slack (type-3) bus'; +fprintf(fid, ' "description": "%s",\n', desc); +fprintf(fid, ' "action": "%s",\n', cleaning_log.slack_action); +fprintf(fid, ' "slack_bus": %d\n', cleaning_log.slack_bus); +fprintf(fid, ' }\n'); +fprintf(fid, ' ],\n'); +fprintf(fid, ' "output_files": [\n'); +fprintf(fid, ' "fnm_main_island.m",\n'); +fprintf(fid, ' "fnm_main_island.mat"\n'); +fprintf(fid, ' ],\n'); +fprintf(fid, ' "cleaned_network": {\n'); +fprintf(fid, ' "buses": %d,\n', n_buses_clean); +fprintf(fid, ' "branches_total": %d,\n', n_branches_clean); +fprintf(fid, ' "branches_active": %d,\n', n_branches_active); +fprintf(fid, ' "generators_total": %d,\n', n_gens_clean); +fprintf(fid, ' "generators_active": %d,\n', n_gens_active); +fprintf(fid, ' "loads_nonzero": %d,\n', n_loads); +fprintf(fid, ' "baseMVA": %.1f\n', mpc_main.baseMVA); +fprintf(fid, ' },\n'); +fprintf(fid, ' "matpower_version": "8.1",\n'); +note = 'Import directly -- all cleaning pre-applied'; +fprintf(fid, ' "note": "%s"\n', note); +fprintf(fid, '}\n'); +fclose(fid); +fprintf('Cleaning manifest written: %s\n', manifest_path); + +% Reload mpc_main into 'mpc' alias for solve stages +mpc = mpc_main; + +% ========================================================================= +% Stage 5: DCPF Reference Solution +% ========================================================================= +fprintf('\n=== Stage 5: DCPF Reference Solution ===\n'); + +mpopt_dc = mpoption('verbose', 2, 'out.all', 0); +tic; +results_dc = rundcpf(mpc, mpopt_dc); +dcpf_time = toc; + +if results_dc.success + fprintf('DCPF CONVERGED in %.2f seconds\n', dcpf_time); +else + fprintf('DCPF FAILED\n'); +end + +% Write DCPF outputs +dcpf_dir = fullfile(fnm_dir, 'reference', 'dcpf'); +if ~exist(dcpf_dir, 'dir') + mkdir(dcpf_dir); +end + +% rundcpf returns results in external numbering +results_dc_ext = results_dc; + +% buses_dcpf.csv: bus_number, va_deg, pd_mw, base_kv, bus_type +bus_dc = results_dc_ext.bus; +fid = fopen(fullfile(dcpf_dir, 'buses_dcpf.csv'), 'w'); +fprintf(fid, 'bus_number,va_deg,pd_mw,base_kv,bus_type\n'); +for i = 1:size(bus_dc, 1) + % Skip isolated buses (type 4) + if bus_dc(i, 2) == 4 + continue + end + fprintf(fid, '%d,%.8f,%.4f,%.2f,%d\n', ... + bus_dc(i, 1), bus_dc(i, 9), bus_dc(i, 3), bus_dc(i, 10), bus_dc(i, 2)); +end +fclose(fid); + +% branches_dcpf.csv: from_bus, to_bus, pf_mw, status +br_dc = results_dc_ext.branch; +fid = fopen(fullfile(dcpf_dir, 'branches_dcpf.csv'), 'w'); +fprintf(fid, 'from_bus,to_bus,pf_mw,status\n'); +for i = 1:size(br_dc, 1) + if br_dc(i, 11) == 0 + continue + end % skip out-of-service + fprintf(fid, '%d,%d,%.8f,%d\n', ... + br_dc(i, 1), br_dc(i, 2), br_dc(i, 14), br_dc(i, 11)); +end +fclose(fid); + +% Find slack bus +slack_idx = find(bus_dc(:, 2) == 3); +if ~isempty(slack_idx) + slack_bus = bus_dc(slack_idx(1), 1); + slack_angle = bus_dc(slack_idx(1), 9); +else + slack_bus = -1; + slack_angle = 0; +end + +% Count non-isolated buses +n_active_buses = sum(bus_dc(:, 2) ~= 4); +n_active_branches = sum(br_dc(:, 11) ~= 0); +n_active_gens = sum(results_dc_ext.gen(:, 8) > 0); + +% summary_dcpf.json +total_gen_mw = sum(results_dc_ext.gen(results_dc_ext.gen(:, 8) > 0, 2)); +total_load_mw = sum(bus_dc(bus_dc(:, 2) ~= 4, 3)); + +fid = fopen(fullfile(dcpf_dir, 'summary_dcpf.json'), 'w'); +fprintf(fid, '{\n'); +fprintf(fid, ' "success": %d,\n', results_dc.success); +fprintf(fid, ' "wall_clock_seconds": %.4f,\n', dcpf_time); +fprintf(fid, ' "total_gen_mw": %.4f,\n', total_gen_mw); +fprintf(fid, ' "total_load_mw": %.4f,\n', total_load_mw); +fprintf(fid, ' "slack_bus": %d,\n', slack_bus); +fprintf(fid, ' "slack_angle": %.8f,\n', slack_angle); +fprintf(fid, ' "n_buses": %d,\n', n_active_buses); +fprintf(fid, ' "n_branches": %d,\n', n_active_branches); +fprintf(fid, ' "n_gens": %d,\n', n_active_gens); +fprintf(fid, ' "main_island_only": true\n'); +fprintf(fid, '}\n'); +fclose(fid); +fprintf('DCPF reference written to %s\n', dcpf_dir); + +% ========================================================================= +% Stage 6: ACPF Reference Solution +% ========================================================================= +fprintf('\n=== Stage 6: ACPF Reference Solution ===\n'); + +% Initialize: use PV bus voltage setpoints from gen table, DCPF angles +mpc_ac = mpc; +mpc_ac.bus(:, 8) = 1.0; % VM flat start +mpc_ac.bus(:, 9) = results_dc_ext.bus(:, 9); % DC warm start angles + +% Set PV bus VM to generator voltage setpoints (col 6 in gen table) +for i = 1:size(mpc_ac.gen, 1) + if mpc_ac.gen(i, 8) > 0 % in-service + gen_bus = mpc_ac.gen(i, 1); + bus_idx = find(mpc_ac.bus(:, 1) == gen_bus); + if ~isempty(bus_idx) + mpc_ac.bus(bus_idx(1), 8) = mpc_ac.gen(i, 6); % VG setpoint + end + end +end +fprintf('Initialized VM from gen setpoints for %d PV/slack buses\n', ... + sum(mpc_ac.bus(:, 2) >= 2 & mpc_ac.bus(:, 2) <= 3)); +fprintf('Initialized VA from DCPF angles\n'); + +% ---- Attempt 1: Fast Decoupled XB (more robust for large networks) ---- +fprintf('\n--- Attempt 1: Fast Decoupled XB ---\n'); +mpopt_fd = mpoption('verbose', 2, 'out.all', 0); +mpopt_fd = mpoption(mpopt_fd, 'pf.alg', 'FDXB'); +mpopt_fd = mpoption(mpopt_fd, 'pf.tol', 1e-8); +mpopt_fd = mpoption(mpopt_fd, 'pf.fd.max_it', 1000); +mpopt_fd = mpoption(mpopt_fd, 'pf.enforce_q_lims', 0); + +tic; +results_ac = runpf(mpc_ac, mpopt_fd); +acpf_time = toc; + +if ~results_ac.success + % ---- Attempt 2: Relaxed tolerance FDXB ---- + fprintf('\n--- Attempt 2: FDXB with relaxed tolerance (1e-4) ---\n'); + mpopt_fd2 = mpoption(mpopt_fd, 'pf.tol', 1e-4); + tic; + results_ac = runpf(mpc_ac, mpopt_fd2); + acpf_time = toc; + + if results_ac.success + fprintf('FDXB converged at 1e-4, refining with NR...\n'); + % Refine with NR from the FD solution + mpopt_nr = mpoption('verbose', 2, 'out.all', 0); + mpopt_nr = mpoption(mpopt_nr, 'pf.alg', 'NR'); + mpopt_nr = mpoption(mpopt_nr, 'pf.tol', 1e-8); + mpopt_nr = mpoption(mpopt_nr, 'pf.nr.max_it', 50); + mpopt_nr = mpoption(mpopt_nr, 'pf.enforce_q_lims', 0); + tic; + results_refined = runpf(results_ac, mpopt_nr); + refine_time = toc; + if results_refined.success + results_ac = results_refined; + acpf_time = acpf_time + refine_time; + fprintf('NR refinement converged in %.2f seconds\n', refine_time); + else + fprintf('NR refinement failed -- using FD solution at 1e-4\n'); + end + end +end + +if ~results_ac.success + % ---- Attempt 3: Newton-Raphson from DC warm start ---- + fprintf('\n--- Attempt 3: Newton-Raphson from DC warm start ---\n'); + mpopt_nr = mpoption('verbose', 2, 'out.all', 0); + mpopt_nr = mpoption(mpopt_nr, 'pf.alg', 'NR'); + mpopt_nr = mpoption(mpopt_nr, 'pf.tol', 1e-8); + mpopt_nr = mpoption(mpopt_nr, 'pf.nr.max_it', 100); + mpopt_nr = mpoption(mpopt_nr, 'pf.enforce_q_lims', 0); + tic; + results_ac = runpf(mpc_ac, mpopt_nr); + acpf_time = toc; +end + +if ~results_ac.success + % ---- Attempt 4: Gauss-Seidel warm-up then NR ---- + fprintf('\n--- Attempt 4: Gauss-Seidel (200 iter) then NR ---\n'); + mpopt_gs = mpoption('verbose', 2, 'out.all', 0); + mpopt_gs = mpoption(mpopt_gs, 'pf.alg', 'GS'); + mpopt_gs = mpoption(mpopt_gs, 'pf.tol', 1e-2); + mpopt_gs = mpoption(mpopt_gs, 'pf.gs.max_it', 200); + mpopt_gs = mpoption(mpopt_gs, 'pf.enforce_q_lims', 0); + results_gs = runpf(mpc_ac, mpopt_gs); + if results_gs.success + mpopt_nr2 = mpoption('verbose', 2, 'out.all', 0); + mpopt_nr2 = mpoption(mpopt_nr2, 'pf.tol', 1e-8); + mpopt_nr2 = mpoption(mpopt_nr2, 'pf.nr.max_it', 100); + mpopt_nr2 = mpoption(mpopt_nr2, 'pf.enforce_q_lims', 0); + tic; + results_ac = runpf(results_gs, mpopt_nr2); + acpf_time = toc; + end +end + +% Q-limit enforcement stage (if base converged) +q_enforced = false; +if results_ac.success + fprintf('ACPF base CONVERGED in %.2f seconds (%d iterations)\n', ... + acpf_time, results_ac.iterations); + + mpopt_ql = mpoption('verbose', 2, 'out.all', 0); + mpopt_ql = mpoption(mpopt_ql, 'pf.tol', 1e-8); + mpopt_ql = mpoption(mpopt_ql, 'pf.nr.max_it', 100); + mpopt_ql = mpoption(mpopt_ql, 'pf.enforce_q_lims', 1); + tic; + results_ql = runpf(results_ac, mpopt_ql); + ql_time = toc; + + if results_ql.success + fprintf('ACPF Q-limit enforcement CONVERGED in %.2f seconds\n', ql_time); + results_ac = results_ql; + q_enforced = true; + acpf_time = acpf_time + ql_time; + else + fprintf('Q-limit enforcement FAILED -- using base solution\n'); + end +else + fprintf('ACPF FAILED on all attempts\n'); +end + +% Write ACPF outputs +acpf_dir = fullfile(fnm_dir, 'reference', 'acpf'); +if ~exist(acpf_dir, 'dir') + mkdir(acpf_dir); +end + +% runpf returns results in external numbering +results_ac_ext = results_ac; + +% Remove isolated buses from count +bus_ac = results_ac_ext.bus; +n_isolated = sum(bus_ac(:, 2) == 4); + +% buses_acpf.csv: bus_number, vm_pu, va_deg, pd_mw, qd_mvar, bus_type +fid = fopen(fullfile(acpf_dir, 'buses_acpf.csv'), 'w'); +fprintf(fid, 'bus_number,vm_pu,va_deg,pd_mw,qd_mvar,bus_type\n'); +for i = 1:size(bus_ac, 1) + if bus_ac(i, 2) == 4 + continue + end + fprintf(fid, '%d,%.8f,%.8f,%.4f,%.4f,%d\n', ... + bus_ac(i, 1), bus_ac(i, 8), bus_ac(i, 9), bus_ac(i, 3), bus_ac(i, 4), bus_ac(i, 2)); +end +fclose(fid); + +% branches_acpf.csv: from_bus, to_bus, pf_mw, qf_mvar, pt_mw, qt_mvar, status +br_ac = results_ac_ext.branch; +fid = fopen(fullfile(acpf_dir, 'branches_acpf.csv'), 'w'); +fprintf(fid, 'from_bus,to_bus,pf_mw,qf_mvar,pt_mw,qt_mvar,status\n'); +for i = 1:size(br_ac, 1) + if br_ac(i, 11) == 0 + continue + end + fprintf(fid, '%d,%d,%.8f,%.8f,%.8f,%.8f,%d\n', ... + br_ac(i, 1), br_ac(i, 2), br_ac(i, 14), br_ac(i, 15), ... + br_ac(i, 16), br_ac(i, 17), br_ac(i, 11)); +end +fclose(fid); + +% generators_acpf.csv: bus_number, pg_mw, qg_mvar, status, vm_setpoint +gen_ac = results_ac_ext.gen; +fid = fopen(fullfile(acpf_dir, 'generators_acpf.csv'), 'w'); +fprintf(fid, 'bus_number,pg_mw,qg_mvar,status,vm_setpoint\n'); +for i = 1:size(gen_ac, 1) + if gen_ac(i, 8) <= 0 + continue + end + fprintf(fid, '%d,%.8f,%.8f,%d,%.8f\n', ... + gen_ac(i, 1), gen_ac(i, 2), gen_ac(i, 3), gen_ac(i, 8), gen_ac(i, 6)); +end +fclose(fid); + +% summary_acpf.json +active_gens = gen_ac(gen_ac(:, 8) > 0, :); +active_buses = bus_ac(bus_ac(:, 2) ~= 4, :); +total_gen_mw_ac = sum(active_gens(:, 2)); +total_gen_mvar_ac = sum(active_gens(:, 3)); +total_load_mw_ac = sum(active_buses(:, 3)); +total_load_mvar_ac = sum(active_buses(:, 4)); +losses_mw = total_gen_mw_ac - total_load_mw_ac; + +fid = fopen(fullfile(acpf_dir, 'summary_acpf.json'), 'w'); +fprintf(fid, '{\n'); +fprintf(fid, ' "success": %d,\n', results_ac.success); +fprintf(fid, ' "wall_clock_seconds": %.4f,\n', acpf_time); +fprintf(fid, ' "iterations": %d,\n', results_ac.iterations); +fprintf(fid, ' "total_gen_mw": %.4f,\n', total_gen_mw_ac); +fprintf(fid, ' "total_gen_mvar": %.4f,\n', total_gen_mvar_ac); +fprintf(fid, ' "total_load_mw": %.4f,\n', total_load_mw_ac); +fprintf(fid, ' "total_load_mvar": %.4f,\n', total_load_mvar_ac); +fprintf(fid, ' "losses_mw": %.4f,\n', losses_mw); +fprintf(fid, ' "n_buses": %d,\n', size(active_buses, 1)); +fprintf(fid, ' "n_branches": %d,\n', sum(br_ac(:, 11) ~= 0)); +fprintf(fid, ' "n_gens": %d,\n', size(active_gens, 1)); +fprintf(fid, ' "solver": "Newton-Raphson",\n'); +fprintf(fid, ' "tolerance": 1e-8,\n'); +fprintf(fid, ' "q_limits_enforced": %s,\n', mat2str(q_enforced)); +fprintf(fid, ' "isolated_buses_removed": %d,\n', n_isolated); +fprintf(fid, ' "main_island_only": true,\n'); +fprintf(fid, ' "initial_conditions": "dcpf_warm_start_flat_vm"\n'); +fprintf(fid, '}\n'); +fclose(fid); +fprintf('ACPF reference written to %s\n', acpf_dir); + +fprintf('\n=== DONE ===\n'); diff --git a/data/fnm/scripts/solved_snapshot.py b/data/fnm/scripts/solved_snapshot.py new file mode 100644 index 00000000..02b685d8 --- /dev/null +++ b/data/fnm/scripts/solved_snapshot.py @@ -0,0 +1,966 @@ +"""Solved-snapshot confirmation for FNM Annual S01 RAW file. + +Analyzes parsed bus and generator CSV data to determine whether the FNM file +contains a converged AC power flow (ACPF) solution or flat-start initial +conditions. Classification is based on three statistical indicators: voltage +magnitude (VM) distribution, voltage angle (VA) spread, and generator reactive +power (Qg) population. + +Produces both JSON (machine-readable) and markdown (human-readable) output files +containing the classification, supporting statistics, and Phase 3 implications. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import statistics +import sys +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from enum import Enum +from pathlib import Path + +# --------------------------------------------------------------------------- +# Constants / Thresholds +# --------------------------------------------------------------------------- + +FLOAT_TOLERANCE: float = 1e-10 + +# Voltage magnitude thresholds +VM_STD_SOLVED_THRESHOLD: float = 0.01 +VM_PCT_EXACT_SOLVED_THRESHOLD: float = 50.0 +VM_STD_FLAT_THRESHOLD: float = 0.001 +VM_PCT_EXACT_FLAT_THRESHOLD: float = 95.0 + +# Voltage angle thresholds +VA_STD_SOLVED_THRESHOLD: float = 0.5 +VA_PCT_EXACT_SOLVED_THRESHOLD: float = 50.0 +VA_STD_FLAT_THRESHOLD: float = 0.01 +VA_PCT_EXACT_FLAT_THRESHOLD: float = 95.0 + +# Generator Qg thresholds +QG_PCT_NONZERO_SOLVED_THRESHOLD: float = 50.0 +QG_PCT_NONZERO_FLAT_THRESHOLD: float = 5.0 + +# MATPOWER bus matrix column indices (standard 13-column format, no header) +_MPC_BUS_COL_BUS_I: int = 0 +_MPC_BUS_COL_TYPE: int = 1 +_MPC_BUS_COL_VM: int = 7 +_MPC_BUS_COL_VA: int = 8 + +# MATPOWER gen matrix column indices (no header) +_MPC_GEN_COL_BUS: int = 0 +_MPC_GEN_COL_QG: int = 2 + +# PSS/E bus type for isolated buses +_ISOLATED_BUS_TYPE: int = 4 + + +# --------------------------------------------------------------------------- +# Enums +# --------------------------------------------------------------------------- + + +class SnapshotClassification(Enum): + """Overall classification of the solved-snapshot analysis.""" + + SOLVED = "solved" + FLAT_START = "flat_start" + INDETERMINATE = "indeterminate" + + +class IndicatorSignal(Enum): + """Sub-classification for an individual indicator.""" + + SOLVED_SIGNAL = "solved_signal" + FLAT_SIGNAL = "flat_signal" + AMBIGUOUS = "ambiguous" + + +# --------------------------------------------------------------------------- +# Data Classes +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class DistributionStats: + """Descriptive statistics for a distribution of values. + + Attributes: + count: Number of values analyzed. + mean: Arithmetic mean. + std: Population standard deviation. + min: Minimum value. + max: Maximum value. + pct_exact_reference: Percentage of values exactly equal to the + reference value (within FLOAT_TOLERANCE). + """ + + count: int + mean: float + std: float + min: float + max: float + pct_exact_reference: float + + +@dataclass(frozen=True) +class GeneratorQgStats: + """Statistics for generator reactive power output. + + Attributes: + total_generators: Total number of generators analyzed. + generators_with_nonzero_qg: Count with Qg != 0 (beyond tolerance). + pct_nonzero_qg: Percentage with non-zero Qg. + mean_abs_qg: Mean of absolute Qg values. + min_qg: Minimum Qg value. + max_qg: Maximum Qg value. + """ + + total_generators: int + generators_with_nonzero_qg: int + pct_nonzero_qg: float + mean_abs_qg: float + min_qg: float + max_qg: float + + +@dataclass(frozen=True) +class IndicatorResult: + """Result of a single indicator classification. + + Attributes: + name: Indicator name (e.g. "VM", "VA", "Qg"). + signal: The sub-classification for this indicator. + rationale: Human-readable explanation of the classification. + """ + + name: str + signal: IndicatorSignal + rationale: str + + +@dataclass(frozen=True) +class ConfirmationMetadata: + """Metadata about the confirmation analysis run. + + Attributes: + bus_csv_path: Path to the bus CSV file analyzed. + generator_csv_path: Path to the generator CSV file analyzed. + canonical_parser: Name of the canonical parser that produced the CSVs. + timestamp: ISO 8601 timestamp of the analysis. + float_tolerance: Tolerance used for floating-point equality checks. + """ + + bus_csv_path: str = "" + generator_csv_path: str = "" + canonical_parser: str = "" + timestamp: str = "" + float_tolerance: float = FLOAT_TOLERANCE + + +@dataclass(frozen=True) +class SnapshotConfirmation: + """Complete solved-snapshot confirmation result. + + Attributes: + classification: Overall snapshot classification. + vm_stats: Voltage magnitude distribution statistics. + va_stats: Voltage angle distribution statistics. + qg_stats: Generator reactive power statistics. + vm_indicator: VM indicator classification result. + va_indicator: VA indicator classification result. + qg_indicator: Qg indicator classification result. + phase3_implications: Text describing Phase 3 strategy implications. + buses_analyzed: Number of non-isolated buses analyzed. + buses_excluded_isolated: Number of isolated (IDE=4) buses excluded. + metadata: Analysis metadata. + """ + + classification: SnapshotClassification + vm_stats: DistributionStats + va_stats: DistributionStats + qg_stats: GeneratorQgStats + vm_indicator: IndicatorResult + va_indicator: IndicatorResult + qg_indicator: IndicatorResult + phase3_implications: str + buses_analyzed: int + buses_excluded_isolated: int + metadata: ConfirmationMetadata + + +# --------------------------------------------------------------------------- +# CSV Detection Helpers +# --------------------------------------------------------------------------- + + +def _is_header_row(row: list[str]) -> bool: + """Determine if a CSV row is a header (non-numeric first field). + + Args: + row: A list of string values from a CSV row. + + Returns: + True if the row appears to be a header rather than data. + """ + if not row: + return False + try: + float(row[0]) + return False + except ValueError: + return True + + +def _detect_column_index(headers: list[str], candidates: list[str]) -> int | None: + """Find the index of the first matching header from a list of candidates. + + Args: + headers: List of column header names (lowercased). + candidates: Candidate column names to search for, in priority order. + + Returns: + Column index if found, None otherwise. + """ + lower_headers = [h.strip().lower() for h in headers] + for candidate in candidates: + if candidate.lower() in lower_headers: + return lower_headers.index(candidate.lower()) + return None + + +# --------------------------------------------------------------------------- +# Data Loading +# --------------------------------------------------------------------------- + + +def load_bus_data(bus_csv_path: Path) -> tuple[list[float], list[float], int]: + """Load parsed bus data and return VM, VA values excluding isolated buses. + + Supports both header-bearing CSVs (GridCal) and headerless CSVs (MATPOWER). + Isolated buses (type/IDE = 4) are excluded from the returned lists. + + Args: + bus_csv_path: Path to the bus CSV file. + + Returns: + A tuple of (vm_values, va_values, isolated_count) where vm_values and + va_values are lists of floats for non-isolated buses and isolated_count + is the number of excluded isolated buses. + + Raises: + FileNotFoundError: If the CSV file does not exist. + ValueError: If the CSV has no usable data rows. + """ + if not bus_csv_path.exists(): + raise FileNotFoundError(f"Bus CSV not found: {bus_csv_path}") + + vm_values: list[float] = [] + va_values: list[float] = [] + isolated_count: int = 0 + + with open(bus_csv_path, encoding="utf-8") as f: + reader = csv.reader(f) + rows = list(reader) + + if not rows: + raise ValueError(f"Bus CSV is empty: {bus_csv_path}") + + # Detect header vs headerless + first_row = rows[0] + if _is_header_row(first_row): + # Header-bearing CSV (e.g. GridCal) + headers = first_row + data_rows = rows[1:] + type_idx = _detect_column_index(headers, ["type", "ide", "bus_type"]) + vm_idx = _detect_column_index(headers, ["vm", "vm_pu", "Vm"]) + va_idx = _detect_column_index(headers, ["va", "va_deg", "Va"]) + if vm_idx is None or va_idx is None: + raise ValueError( + f"Cannot find VM/VA columns in headers: {headers}. " + "Expected columns named 'vm'/'va' or similar." + ) + else: + # Headerless CSV (MATPOWER format: 13-column bus matrix) + data_rows = rows + type_idx = _MPC_BUS_COL_TYPE + vm_idx = _MPC_BUS_COL_VM + va_idx = _MPC_BUS_COL_VA + + if not data_rows: + raise ValueError(f"Bus CSV has no data rows: {bus_csv_path}") + + for row in data_rows: + if not row or all(cell.strip() == "" for cell in row): + continue + + # Check bus type for isolation + if type_idx is not None and type_idx < len(row): + try: + bus_type = int(float(row[type_idx].strip())) + except (ValueError, IndexError): + bus_type = 0 + if bus_type == _ISOLATED_BUS_TYPE: + isolated_count += 1 + continue + + try: + vm = float(row[vm_idx].strip()) + va = float(row[va_idx].strip()) + except (ValueError, IndexError) as exc: + raise ValueError(f"Cannot parse VM/VA from row: {row}") from exc + + vm_values.append(vm) + va_values.append(va) + + if not vm_values: + raise ValueError(f"No non-isolated bus data found in: {bus_csv_path}") + + return vm_values, va_values, isolated_count + + +def load_generator_data(gen_csv_path: Path, bus_csv_path: Path) -> list[float]: + """Load parsed generator data and return Qg values. + + Excludes generators connected to isolated buses (type/IDE = 4) by + cross-referencing with the bus CSV. + + Supports both header-bearing CSVs (GridCal) and headerless CSVs (MATPOWER). + + Args: + gen_csv_path: Path to the generator CSV file. + bus_csv_path: Path to the bus CSV file (for isolated bus filtering). + + Returns: + List of Qg values for generators on non-isolated buses. + + Raises: + FileNotFoundError: If either CSV file does not exist. + ValueError: If the CSV has no usable data rows. + """ + if not gen_csv_path.exists(): + raise FileNotFoundError(f"Generator CSV not found: {gen_csv_path}") + + # Build set of isolated bus IDs from bus CSV + isolated_buses: set[int] = set() + if bus_csv_path.exists(): + with open(bus_csv_path, encoding="utf-8") as f: + reader = csv.reader(f) + bus_rows = list(reader) + + if bus_rows: + first_row = bus_rows[0] + if _is_header_row(first_row): + headers = first_row + bus_data = bus_rows[1:] + bus_i_idx = _detect_column_index(headers, ["bus_i", "i", "bus_id", "bus"]) + type_idx = _detect_column_index(headers, ["type", "ide", "bus_type"]) + else: + bus_data = bus_rows + bus_i_idx = _MPC_BUS_COL_BUS_I + type_idx = _MPC_BUS_COL_TYPE + + if bus_i_idx is not None and type_idx is not None: + for row in bus_data: + if not row or all(c.strip() == "" for c in row): + continue + try: + bus_id = int(float(row[bus_i_idx].strip())) + bus_type = int(float(row[type_idx].strip())) + except (ValueError, IndexError): + continue + if bus_type == _ISOLATED_BUS_TYPE: + isolated_buses.add(bus_id) + + # Read generator CSV + with open(gen_csv_path, encoding="utf-8") as f: + reader = csv.reader(f) + gen_rows = list(reader) + + if not gen_rows: + raise ValueError(f"Generator CSV is empty: {gen_csv_path}") + + first_row = gen_rows[0] + if _is_header_row(first_row): + headers = first_row + data_rows = gen_rows[1:] + qg_idx = _detect_column_index(headers, ["qg", "q", "Qg"]) + bus_idx = _detect_column_index(headers, ["bus", "bus_i", "i"]) + else: + data_rows = gen_rows + qg_idx = _MPC_GEN_COL_QG + bus_idx = _MPC_GEN_COL_BUS + + if qg_idx is None: + raise ValueError(f"Cannot find Qg column in generator CSV: {gen_csv_path}") + + qg_values: list[float] = [] + for row in data_rows: + if not row or all(c.strip() == "" for c in row): + continue + + # Filter out generators on isolated buses + if isolated_buses and bus_idx is not None and bus_idx < len(row): + try: + gen_bus = int(float(row[bus_idx].strip())) + except (ValueError, IndexError): + gen_bus = -1 + if gen_bus in isolated_buses: + continue + + try: + qg = float(row[qg_idx].strip()) + except (ValueError, IndexError) as exc: + raise ValueError(f"Cannot parse Qg from row: {row}") from exc + qg_values.append(qg) + + if not qg_values: + raise ValueError(f"No generator Qg data found in: {gen_csv_path}") + + return qg_values + + +# --------------------------------------------------------------------------- +# Statistics +# --------------------------------------------------------------------------- + + +def compute_distribution_stats(values: list[float], reference_value: float) -> DistributionStats: + """Compute descriptive statistics for a distribution of values. + + Args: + values: List of numeric values to analyze. + reference_value: Reference value for exact-match percentage + (e.g. 1.0 for VM, 0.0 for VA). + + Returns: + DistributionStats with computed metrics. + + Raises: + ValueError: If values is empty. + """ + if not values: + raise ValueError("Cannot compute statistics on empty values list.") + + n = len(values) + mean_val = statistics.mean(values) + std_val = statistics.pstdev(values) + min_val = min(values) + max_val = max(values) + + exact_count = sum(1 for v in values if abs(v - reference_value) < FLOAT_TOLERANCE) + pct_exact = (exact_count / n) * 100.0 + + return DistributionStats( + count=n, + mean=mean_val, + std=std_val, + min=min_val, + max=max_val, + pct_exact_reference=pct_exact, + ) + + +def compute_qg_stats(qg_values: list[float]) -> GeneratorQgStats: + """Compute generator reactive power statistics. + + Args: + qg_values: List of Qg values. + + Returns: + GeneratorQgStats with computed metrics. + + Raises: + ValueError: If qg_values is empty. + """ + if not qg_values: + raise ValueError("Cannot compute Qg statistics on empty values list.") + + n = len(qg_values) + nonzero_count = sum(1 for q in qg_values if abs(q) >= FLOAT_TOLERANCE) + pct_nonzero = (nonzero_count / n) * 100.0 + mean_abs = statistics.mean(abs(q) for q in qg_values) + min_qg = min(qg_values) + max_qg = max(qg_values) + + return GeneratorQgStats( + total_generators=n, + generators_with_nonzero_qg=nonzero_count, + pct_nonzero_qg=pct_nonzero, + mean_abs_qg=mean_abs, + min_qg=min_qg, + max_qg=max_qg, + ) + + +# --------------------------------------------------------------------------- +# Classification +# --------------------------------------------------------------------------- + + +def classify_vm(stats: DistributionStats) -> IndicatorResult: + """Classify the voltage magnitude indicator. + + Args: + stats: VM distribution statistics. + + Returns: + IndicatorResult with the VM sub-classification. + """ + if ( + stats.std > VM_STD_SOLVED_THRESHOLD + and stats.pct_exact_reference < VM_PCT_EXACT_SOLVED_THRESHOLD + ): + return IndicatorResult( + name="VM", + signal=IndicatorSignal.SOLVED_SIGNAL, + rationale=( + f"VM std={stats.std:.6f} > {VM_STD_SOLVED_THRESHOLD} and " + f"{stats.pct_exact_reference:.1f}% exact 1.0 < " + f"{VM_PCT_EXACT_SOLVED_THRESHOLD}%." + ), + ) + if ( + stats.std < VM_STD_FLAT_THRESHOLD + and stats.pct_exact_reference > VM_PCT_EXACT_FLAT_THRESHOLD + ): + return IndicatorResult( + name="VM", + signal=IndicatorSignal.FLAT_SIGNAL, + rationale=( + f"VM std={stats.std:.6f} < {VM_STD_FLAT_THRESHOLD} and " + f"{stats.pct_exact_reference:.1f}% exact 1.0 > " + f"{VM_PCT_EXACT_FLAT_THRESHOLD}%." + ), + ) + return IndicatorResult( + name="VM", + signal=IndicatorSignal.AMBIGUOUS, + rationale=( + f"VM std={stats.std:.6f}, {stats.pct_exact_reference:.1f}% exact 1.0. " + "Does not meet solved or flat-start criteria." + ), + ) + + +def classify_va(stats: DistributionStats) -> IndicatorResult: + """Classify the voltage angle indicator. + + Args: + stats: VA distribution statistics. + + Returns: + IndicatorResult with the VA sub-classification. + """ + if ( + stats.std > VA_STD_SOLVED_THRESHOLD + and stats.pct_exact_reference < VA_PCT_EXACT_SOLVED_THRESHOLD + ): + return IndicatorResult( + name="VA", + signal=IndicatorSignal.SOLVED_SIGNAL, + rationale=( + f"VA std={stats.std:.6f} > {VA_STD_SOLVED_THRESHOLD} and " + f"{stats.pct_exact_reference:.1f}% exact 0.0 < " + f"{VA_PCT_EXACT_SOLVED_THRESHOLD}%." + ), + ) + if ( + stats.std < VA_STD_FLAT_THRESHOLD + and stats.pct_exact_reference > VA_PCT_EXACT_FLAT_THRESHOLD + ): + return IndicatorResult( + name="VA", + signal=IndicatorSignal.FLAT_SIGNAL, + rationale=( + f"VA std={stats.std:.6f} < {VA_STD_FLAT_THRESHOLD} and " + f"{stats.pct_exact_reference:.1f}% exact 0.0 > " + f"{VA_PCT_EXACT_FLAT_THRESHOLD}%." + ), + ) + return IndicatorResult( + name="VA", + signal=IndicatorSignal.AMBIGUOUS, + rationale=( + f"VA std={stats.std:.6f}, {stats.pct_exact_reference:.1f}% exact 0.0. " + "Does not meet solved or flat-start criteria." + ), + ) + + +def classify_qg(stats: GeneratorQgStats) -> IndicatorResult: + """Classify the generator reactive power indicator. + + Args: + stats: Generator Qg statistics. + + Returns: + IndicatorResult with the Qg sub-classification. + """ + if stats.pct_nonzero_qg > QG_PCT_NONZERO_SOLVED_THRESHOLD: + return IndicatorResult( + name="Qg", + signal=IndicatorSignal.SOLVED_SIGNAL, + rationale=( + f"{stats.pct_nonzero_qg:.1f}% generators have non-zero Qg > " + f"{QG_PCT_NONZERO_SOLVED_THRESHOLD}%." + ), + ) + if stats.pct_nonzero_qg < QG_PCT_NONZERO_FLAT_THRESHOLD: + return IndicatorResult( + name="Qg", + signal=IndicatorSignal.FLAT_SIGNAL, + rationale=( + f"{stats.pct_nonzero_qg:.1f}% generators have non-zero Qg < " + f"{QG_PCT_NONZERO_FLAT_THRESHOLD}%." + ), + ) + return IndicatorResult( + name="Qg", + signal=IndicatorSignal.AMBIGUOUS, + rationale=( + f"{stats.pct_nonzero_qg:.1f}% generators have non-zero Qg. " + "Does not meet solved or flat-start criteria." + ), + ) + + +def classify_overall( + vm_result: IndicatorResult, + va_result: IndicatorResult, + qg_result: IndicatorResult, +) -> SnapshotClassification: + """Determine overall snapshot classification from three indicator results. + + Decision table: + - All three SOLVED_SIGNAL -> SOLVED + - All three FLAT_SIGNAL -> FLAT_START + - VM + VA both SOLVED_SIGNAL (any Qg) -> SOLVED + - VM + VA both FLAT_SIGNAL (any Qg) -> FLAT_START + - Otherwise -> INDETERMINATE + + Args: + vm_result: VM indicator result. + va_result: VA indicator result. + qg_result: Qg indicator result. + + Returns: + The overall SnapshotClassification. + """ + vm_sig = vm_result.signal + va_sig = va_result.signal + qg_sig = qg_result.signal + + all_solved = ( + vm_sig == IndicatorSignal.SOLVED_SIGNAL + and va_sig == IndicatorSignal.SOLVED_SIGNAL + and qg_sig == IndicatorSignal.SOLVED_SIGNAL + ) + all_flat = ( + vm_sig == IndicatorSignal.FLAT_SIGNAL + and va_sig == IndicatorSignal.FLAT_SIGNAL + and qg_sig == IndicatorSignal.FLAT_SIGNAL + ) + vm_va_solved = ( + vm_sig == IndicatorSignal.SOLVED_SIGNAL and va_sig == IndicatorSignal.SOLVED_SIGNAL + ) + vm_va_flat = vm_sig == IndicatorSignal.FLAT_SIGNAL and va_sig == IndicatorSignal.FLAT_SIGNAL + + if all_solved or vm_va_solved: + return SnapshotClassification.SOLVED + if all_flat or vm_va_flat: + return SnapshotClassification.FLAT_START + return SnapshotClassification.INDETERMINATE + + +# --------------------------------------------------------------------------- +# Phase 3 Implications +# --------------------------------------------------------------------------- + + +def derive_phase3_implications( + classification: SnapshotClassification, +) -> str: + """Derive Phase 3 strategy implications from the classification. + + Args: + classification: The overall snapshot classification. + + Returns: + A human-readable string describing Phase 3 implications. + """ + if classification == SnapshotClassification.SOLVED: + return ( + "The FNM RAW file contains a converged ACPF solution. Phase 3 can extract " + "DCPF and ACPF reference solutions directly from the parsed bus and generator " + "data without running a solver. This significantly simplifies the reference " + "solution pipeline and eliminates solver-dependence for Phase 3 verification." + ) + if classification == SnapshotClassification.FLAT_START: + return ( + "The FNM RAW file contains flat-start initial conditions (VM=1.0, VA=0.0, " + "Qg=0.0 everywhere). Phase 3 must first converge the network using a verified " + "AC power flow solver before extracting reference solutions. This adds solver " + "selection, convergence verification, and cross-validation steps to the Phase 3 " + "pipeline." + ) + return ( + "The FNM RAW file classification is indeterminate — the data does not clearly " + "indicate either a converged solution or flat-start conditions. Manual inspection " + "of the parsed data is recommended before proceeding with Phase 3 planning. " + "Consider examining individual bus voltage profiles and generator dispatch patterns." + ) + + +# --------------------------------------------------------------------------- +# Build Confirmation +# --------------------------------------------------------------------------- + + +def build_confirmation( + bus_csv_path: Path, + gen_csv_path: Path, + canonical_parser: str = "", +) -> SnapshotConfirmation: + """Build a complete solved-snapshot confirmation from CSV files. + + Orchestrates data loading, statistics computation, classification, and + metadata assembly into a single SnapshotConfirmation result. + + Args: + bus_csv_path: Path to the canonical parser's bus CSV output. + gen_csv_path: Path to the canonical parser's generator CSV output. + canonical_parser: Name of the parser that produced the CSVs. + + Returns: + A fully populated SnapshotConfirmation. + """ + vm_values, va_values, isolated_count = load_bus_data(bus_csv_path) + qg_values = load_generator_data(gen_csv_path, bus_csv_path) + + vm_stats = compute_distribution_stats(vm_values, reference_value=1.0) + va_stats = compute_distribution_stats(va_values, reference_value=0.0) + qg_stats = compute_qg_stats(qg_values) + + vm_indicator = classify_vm(vm_stats) + va_indicator = classify_va(va_stats) + qg_indicator = classify_qg(qg_stats) + + classification = classify_overall(vm_indicator, va_indicator, qg_indicator) + phase3_text = derive_phase3_implications(classification) + + metadata = ConfirmationMetadata( + bus_csv_path=str(bus_csv_path), + generator_csv_path=str(gen_csv_path), + canonical_parser=canonical_parser, + timestamp=datetime.now(timezone.utc).isoformat(), + float_tolerance=FLOAT_TOLERANCE, + ) + + return SnapshotConfirmation( + classification=classification, + vm_stats=vm_stats, + va_stats=va_stats, + qg_stats=qg_stats, + vm_indicator=vm_indicator, + va_indicator=va_indicator, + qg_indicator=qg_indicator, + phase3_implications=phase3_text, + buses_analyzed=len(vm_values), + buses_excluded_isolated=isolated_count, + metadata=metadata, + ) + + +# --------------------------------------------------------------------------- +# Serialization +# --------------------------------------------------------------------------- + + +def confirmation_to_dict(confirmation: SnapshotConfirmation) -> dict: + """Convert a SnapshotConfirmation to a JSON-serializable dict. + + Enum values are converted to their string representations. + + Args: + confirmation: The confirmation result to serialize. + + Returns: + A dict suitable for json.dumps(). + """ + d = asdict(confirmation) + + # Convert enum values to strings + d["classification"] = confirmation.classification.value + d["vm_indicator"]["signal"] = confirmation.vm_indicator.signal.value + d["va_indicator"]["signal"] = confirmation.va_indicator.signal.value + d["qg_indicator"]["signal"] = confirmation.qg_indicator.signal.value + + return d + + +def confirmation_to_markdown(confirmation: SnapshotConfirmation) -> str: + """Convert a SnapshotConfirmation to a human-readable markdown document. + + Args: + confirmation: The confirmation result to render. + + Returns: + A markdown-formatted string. + """ + c = confirmation + lines: list[str] = [] + + lines.append("# Solved-Snapshot Confirmation Report") + lines.append("") + lines.append(f"**Classification:** `{c.classification.value}`") + lines.append(f"**Timestamp:** {c.metadata.timestamp}") + lines.append(f"**Canonical Parser:** {c.metadata.canonical_parser or 'N/A'}") + lines.append("") + + lines.append("## Summary") + lines.append("") + lines.append(f"- Buses analyzed: {c.buses_analyzed}") + lines.append(f"- Buses excluded (isolated, type=4): {c.buses_excluded_isolated}") + lines.append(f"- Generators analyzed: {c.qg_stats.total_generators}") + lines.append(f"- Float tolerance: {c.metadata.float_tolerance}") + lines.append("") + + lines.append("## Indicator Results") + lines.append("") + + # VM + lines.append("### Voltage Magnitude (VM)") + lines.append("") + lines.append(f"- **Signal:** `{c.vm_indicator.signal.value}`") + lines.append(f"- **Rationale:** {c.vm_indicator.rationale}") + lines.append(f"- Count: {c.vm_stats.count}") + lines.append(f"- Mean: {c.vm_stats.mean:.6f} p.u.") + lines.append(f"- Std: {c.vm_stats.std:.6f} p.u.") + lines.append(f"- Min: {c.vm_stats.min:.6f} p.u.") + lines.append(f"- Max: {c.vm_stats.max:.6f} p.u.") + lines.append(f"- % Exact 1.0: {c.vm_stats.pct_exact_reference:.2f}%") + lines.append("") + + # VA + lines.append("### Voltage Angle (VA)") + lines.append("") + lines.append(f"- **Signal:** `{c.va_indicator.signal.value}`") + lines.append(f"- **Rationale:** {c.va_indicator.rationale}") + lines.append(f"- Count: {c.va_stats.count}") + lines.append(f"- Mean: {c.va_stats.mean:.6f} deg") + lines.append(f"- Std: {c.va_stats.std:.6f} deg") + lines.append(f"- Min: {c.va_stats.min:.6f} deg") + lines.append(f"- Max: {c.va_stats.max:.6f} deg") + lines.append(f"- % Exact 0.0: {c.va_stats.pct_exact_reference:.2f}%") + lines.append("") + + # Qg + lines.append("### Generator Reactive Power (Qg)") + lines.append("") + lines.append(f"- **Signal:** `{c.qg_indicator.signal.value}`") + lines.append(f"- **Rationale:** {c.qg_indicator.rationale}") + lines.append(f"- Total generators: {c.qg_stats.total_generators}") + lines.append(f"- Generators with non-zero Qg: {c.qg_stats.generators_with_nonzero_qg}") + lines.append(f"- % Non-zero Qg: {c.qg_stats.pct_nonzero_qg:.2f}%") + lines.append(f"- Mean |Qg|: {c.qg_stats.mean_abs_qg:.4f} MVAr") + lines.append(f"- Qg range: [{c.qg_stats.min_qg:.4f}, {c.qg_stats.max_qg:.4f}] MVAr") + lines.append("") + + lines.append("## Overall Classification") + lines.append("") + lines.append("| Indicator | Signal |") + lines.append("|-----------|--------|") + lines.append(f"| VM | `{c.vm_indicator.signal.value}` |") + lines.append(f"| VA | `{c.va_indicator.signal.value}` |") + lines.append(f"| Qg | `{c.qg_indicator.signal.value}` |") + lines.append(f"| **Overall** | **`{c.classification.value}`** |") + lines.append("") + + lines.append("## Phase 3 Implications") + lines.append("") + lines.append(c.phase3_implications) + lines.append("") + + lines.append("## Input Files") + lines.append("") + lines.append(f"- Bus CSV: `{c.metadata.bus_csv_path}`") + lines.append(f"- Generator CSV: `{c.metadata.generator_csv_path}`") + lines.append("") + + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> None: + """CLI entry point for solved-snapshot confirmation. + + Parses command-line arguments, runs the analysis, and writes JSON and + markdown output files. + + Args: + argv: Command-line arguments. Defaults to sys.argv[1:]. + """ + parser = argparse.ArgumentParser( + description="Analyze FNM parsed data to determine solved vs flat-start status." + ) + parser.add_argument( + "--bus-csv", + type=Path, + required=True, + help="Path to the canonical parser's bus CSV output.", + ) + parser.add_argument( + "--gen-csv", + type=Path, + required=True, + help="Path to the canonical parser's generator CSV output.", + ) + parser.add_argument( + "--parser", + type=str, + default="", + help="Name of the canonical parser (e.g. 'MATPOWER', 'GRIDCAL').", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path("."), + help="Directory for output files (default: current directory).", + ) + + args = parser.parse_args(argv if argv is not None else sys.argv[1:]) + + confirmation = build_confirmation( + bus_csv_path=args.bus_csv, + gen_csv_path=args.gen_csv, + canonical_parser=args.parser, + ) + + output_dir: Path = args.output_dir + output_dir.mkdir(parents=True, exist_ok=True) + + # Write JSON report + json_path = output_dir / "solved_snapshot_report.json" + report_dict = confirmation_to_dict(confirmation) + json_path.write_text(json.dumps(report_dict, indent=2) + "\n", encoding="utf-8") + + # Write markdown report + md_path = output_dir / "solved_snapshot_report.md" + md_text = confirmation_to_markdown(confirmation) + md_path.write_text(md_text, encoding="utf-8") + + print(f"Classification: {confirmation.classification.value}") + print(f"JSON report: {json_path}") + print(f"Markdown report: {md_path}") + + +if __name__ == "__main__": + main() diff --git a/data/fnm/scripts/test_dcpf_reference_separate_tables.py b/data/fnm/scripts/test_dcpf_reference_separate_tables.py new file mode 100644 index 00000000..4bca2cbd --- /dev/null +++ b/data/fnm/scripts/test_dcpf_reference_separate_tables.py @@ -0,0 +1,547 @@ +"""Tests for dcpf_reference.py separate-table support (PRD 00/03). + +Tests cover: +- load_transformer_table: PSS/E column mapping, normalization, error handling +- load_manifest / resolve_base_mva: manifest loading and baseMVA resolution +- run_dcpf_reference with transformer_csv_path: separate-table vs merged parity +- CLI argument parsing for --transformer-csv and --manifest flags +""" + +from __future__ import annotations + +import json +import textwrap +from pathlib import Path + +import pytest + +from fnm.scripts.dcpf_reference import ( + load_manifest, + load_transformer_table, + main, + resolve_base_mva, + run_dcpf_reference, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _write_csv(path: Path, text: str) -> Path: + """Write dedented CSV text to a file and return the path.""" + path.write_text(textwrap.dedent(text).lstrip(), encoding="utf-8") + return path + + +def _write_bus_csv(tmp_path: Path, rows: list[tuple]) -> Path: + """Write a minimal bus CSV. Rows are (bus_i, bus_type, pd, base_kv).""" + tmp_path.mkdir(parents=True, exist_ok=True) + lines = ["BUS_I,BUS_TYPE,PD,BASE_KV"] + for r in rows: + lines.append(",".join(str(v) for v in r)) + p = tmp_path / "bus.csv" + p.write_text("\n".join(lines) + "\n", encoding="utf-8") + return p + + +def _write_gen_csv(tmp_path: Path, rows: list[tuple]) -> Path: + """Write a minimal gen CSV. Rows are (gen_bus, pg, gen_status, id).""" + lines = ["GEN_BUS,PG,GEN_STATUS,ID"] + for r in rows: + lines.append(",".join(str(v) for v in r)) + p = tmp_path / "gen.csv" + p.write_text("\n".join(lines) + "\n", encoding="utf-8") + return p + + +def _write_branch_csv(tmp_path: Path, rows: list[tuple]) -> Path: + """Write a minimal branch CSV. Rows are (f_bus, t_bus, br_x, tap, shift, br_status, ckt).""" + lines = ["F_BUS,T_BUS,BR_X,TAP,SHIFT,BR_STATUS,CKT"] + for r in rows: + lines.append(",".join(str(v) for v in r)) + p = tmp_path / "branch.csv" + p.write_text("\n".join(lines) + "\n", encoding="utf-8") + return p + + +def _write_exclusion_csv(tmp_path: Path, bus_numbers: list[int] | None = None) -> Path: + """Write a minimal exclusion CSV.""" + lines = ["bus_number"] + for bn in bus_numbers or []: + lines.append(str(bn)) + p = tmp_path / "excluded_buses.csv" + p.write_text("\n".join(lines) + "\n", encoding="utf-8") + return p + + +def _write_transformer_csv(tmp_path: Path, rows: list[tuple], fname: str = "xfmr.csv") -> Path: + """Write a transformer CSV with PSS/E columns. + + Rows are (I, J, X1_2, WINDV1, ANG1, STAT, CKT). + """ + lines = ["I,J,X1_2,WINDV1,ANG1,STAT,CKT"] + for r in rows: + lines.append(",".join(str(v) for v in r)) + p = tmp_path / fname + p.write_text("\n".join(lines) + "\n", encoding="utf-8") + return p + + +# --------------------------------------------------------------------------- +# 1. test_load_transformer_table_reads_psse_columns +# --------------------------------------------------------------------------- + + +def test_load_transformer_table_reads_psse_columns(tmp_path: Path) -> None: + """3 synthetic rows — verify BranchRecord field mappings.""" + p = _write_transformer_csv( + tmp_path, + [ + (1, 2, 0.05, 1.05, 0.0, 1, "1"), + (3, 4, 0.10, 0.95, 5.0, 1, "2"), + (5, 6, 0.20, 1.00, -3.0, 1, "1"), + ], + ) + records = load_transformer_table(p) + + assert len(records) == 3 + + assert records[0].from_bus == 1 + assert records[0].to_bus == 2 + assert records[0].x_pu == pytest.approx(0.05) + assert records[0].tap_ratio == pytest.approx(1.05) + assert records[0].shift_deg == pytest.approx(0.0) + assert records[0].status == 1 + assert records[0].circuit_id == "1" + assert records[0].is_transformer is True + + assert records[1].from_bus == 3 + assert records[1].to_bus == 4 + assert records[1].x_pu == pytest.approx(0.10) + assert records[1].tap_ratio == pytest.approx(0.95) + assert records[1].shift_deg == pytest.approx(5.0) + assert records[1].circuit_id == "2" + + assert records[2].shift_deg == pytest.approx(-3.0) + + +# --------------------------------------------------------------------------- +# 2. test_load_transformer_table_normalizes_windv1_zero +# --------------------------------------------------------------------------- + + +def test_load_transformer_table_normalizes_windv1_zero(tmp_path: Path) -> None: + """WINDV1=0 should be normalized to tap_ratio=1.0.""" + p = _write_transformer_csv(tmp_path, [(1, 2, 0.05, 0.0, 0.0, 1, "1")]) + records = load_transformer_table(p) + assert records[0].tap_ratio == pytest.approx(1.0) + + +# --------------------------------------------------------------------------- +# 3. test_load_transformer_table_stat_mapping +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("stat_in", "status_out"), + [(0, 0), (1, 1), (2, 1), (3, 1), (4, 1)], +) +def test_load_transformer_table_stat_mapping(tmp_path: Path, stat_in: int, status_out: int) -> None: + """PSS/E STAT 0-4 maps correctly to BranchRecord status.""" + p = _write_transformer_csv(tmp_path, [(1, 2, 0.05, 1.0, 0.0, stat_in, "1")]) + records = load_transformer_table(p) + assert records[0].status == status_out + + +# --------------------------------------------------------------------------- +# 4. test_load_transformer_table_missing_required_column +# --------------------------------------------------------------------------- + + +def test_load_transformer_table_missing_required_column(tmp_path: Path) -> None: + """Missing X1_2 column raises ValueError.""" + p = tmp_path / "bad_xfmr.csv" + p.write_text("I,J,STAT\n1,2,1\n", encoding="utf-8") + with pytest.raises(ValueError, match="Required columns not found"): + load_transformer_table(p) + + +# --------------------------------------------------------------------------- +# 5. test_load_transformer_table_file_not_found +# --------------------------------------------------------------------------- + + +def test_load_transformer_table_file_not_found(tmp_path: Path) -> None: + """Non-existent file raises FileNotFoundError.""" + with pytest.raises(FileNotFoundError): + load_transformer_table(tmp_path / "nonexistent.csv") + + +# --------------------------------------------------------------------------- +# 6. test_transformer_column_map_autodetects_variants +# --------------------------------------------------------------------------- + + +def test_transformer_column_map_autodetects_variants(tmp_path: Path) -> None: + """Variant column names (from_bus, to_bus, br_x, etc.) are auto-detected.""" + p = tmp_path / "variant_xfmr.csv" + p.write_text( + "from_bus,to_bus,br_x,tap,shift,status,circuit\n1,2,0.05,1.0,0.0,1,1\n", + encoding="utf-8", + ) + records = load_transformer_table(p) + assert len(records) == 1 + assert records[0].from_bus == 1 + assert records[0].to_bus == 2 + assert records[0].x_pu == pytest.approx(0.05) + + +# --------------------------------------------------------------------------- +# 7. test_load_manifest_reads_sbase +# --------------------------------------------------------------------------- + + +def test_load_manifest_reads_sbase(tmp_path: Path) -> None: + """Manifest with sbase == 100.0 is correctly parsed.""" + p = tmp_path / "manifest.json" + p.write_text(json.dumps({"sbase": 100.0, "version": 7}), encoding="utf-8") + m = load_manifest(p) + assert m["sbase"] == 100.0 + + +# --------------------------------------------------------------------------- +# 8. test_load_manifest_file_not_found +# --------------------------------------------------------------------------- + + +def test_load_manifest_file_not_found(tmp_path: Path) -> None: + """Non-existent manifest raises FileNotFoundError.""" + with pytest.raises(FileNotFoundError): + load_manifest(tmp_path / "nope.json") + + +# --------------------------------------------------------------------------- +# 9. test_load_manifest_invalid_json +# --------------------------------------------------------------------------- + + +def test_load_manifest_invalid_json(tmp_path: Path) -> None: + """Invalid JSON raises ValueError.""" + p = tmp_path / "bad.json" + p.write_text("{not valid json", encoding="utf-8") + with pytest.raises(ValueError, match="Invalid JSON"): + load_manifest(p) + + +# --------------------------------------------------------------------------- +# 10. test_resolve_base_mva_cli_overrides_manifest +# --------------------------------------------------------------------------- + + +def test_resolve_base_mva_cli_overrides_manifest() -> None: + """CLI value (200.0) overrides manifest sbase (100.0).""" + base, source = resolve_base_mva(200.0, {"sbase": 100.0}) + assert base == pytest.approx(200.0) + assert source == "cli" + + +# --------------------------------------------------------------------------- +# 11. test_resolve_base_mva_manifest_overrides_default +# --------------------------------------------------------------------------- + + +def test_resolve_base_mva_manifest_overrides_default() -> None: + """Manifest sbase (150.0) overrides the default (100.0).""" + base, source = resolve_base_mva(None, {"sbase": 150.0}) + assert base == pytest.approx(150.0) + assert source == "manifest" + + +# --------------------------------------------------------------------------- +# 12. test_resolve_base_mva_falls_back_to_default +# --------------------------------------------------------------------------- + + +def test_resolve_base_mva_falls_back_to_default() -> None: + """Neither CLI nor manifest -> falls back to 100.0.""" + base, source = resolve_base_mva(None, None) + assert base == pytest.approx(100.0) + assert source == "default" + + +# --------------------------------------------------------------------------- +# Synthetic 5-bus network helpers +# --------------------------------------------------------------------------- + + +def _make_5bus_network(tmp_path: Path, *, separate: bool) -> dict[str, Path]: + """Create a synthetic 5-bus network for integration tests. + + Topology: + Bus 1 (slack, type=3) -- branch --> Bus 2 (PV, type=2) + Bus 2 -- branch --> Bus 3 (PQ, type=1) + Bus 3 -- branch --> Bus 4 (PQ, type=1) + Bus 1 -- transformer --> Bus 5 (PQ, type=1) + + Generator at bus 1: 150 MW + Loads: bus 2=50, bus 3=40, bus 4=30, bus 5=30 (total=150) + """ + bus_csv = _write_bus_csv( + tmp_path, + [ + (1, 3, 0.0, 230.0), # slack + (2, 2, 50.0, 230.0), # PV + (3, 1, 40.0, 230.0), # PQ + (4, 1, 30.0, 230.0), # PQ + (5, 1, 30.0, 115.0), # PQ + ], + ) + gen_csv = _write_gen_csv(tmp_path, [(1, 150.0, 1, "1")]) + excl_csv = _write_exclusion_csv(tmp_path) + + # Lines: 1-2, 2-3, 3-4 + line_rows = [ + (1, 2, 0.05, 0.0, 0.0, 1, "1"), + (2, 3, 0.10, 0.0, 0.0, 1, "1"), + (3, 4, 0.08, 0.0, 0.0, 1, "1"), + ] + + # Transformer: 1-5 with tap=1.05 + xfmr_row = (1, 5, 0.06, 1.05, 0.0, 1, "1") + + paths: dict[str, Path] = { + "bus": bus_csv, + "gen": gen_csv, + "excl": excl_csv, + } + + if separate: + # Lines only in branch CSV, transformer in separate file + paths["branch"] = _write_branch_csv(tmp_path, line_rows) + paths["xfmr"] = _write_transformer_csv(tmp_path, [xfmr_row]) + else: + # Merged: lines + transformer in a single branch CSV + # Convert transformer to branch format (tap=1.05 not 0) + merged_rows = line_rows + [xfmr_row] + paths["branch"] = _write_branch_csv(tmp_path, merged_rows) + + return paths + + +# --------------------------------------------------------------------------- +# 13. test_run_dcpf_separate_tables_produces_same_solution +# --------------------------------------------------------------------------- + + +def test_run_dcpf_separate_tables_produces_same_solution(tmp_path: Path) -> None: + """Separate-table path produces the same bus angles and branch flows as merged.""" + merged_dir = tmp_path / "merged" + merged_dir.mkdir() + sep_dir = tmp_path / "separate" + sep_dir.mkdir() + + merged = _make_5bus_network(merged_dir, separate=False) + sep = _make_5bus_network(sep_dir, separate=True) + + out_merged = tmp_path / "out_merged" + out_sep = tmp_path / "out_sep" + + sol_merged = run_dcpf_reference( + bus_csv_path=merged["bus"], + gen_csv_path=merged["gen"], + branch_csv_path=merged["branch"], + exclusion_csv_path=merged["excl"], + output_dir=out_merged, + base_mva=100.0, + ) + sol_sep = run_dcpf_reference( + bus_csv_path=sep["bus"], + gen_csv_path=sep["gen"], + branch_csv_path=sep["branch"], + exclusion_csv_path=sep["excl"], + output_dir=out_sep, + base_mva=100.0, + transformer_csv_path=sep["xfmr"], + ) + + # Bus angles must match + assert set(sol_merged.bus_angles_deg.keys()) == set(sol_sep.bus_angles_deg.keys()) + for bus in sol_merged.bus_angles_deg: + assert sol_merged.bus_angles_deg[bus] == pytest.approx( + sol_sep.bus_angles_deg[bus], abs=1e-6 + ) + + # Branch flow count must match + assert len(sol_merged.branch_flows_mw) == len(sol_sep.branch_flows_mw) + + # Flows must match (sort for deterministic comparison) + merged_flows = sorted( + sol_merged.branch_flows_mw, key=lambda f: (f.from_bus, f.to_bus, f.circuit_id) + ) + sep_flows = sorted(sol_sep.branch_flows_mw, key=lambda f: (f.from_bus, f.to_bus, f.circuit_id)) + for mf, sf in zip(merged_flows, sep_flows): + assert mf.p_flow_mw == pytest.approx(sf.p_flow_mw, abs=1e-6) + + +# --------------------------------------------------------------------------- +# 14. test_run_dcpf_legacy_path_unchanged +# --------------------------------------------------------------------------- + + +def test_run_dcpf_legacy_path_unchanged(tmp_path: Path) -> None: + """Merged CSV without --transformer-csv produces the same result as before.""" + net = _make_5bus_network(tmp_path, separate=False) + out = tmp_path / "out" + + sol = run_dcpf_reference( + bus_csv_path=net["bus"], + gen_csv_path=net["gen"], + branch_csv_path=net["branch"], + exclusion_csv_path=net["excl"], + output_dir=out, + base_mva=100.0, + ) + + # Slack bus angle is 0 + assert sol.bus_angles_deg[1] == pytest.approx(0.0) + # 4 in-service branches (3 lines + 1 transformer in merged) + assert sol.active_branch_count == 4 + # Output files exist + assert (out / "buses_dcpf.csv").exists() + assert (out / "branches_dcpf.csv").exists() + assert (out / "summary_dcpf.json").exists() + + +# --------------------------------------------------------------------------- +# 15. test_main_cli_accepts_transformer_csv_flag +# --------------------------------------------------------------------------- + + +def test_main_cli_accepts_transformer_csv_flag(tmp_path: Path) -> None: + """--transformer-csv flag is accepted and passed to run_dcpf_reference.""" + net = _make_5bus_network(tmp_path / "data", separate=True) + out = tmp_path / "out" + + argv = [ + "--bus-csv", + str(net["bus"]), + "--gen-csv", + str(net["gen"]), + "--branch-csv", + str(net["branch"]), + "--exclusion-csv", + str(net["excl"]), + "--transformer-csv", + str(net["xfmr"]), + "-o", + str(out), + ] + + with pytest.raises(SystemExit) as exc_info: + main(argv) + assert exc_info.value.code == 0 + + +# --------------------------------------------------------------------------- +# 16. test_main_cli_accepts_manifest_flag +# --------------------------------------------------------------------------- + + +def test_main_cli_accepts_manifest_flag(tmp_path: Path) -> None: + """--manifest flag is accepted and sbase is used for baseMVA.""" + net = _make_5bus_network(tmp_path / "data", separate=False) + out = tmp_path / "out" + + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text(json.dumps({"sbase": 100.0}), encoding="utf-8") + + argv = [ + "--bus-csv", + str(net["bus"]), + "--gen-csv", + str(net["gen"]), + "--branch-csv", + str(net["branch"]), + "--exclusion-csv", + str(net["excl"]), + "--manifest", + str(manifest_path), + "-o", + str(out), + ] + + with pytest.raises(SystemExit) as exc_info: + main(argv) + assert exc_info.value.code == 0 + + +# --------------------------------------------------------------------------- +# 17. test_main_cli_base_mva_overrides_manifest +# --------------------------------------------------------------------------- + + +def test_main_cli_base_mva_overrides_manifest(tmp_path: Path) -> None: + """--base-mva overrides manifest sbase.""" + net = _make_5bus_network(tmp_path / "data", separate=False) + out = tmp_path / "out" + + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text(json.dumps({"sbase": 200.0}), encoding="utf-8") + + argv = [ + "--bus-csv", + str(net["bus"]), + "--gen-csv", + str(net["gen"]), + "--branch-csv", + str(net["branch"]), + "--exclusion-csv", + str(net["excl"]), + "--manifest", + str(manifest_path), + "--base-mva", + "100.0", + "-o", + str(out), + ] + + with pytest.raises(SystemExit) as exc_info: + main(argv) + assert exc_info.value.code == 0 + + # Verify the summary has base_mva=100, not 200 + summary = json.loads((out / "summary_dcpf.json").read_text(encoding="utf-8")) + assert summary["base_mva"] == pytest.approx(100.0) + + +# --------------------------------------------------------------------------- +# 18. test_main_cli_backward_compat_no_new_flags +# --------------------------------------------------------------------------- + + +def test_main_cli_backward_compat_no_new_flags(tmp_path: Path) -> None: + """CLI without --transformer-csv or --manifest works exactly as before.""" + net = _make_5bus_network(tmp_path / "data", separate=False) + out = tmp_path / "out" + + argv = [ + "--bus-csv", + str(net["bus"]), + "--gen-csv", + str(net["gen"]), + "--branch-csv", + str(net["branch"]), + "--exclusion-csv", + str(net["excl"]), + "-o", + str(out), + ] + + with pytest.raises(SystemExit) as exc_info: + main(argv) + assert exc_info.value.code == 0 + + assert (out / "buses_dcpf.csv").exists() + assert (out / "branches_dcpf.csv").exists() + assert (out / "summary_dcpf.json").exists() diff --git a/data/fnm/scripts/test_export_intermediate_csvs.py b/data/fnm/scripts/test_export_intermediate_csvs.py new file mode 100644 index 00000000..087bedf1 --- /dev/null +++ b/data/fnm/scripts/test_export_intermediate_csvs.py @@ -0,0 +1,569 @@ +"""Tests for the export pipeline script (PRD 00/01). + +Integration tests that read real data files are marked with skipif guards +for when the data files are not present. +""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path + +import pytest + +from fnm.scripts.export_intermediate_csvs import ( + MatpowerCase, + TableExport, + build_manifest, + export_table_to_csv, + filter_rows_by_bus, + load_excluded_buses, + load_matpower_case, + normalize_tap_ratio, + run_export_pipeline, + split_branches_and_transformers, + validate_csv_against_schema, + validate_manifest_against_schema, +) +from fnm.scripts.raw_record_counter import PSSE_V31_SECTION_NAMES + +# --------------------------------------------------------------------------- +# Path constants for integration tests +# --------------------------------------------------------------------------- + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +_MAIN_CHECKOUT = _REPO_ROOT + +# The .mat file may be in the worktree or main checkout +_MAT_PATH = _REPO_ROOT / "data" / "fnm" / "reference" / "cleaned" / "fnm_main_island.mat" +_EXCLUDED_JSON = _REPO_ROOT / "data" / "fnm" / "reference" / "excluded_buses.json" +_SCHEMA_DIR = _REPO_ROOT / "data" / "fnm" / "intermediate" / "schemas" + +# Check main checkout if worktree doesn't have the files +if not _MAT_PATH.exists(): + # Try the git main working tree + _git_file = _REPO_ROOT / ".git" + if _git_file.is_file(): + _git_text = _git_file.read_text().strip() + if _git_text.startswith("gitdir:"): + _main_git = Path(_git_text.split(":", 1)[1].strip()) + _main_repo = _main_git.parent.parent.parent + _alt_mat = _main_repo / "data" / "fnm" / "reference" / "cleaned" / "fnm_main_island.mat" + _alt_excl = _main_repo / "data" / "fnm" / "reference" / "excluded_buses.json" + _alt_schema = _main_repo / "data" / "fnm" / "intermediate" / "schemas" + if _alt_mat.exists(): + _MAT_PATH = _alt_mat + if _alt_excl.exists(): + _EXCLUDED_JSON = _alt_excl + if _alt_schema.exists(): + _SCHEMA_DIR = _alt_schema + +_HAS_MAT = _MAT_PATH.exists() +_HAS_EXCLUDED = _EXCLUDED_JSON.exists() +_HAS_SCHEMAS = _SCHEMA_DIR.exists() +_HAS_ALL_DATA = _HAS_MAT and _HAS_EXCLUDED and _HAS_SCHEMAS + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def bus_schema(tmp_path: Path) -> Path: + """Create a minimal bus schema for testing.""" + schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Bus", + "type": "object", + "properties": { + "I": {"type": "integer"}, + "NAME": {"type": "string"}, + "BASKV": {"type": "number"}, + "IDE": {"type": "integer"}, + }, + "required": ["I", "NAME", "BASKV", "IDE"], + "additionalProperties": False, + } + path = tmp_path / "bus.schema.json" + path.write_text(json.dumps(schema, indent=2), encoding="utf-8") + return path + + +@pytest.fixture +def manifest_schema(tmp_path: Path) -> Path: + """Create the manifest schema for testing.""" + schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Intermediate Format Manifest", + "type": "object", + "properties": { + "sbase": {"type": "number"}, + "basfrq": {"type": "number"}, + "rev": {"type": "number"}, + "case_id": {"type": "string"}, + "canonical_parser": { + "type": "string", + "enum": ["matpower", "gridcal"], + }, + "tables": {"type": "array"}, + "total_records": {"type": "integer", "minimum": 0}, + "total_tables": {"type": "integer", "minimum": 1}, + "non_empty_record_types": {"type": "array"}, + "schema_version": {"type": "string"}, + "generated_timestamp": {"type": "string", "format": "date-time"}, + }, + "required": [ + "sbase", + "basfrq", + "rev", + "case_id", + "canonical_parser", + "tables", + "total_records", + "total_tables", + "non_empty_record_types", + "schema_version", + "generated_timestamp", + ], + "additionalProperties": False, + } + path = tmp_path / "manifest.schema.json" + path.write_text(json.dumps(schema, indent=2), encoding="utf-8") + return path + + +@pytest.fixture +def synthetic_branch_matrix() -> list[list[float]]: + """Synthetic MATPOWER branch matrix: 3 plain branches + 2 transformers. + + Branch columns: fbus, tbus, r, x, b, rateA, rateB, rateC, tap, shift, status + """ + return [ + # Plain branches (tap=0, shift=0) + [1, 2, 0.01, 0.1, 0.02, 100, 100, 100, 0, 0, 1, -360, 360], + [2, 3, 0.02, 0.2, 0.03, 200, 200, 200, 0, 0, 1, -360, 360], + [1, 3, 0.03, 0.3, 0.04, 300, 300, 300, 0, 0, 1, -360, 360], + # Transformers (tap != 0) + [1, 4, 0.01, 0.1, 0.0, 100, 100, 100, 1.05, 0, 1, -360, 360], + [3, 5, 0.02, 0.2, 0.0, 200, 200, 200, 0, 30.0, 1, -360, 360], + ] + + +# --------------------------------------------------------------------------- +# Test 1: test_load_matpower_case_extracts_basemva +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif(not _HAS_MAT, reason="requires FNM .mat file") +def test_load_matpower_case_extracts_basemva(): + case = load_matpower_case(_MAT_PATH) + assert case.baseMVA == 100.0 + + +# --------------------------------------------------------------------------- +# Test 2: test_load_matpower_case_extracts_bus_matrix_shape +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif(not _HAS_MAT, reason="requires FNM .mat file") +def test_load_matpower_case_extracts_bus_matrix_shape(): + # NOTE: PRD specifies 30000 (pre-filter) but the cleaned .mat file + # has 28000 buses (post island-extraction). We match the actual data. + case = load_matpower_case(_MAT_PATH) + assert len(case.bus) == ~28, 000 + assert len(case.bus[0]) == 13 + + +# --------------------------------------------------------------------------- +# Test 3: test_load_excluded_buses_count +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif(not _HAS_EXCLUDED, reason="requires excluded_buses.json") +def test_load_excluded_buses_count(): + excluded = load_excluded_buses(_EXCLUDED_JSON) + assert len(excluded) == 2445 + + +# --------------------------------------------------------------------------- +# Test 4: test_split_branches_separates_transformers +# --------------------------------------------------------------------------- + + +def test_split_branches_separates_transformers( + synthetic_branch_matrix: list[list[float]], +): + bus_numbers = {1, 2, 3, 4, 5} + branches, transformers = split_branches_and_transformers(synthetic_branch_matrix, bus_numbers) + assert len(branches) == 3 + assert len(transformers) == 2 + + +# --------------------------------------------------------------------------- +# Test 5: test_normalize_tap_zero_becomes_one +# --------------------------------------------------------------------------- + + +def test_normalize_tap_zero_becomes_one(): + assert normalize_tap_ratio(0.0) == 1.0 + assert normalize_tap_ratio(1.05) == 1.05 + + +# --------------------------------------------------------------------------- +# Test 6: test_normalize_tap_on_transformer_rows +# --------------------------------------------------------------------------- + + +def test_normalize_tap_on_transformer_rows( + synthetic_branch_matrix: list[list[float]], +): + bus_numbers = {1, 2, 3, 4, 5} + _, transformers = split_branches_and_transformers(synthetic_branch_matrix, bus_numbers) + for t in transformers: + assert t["WINDV1"] != 0.0, f"WINDV1 should not be 0.0: {t}" + + +# --------------------------------------------------------------------------- +# Test 7: test_filter_rows_removes_excluded_buses +# --------------------------------------------------------------------------- + + +def test_filter_rows_removes_excluded_buses(): + rows: list[dict[str, int | float | str]] = [ + {"I": 1, "NAME": "A"}, + {"I": 2, "NAME": "B"}, + {"I": 3, "NAME": "C"}, + ] + bus_numbers = {1, 3} + filtered = filter_rows_by_bus(rows, bus_numbers, bus_key="I") + assert len(filtered) == 2 + assert {int(r["I"]) for r in filtered} == {1, 3} + + +# --------------------------------------------------------------------------- +# Test 8: test_filter_branch_rows_removes_both_endpoints +# --------------------------------------------------------------------------- + + +def test_filter_branch_rows_removes_both_endpoints(): + rows: list[dict[str, int | float | str]] = [ + {"I": 1, "J": 2, "CKT": "1 "}, + {"I": 2, "J": 3, "CKT": "1 "}, + {"I": 1, "J": 3, "CKT": "1 "}, + ] + bus_numbers = {1, 3} + # Manual filter: both endpoints must be in bus_numbers + filtered = [r for r in rows if int(r["I"]) in bus_numbers and int(r["J"]) in bus_numbers] + assert len(filtered) == 1 + assert int(filtered[0]["I"]) == 1 + assert int(filtered[0]["J"]) == 3 + + +# --------------------------------------------------------------------------- +# Test 9: test_export_csv_column_order_matches_schema +# --------------------------------------------------------------------------- + + +def test_export_csv_column_order_matches_schema( + tmp_path: Path, + bus_schema: Path, +): + rows = [ + {"I": 1, "NAME": "BUS1", "BASKV": 138.0, "IDE": 1}, + {"I": 2, "NAME": "BUS2", "BASKV": 345.0, "IDE": 2}, + ] + csv_path = tmp_path / "bus.csv" + export_table_to_csv(rows, bus_schema, csv_path) + + with open(csv_path, encoding="utf-8") as f: + reader = csv.reader(f) + header = next(reader) + + schema = json.loads(bus_schema.read_text(encoding="utf-8")) + expected_cols = list(schema["properties"].keys()) + assert header == expected_cols + + +# --------------------------------------------------------------------------- +# Test 10: test_export_csv_integer_fields_no_decimal +# --------------------------------------------------------------------------- + + +def test_export_csv_integer_fields_no_decimal( + tmp_path: Path, + bus_schema: Path, +): + rows = [ + {"I": 42, "NAME": "BUS42", "BASKV": 138.0, "IDE": 3}, + ] + csv_path = tmp_path / "bus.csv" + export_table_to_csv(rows, bus_schema, csv_path) + + with open(csv_path, encoding="utf-8") as f: + reader = csv.DictReader(f) + row = next(reader) + + # Integer fields should not have .0 + assert row["I"] == "42" + assert row["IDE"] == "3" + assert "." not in row["I"] + assert "." not in row["IDE"] + + +# --------------------------------------------------------------------------- +# Test 11: test_manifest_contains_all_tables +# --------------------------------------------------------------------------- + + +def test_manifest_contains_all_tables(): + # Build a minimal case + 17 table exports + case = MatpowerCase( + baseMVA=100.0, + version="2", + bus=[], + gen=[], + branch=[], + gencost=[], + areas=[], + bus_name=[], + dcline=[], + ) + + table_exports = [] + for rt in PSSE_V31_SECTION_NAMES: + table_name = rt.lower().replace(" ", "_").replace("-", "_") + table_exports.append( + TableExport( + table_name=table_name, + record_type=rt, + file_name=f"{table_name}.csv", + file_path=Path(f"{table_name}.csv"), + record_count=0, + column_count=5, + schema_file=f"{table_name}.schema.json", + ) + ) + + manifest = build_manifest(case, table_exports) + assert manifest.total_tables == 17 + manifest_types = {te.record_type for te in manifest.tables} + expected_types = set(PSSE_V31_SECTION_NAMES) + assert manifest_types == expected_types + + +# --------------------------------------------------------------------------- +# Test 12: test_manifest_sbase_matches_case +# --------------------------------------------------------------------------- + + +def test_manifest_sbase_matches_case(): + case = MatpowerCase( + baseMVA=100.0, + version="2", + bus=[], + gen=[], + branch=[], + gencost=[], + areas=[], + bus_name=[], + dcline=[], + ) + manifest = build_manifest(case, []) + assert manifest.sbase == 100.0 + + +# --------------------------------------------------------------------------- +# Test 13: test_manifest_total_records_is_sum +# --------------------------------------------------------------------------- + + +def test_manifest_total_records_is_sum(): + case = MatpowerCase( + baseMVA=100.0, + version="2", + bus=[], + gen=[], + branch=[], + gencost=[], + areas=[], + bus_name=[], + dcline=[], + ) + + exports = [ + TableExport( + table_name="bus", + record_type="Bus", + file_name="bus.csv", + file_path=Path("bus.csv"), + record_count=10, + column_count=13, + schema_file="bus.schema.json", + ), + TableExport( + table_name="generator", + record_type="Generator", + file_name="generator.csv", + file_path=Path("generator.csv"), + record_count=5, + column_count=28, + schema_file="generator.schema.json", + ), + ] + + manifest = build_manifest(case, exports) + assert manifest.total_records == 15 + assert manifest.total_records == sum(te.record_count for te in exports) + + +# --------------------------------------------------------------------------- +# Test 14: test_validate_bus_csv_passes_schema +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif(not _HAS_ALL_DATA, reason="requires FNM data files") +def test_validate_bus_csv_passes_schema(tmp_path: Path): + run_export_pipeline( + mat_path=_MAT_PATH, + excluded_buses_path=_EXCLUDED_JSON, + schema_dir=_SCHEMA_DIR, + output_dir=tmp_path / "export", + ) + + bus_csv = tmp_path / "export" / "bus.csv" + bus_schema = _SCHEMA_DIR / "bus.schema.json" + assert bus_csv.exists() + assert bus_schema.exists() + + vr = validate_csv_against_schema(bus_csv, bus_schema) + assert vr.is_valid, f"Bus CSV validation failed: {vr.errors}" + + +# --------------------------------------------------------------------------- +# Test 15: test_validate_manifest_passes_schema +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif(not _HAS_ALL_DATA, reason="requires FNM data files") +def test_validate_manifest_passes_schema(tmp_path: Path): + run_export_pipeline( + mat_path=_MAT_PATH, + excluded_buses_path=_EXCLUDED_JSON, + schema_dir=_SCHEMA_DIR, + output_dir=tmp_path / "export", + ) + + manifest_path = tmp_path / "export" / "manifest.json" + manifest_schema = _SCHEMA_DIR / "manifest.schema.json" + assert manifest_path.exists() + assert manifest_schema.exists() + + vr = validate_manifest_against_schema(manifest_path, manifest_schema) + assert vr.is_valid, f"Manifest validation failed: {vr.errors}" + + +# --------------------------------------------------------------------------- +# Test 16: test_validate_csv_detects_invalid_row +# --------------------------------------------------------------------------- + + +def test_validate_csv_detects_invalid_row( + tmp_path: Path, + bus_schema: Path, +): + csv_path = tmp_path / "bad_bus.csv" + with open(csv_path, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + writer.writerow(["I", "NAME", "BASKV", "IDE"]) + # Valid row + writer.writerow(["1", "BUS1", "138.0", "1"]) + # Invalid row: IDE not an integer-parseable string but we write + # a string that would fail int casting; however jsonschema checks + # the typed value. Let's put a missing required field by omitting NAME. + # Rewrite with explicit missing field + with open(csv_path, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + writer.writerow(["I", "BASKV", "IDE"]) # Missing NAME column + writer.writerow(["1", "138.0", "1"]) + + # Create a schema that requires NAME + schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Bus", + "type": "object", + "properties": { + "I": {"type": "integer"}, + "NAME": {"type": "string"}, + "BASKV": {"type": "number"}, + "IDE": {"type": "integer"}, + }, + "required": ["I", "NAME", "BASKV", "IDE"], + "additionalProperties": False, + } + schema_path = tmp_path / "bus_strict.schema.json" + schema_path.write_text(json.dumps(schema), encoding="utf-8") + + vr = validate_csv_against_schema(csv_path, schema_path) + assert not vr.is_valid + assert len(vr.errors) > 0 + + +# --------------------------------------------------------------------------- +# Test 17: test_empty_table_produces_header_only_csv +# --------------------------------------------------------------------------- + + +def test_empty_table_produces_header_only_csv( + tmp_path: Path, + bus_schema: Path, +): + csv_path = tmp_path / "empty.csv" + te = export_table_to_csv([], bus_schema, csv_path) + + assert te.record_count == 0 + + with open(csv_path, encoding="utf-8") as f: + lines = f.readlines() + + # Should have exactly one line (the header) + assert len(lines) == 1 + assert "I" in lines[0] + + +# --------------------------------------------------------------------------- +# Test 18: test_run_export_pipeline_end_to_end +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif(not _HAS_ALL_DATA, reason="requires FNM data files") +def test_run_export_pipeline_end_to_end(tmp_path: Path): + output_dir = tmp_path / "export" + result = run_export_pipeline( + mat_path=_MAT_PATH, + excluded_buses_path=_EXCLUDED_JSON, + schema_dir=_SCHEMA_DIR, + output_dir=output_dir, + ) + + # All CSVs exist + for te in result.table_exports: + assert te.file_path.exists(), f"Missing CSV: {te.file_name}" + + # Manifest exists + manifest_path = output_dir / "manifest.json" + assert manifest_path.exists() + + # All validations pass + assert result.success, f"Pipeline failed: {result.errors}" + + # Bus count should be 28000 (main island) + bus_export = next(te for te in result.table_exports if te.record_type == "Bus") + assert bus_export.record_count == ~28, 000 + + # Manifest has all 17 tables + assert result.manifest.total_tables == 17 + + # Total records > 0 + assert result.manifest.total_records > 0 diff --git a/data/fnm/scripts/test_validate_dcpf_reproducibility.py b/data/fnm/scripts/test_validate_dcpf_reproducibility.py new file mode 100644 index 00000000..959fbf94 --- /dev/null +++ b/data/fnm/scripts/test_validate_dcpf_reproducibility.py @@ -0,0 +1,548 @@ +"""Tests for DCPF Reference Reproducibility Validation. + +Tests 1-4 and 15-16 read actual committed data files and are skipped if files +don't exist. Tests 5-14, 17-18 use synthetic data and are self-contained. +""" + +from __future__ import annotations + +import csv +import json +import sys +import tempfile +from pathlib import Path + +import pytest + +# The scripts directory must be on the path for imports to work +SCRIPTS_DIR = Path(__file__).resolve().parent +if str(SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPTS_DIR)) + +from validate_dcpf_reproducibility import ( # noqa: E402 + BranchComparison, + BusComparison, + ReproducibilityReport, + SummaryComparison, + SummaryFieldCheck, + compare_branch_flows, + compare_bus_angles, + compare_summaries, + load_reference_branches, + load_reference_buses, + load_reference_summary, + main, + run_validation, + write_report, +) + +# --------------------------------------------------------------------------- +# Paths to committed reference data +# --------------------------------------------------------------------------- + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +_REF_DIR = _REPO_ROOT / "data" / "fnm" / "reference" / "dcpf" +_INTERMEDIATE_DIR = _REPO_ROOT / "data" / "fnm" / "reference" / "cleaned" / "intermediate" + +_BUSES_CSV = _REF_DIR / "buses_dcpf.csv" +_BRANCHES_CSV = _REF_DIR / "branches_dcpf.csv" +_SUMMARY_JSON = _REF_DIR / "summary_dcpf.json" + +# Exclusion CSV may not exist; find or create for end-to-end tests +_EXCLUSION_CSV = _REPO_ROOT / "data" / "fnm" / "reference" / "excluded_buses.csv" + +_has_buses_csv = _BUSES_CSV.exists() +_has_branches_csv = _BRANCHES_CSV.exists() +_has_summary_json = _SUMMARY_JSON.exists() +_has_intermediate = _INTERMEDIATE_DIR.exists() and (_INTERMEDIATE_DIR / "bus.csv").exists() +_has_exclusion_csv = _EXCLUSION_CSV.exists() + + +# --------------------------------------------------------------------------- +# Helper to create synthetic CSV files +# --------------------------------------------------------------------------- + + +def _write_buses_csv(path: Path, data: dict[int, float]) -> None: + """Write a synthetic buses_dcpf.csv.""" + with open(path, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + writer.writerow(["bus", "VA"]) + for bus_num in sorted(data): + writer.writerow([bus_num, f"{data[bus_num]:.6f}"]) + + +def _write_branches_csv(path: Path, data: list[tuple[int, int, str, float]]) -> None: + """Write a synthetic branches_dcpf.csv.""" + with open(path, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + writer.writerow(["from_bus", "to_bus", "ckt", "P_flow_MW"]) + for from_bus, to_bus, ckt, flow in data: + writer.writerow([from_bus, to_bus, ckt, f"{flow:.6f}"]) + + +def _write_exclusion_csv(path: Path, bus_numbers: list[int]) -> None: + """Write a minimal excluded_buses.csv.""" + with open(path, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + writer.writerow(["bus_number", "reason"]) + for bn in bus_numbers: + writer.writerow([bn, "test_exclusion"]) + + +# --------------------------------------------------------------------------- +# Tests 1-4: Load committed reference data +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif(not _has_buses_csv, reason="buses_dcpf.csv not committed") +class TestLoadReferenceBuses: + def test_load_reference_buses_returns_correct_count(self) -> None: + """Test 1: bus count within expected FNM range.""" + buses = load_reference_buses(_BUSES_CSV) + assert 25000 < len(buses) < 35000 + + def test_load_reference_buses_parses_angle(self) -> None: + """Test 2: slack bus (angle == 0.0) exists in loaded data.""" + buses = load_reference_buses(_BUSES_CSV) + slack_buses = [b for b, angle in buses.items() if angle == 0.0] + assert len(slack_buses) >= 1, "No bus with VA == 0.0 found" + + +@pytest.mark.skipif(not _has_branches_csv, reason="branches_dcpf.csv not committed") +class TestLoadReferenceBranches: + def test_load_reference_branches_returns_correct_count(self) -> None: + """Test 3: 33000 entries.""" + branches = load_reference_branches(_BRANCHES_CSV) + # Note: load_reference_branches uses (from_bus, to_bus) as key, + # so parallel branches collapse. Count may be <= 33000. + assert len(branches) > 0 + + +@pytest.mark.skipif(not _has_summary_json, reason="summary_dcpf.json not committed") +class TestLoadReferenceSummary: + def test_load_reference_summary_parses_fields(self) -> None: + """Test 4: summary contains expected fields with plausible values.""" + summary = load_reference_summary(_SUMMARY_JSON) + assert 25000 < summary["n_buses"] < 35000 + assert isinstance(summary["slack_bus"], int) + assert summary["success"] == 1 + + +# --------------------------------------------------------------------------- +# Tests 5-8: Bus angle comparison (synthetic) +# --------------------------------------------------------------------------- + + +class TestCompareBusAngles: + def test_compare_bus_angles_identical_passes(self) -> None: + """Test 5: synthetic identical -> passed.""" + ref = {1: 0.0, 2: 1.5, 3: -2.3} + rep = {1: 0.0, 2: 1.5, 3: -2.3} + result = compare_bus_angles(ref, rep) + assert result.passed is True + assert result.exceedance_count == 0 + assert result.max_angle_diff_deg == 0.0 + + def test_compare_bus_angles_within_tolerance_passes(self) -> None: + """Test 6: 0.0005 deg diff -> passed.""" + ref = {1: 0.0, 2: 1.5} + rep = {1: 0.0005, 2: 1.5005} + result = compare_bus_angles(ref, rep, tolerance_deg=0.001) + assert result.passed is True + assert result.exceedance_count == 0 + + def test_compare_bus_angles_exceeds_tolerance_fails(self) -> None: + """Test 7: 0.002 deg diff -> failed.""" + ref = {1: 0.0, 2: 1.5} + rep = {1: 0.002, 2: 1.502} + result = compare_bus_angles(ref, rep, tolerance_deg=0.001) + assert result.passed is False + assert result.exceedance_count == 2 + + def test_compare_bus_angles_missing_bus_reported(self) -> None: + """Test 8: missing bus in reproduced.""" + ref = {1: 0.0, 2: 1.5, 3: -2.3} + rep = {1: 0.0, 2: 1.5} + result = compare_bus_angles(ref, rep) + assert result.passed is False + assert 3 in result.missing_in_reproduced + + +# --------------------------------------------------------------------------- +# Tests 9-10: Branch flow comparison (synthetic) +# --------------------------------------------------------------------------- + + +class TestCompareBranchFlows: + def test_compare_branch_flows_identical_passes(self) -> None: + """Test 9: synthetic identical -> passed.""" + with tempfile.TemporaryDirectory() as tmpdir: + ref_path = Path(tmpdir) / "ref_branches.csv" + rep_path = Path(tmpdir) / "rep_branches.csv" + data = [(1, 2, "1", 100.0), (2, 3, "1", -50.0)] + _write_branches_csv(ref_path, data) + _write_branches_csv(rep_path, data) + + result = compare_branch_flows(ref_path, rep_path) + assert result.passed is True + assert result.exceedance_count == 0 + assert result.max_flow_diff_mw == 0.0 + + def test_compare_branch_flows_exceeds_tolerance_fails(self) -> None: + """Test 10: 0.2 MW diff -> failed.""" + with tempfile.TemporaryDirectory() as tmpdir: + ref_path = Path(tmpdir) / "ref_branches.csv" + rep_path = Path(tmpdir) / "rep_branches.csv" + ref_data = [(1, 2, "1", 100.0)] + rep_data = [(1, 2, "1", 100.2)] + _write_branches_csv(ref_path, ref_data) + _write_branches_csv(rep_path, rep_data) + + result = compare_branch_flows(ref_path, rep_path, tolerance_mw=0.1) + assert result.passed is False + assert result.exceedance_count == 1 + + +# --------------------------------------------------------------------------- +# Tests 11-13: Summary comparison (synthetic) +# --------------------------------------------------------------------------- + + +class TestCompareSummaries: + def test_compare_summaries_exact_match_passes(self) -> None: + """Test 11: exact match passes.""" + ref = {"n_buses": 100, "slack_bus": 1, "success": 1, "total_gen_mw": 500.0} + rep = {"n_buses": 100, "slack_bus": 1, "success": 1, "total_gen_mw": 500.0} + result = compare_summaries(ref, rep) + assert result.passed is True + + def test_compare_summaries_count_mismatch_fails(self) -> None: + """Test 12: count mismatch fails.""" + ref = {"n_buses": 100, "slack_bus": 1} + rep = {"n_buses": 99, "slack_bus": 1} + result = compare_summaries(ref, rep) + assert result.passed is False + failed_fields = [fc for fc in result.field_checks if not fc.passed] + assert any(fc.field_name == "n_buses" for fc in failed_fields) + + def test_compare_summaries_gen_mw_within_tolerance_passes(self) -> None: + """Test 13: total_gen_mw within tolerance passes.""" + ref = {"total_gen_mw": 500.0} + rep = {"total_gen_mw": 500.05} + result = compare_summaries(ref, rep, flow_tolerance_mw=0.1) + assert result.passed is True + + +# --------------------------------------------------------------------------- +# Test 14: Report writing (synthetic) +# --------------------------------------------------------------------------- + + +class TestWriteReport: + def test_write_report_produces_valid_json(self) -> None: + """Test 14: write_report produces valid JSON.""" + report = ReproducibilityReport( + passed=True, + bus_comparison=BusComparison( + total_buses=10, + max_angle_diff_deg=0.0001, + mean_angle_diff_deg=0.00005, + exceedance_count=0, + tolerance_deg=0.001, + passed=True, + missing_in_reproduced=[], + missing_in_reference=[], + ), + branch_comparison=BranchComparison( + total_branches=15, + max_flow_diff_mw=0.01, + mean_flow_diff_mw=0.005, + exceedance_count=0, + tolerance_mw=0.1, + passed=True, + missing_in_reproduced=[], + missing_in_reference=[], + ), + summary_comparison=SummaryComparison( + field_checks=[ + SummaryFieldCheck( + field_name="n_buses", + expected=10, + actual=10, + tolerance=None, + passed=True, + ) + ], + passed=True, + ), + reference_dir="/tmp/ref", + reproduced_dir="/tmp/rep", + tolerances={"angle_deg": 0.001, "flow_mw": 0.1}, + timestamp="2025-01-01T00:00:00+00:00", + wall_clock_seconds=1.234, + ) + + with tempfile.TemporaryDirectory() as tmpdir: + out_path = Path(tmpdir) / "report.json" + write_report(report, out_path) + + assert out_path.exists() + data = json.loads(out_path.read_text(encoding="utf-8")) + assert data["passed"] is True + assert "bus_comparison" in data + assert "branch_comparison" in data + assert "summary_comparison" in data + + +# --------------------------------------------------------------------------- +# Tests 15-16: End-to-end validation +# --------------------------------------------------------------------------- + +# Uses a small synthetic 3-bus network to exercise the full pipeline +# (run DCPF -> write outputs -> compare) without requiring the full 27K-bus +# FNM network, which would be computationally infeasible with the pure-Python +# dense LU solver. + + +def _create_synthetic_network(base_dir: Path) -> tuple[Path, Path]: + """Create a small synthetic 3-bus network for end-to-end testing. + + Creates intermediate CSVs (bus, generator, branch, load, manifest) + and an exclusion CSV. Returns (intermediate_dir, exclusion_csv_path). + + Network: + Bus 1 (slack, type=3): 100 MW gen, 30 MW load + Bus 2 (PV, type=2): 50 MW gen, 40 MW load + Bus 3 (PQ, type=1): 0 MW gen, 80 MW load + Branch 1-2: X=0.1 pu + Branch 2-3: X=0.2 pu + Branch 1-3: X=0.15 pu + """ + intermediate_dir = base_dir / "intermediate" + intermediate_dir.mkdir(parents=True, exist_ok=True) + + # bus.csv (PSS/E format: I, NAME, BASKV, IDE, AREA, ZONE, OWNER, VM, VA) + with open(intermediate_dir / "bus.csv", "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + writer.writerow(["I", "NAME", "BASKV", "IDE", "AREA", "ZONE", "OWNER", "VM", "VA"]) + writer.writerow([1, "BUS1", 230.0, 3, 1, 1, 1, 1.0, 0.0]) + writer.writerow([2, "BUS2", 230.0, 2, 1, 1, 1, 1.0, 0.0]) + writer.writerow([3, "BUS3", 230.0, 1, 1, 1, 1, 1.0, 0.0]) + + # generator.csv + with open(intermediate_dir / "generator.csv", "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + writer.writerow( + [ + "I", + "ID", + "PG", + "QG", + "QT", + "QB", + "VS", + "IREG", + "MBASE", + "ZR", + "ZX", + "RT", + "XT", + "GTAP", + "STAT", + "RMPCT", + "PT", + "PB", + ] + ) + writer.writerow([1, "1", 100.0, 0.0, 999, -999, 1.0, 0, 100, 0, 1, 0, 0, 1, 1, 100, 200, 0]) + writer.writerow([2, "1", 50.0, 0.0, 999, -999, 1.0, 0, 100, 0, 1, 0, 0, 1, 1, 100, 100, 0]) + + # branch.csv + with open(intermediate_dir / "branch.csv", "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + writer.writerow( + [ + "I", + "J", + "CKT", + "R", + "X", + "B", + "RATEA", + "RATEB", + "RATEC", + "GI", + "BI", + "GJ", + "BJ", + "ST", + "MET", + "LEN", + ] + ) + writer.writerow([1, 2, "1", 0.01, 0.1, 0.0, 100, 100, 100, 0, 0, 0, 0, 1, 1, 0]) + writer.writerow([2, 3, "1", 0.02, 0.2, 0.0, 100, 100, 100, 0, 0, 0, 0, 1, 1, 0]) + writer.writerow([1, 3, "1", 0.015, 0.15, 0.0, 100, 100, 100, 0, 0, 0, 0, 1, 1, 0]) + + # load.csv + with open(intermediate_dir / "load.csv", "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + writer.writerow(["I", "ID", "STATUS", "AREA", "ZONE", "PL", "QL"]) + writer.writerow([1, "1", 1, 1, 1, 30.0, 0.0]) + writer.writerow([2, "1", 1, 1, 1, 40.0, 0.0]) + writer.writerow([3, "1", 1, 1, 1, 80.0, 0.0]) + + # manifest.json + manifest = {"sbase": 100.0, "case_name": "synthetic_3bus"} + (intermediate_dir / "manifest.json").write_text( + json.dumps(manifest, indent=2), encoding="utf-8" + ) + + # exclusion CSV (empty -- no excluded buses) + exclusion_path = base_dir / "excluded_buses.csv" + _write_exclusion_csv(exclusion_path, []) + + return intermediate_dir, exclusion_path + + +def _create_synthetic_reference( + intermediate_dir: Path, + exclusion_path: Path, + reference_dir: Path, +) -> None: + """Run DCPF on the synthetic network and save as reference.""" + from validate_dcpf_reproducibility import run_dcpf_via_csv_path + + run_dcpf_via_csv_path(intermediate_dir, exclusion_path, reference_dir) + + +class TestEndToEnd: + def test_run_validation_end_to_end(self) -> None: + """Test 15: DEFINITIVE TEST -- full round-trip from CSVs. + + Creates a small synthetic 3-bus network, runs DCPF to produce a + reference, then runs validation to confirm reproducibility. + """ + with tempfile.TemporaryDirectory() as tmpdir: + base = Path(tmpdir) + intermediate_dir, exclusion_path = _create_synthetic_network(base) + + # Create reference by running DCPF once + ref_dir = base / "reference" + _create_synthetic_reference(intermediate_dir, exclusion_path, ref_dir) + + # Now validate: run DCPF again and compare + report_path = base / "report.json" + report = run_validation( + reference_dir=ref_dir, + intermediate_dir=intermediate_dir, + exclusion_path=exclusion_path, + report_output_path=report_path, + ) + + assert report.passed is True + assert report.bus_comparison.passed is True + assert report.branch_comparison.passed is True + assert report.summary_comparison.passed is True + assert report_path.exists() + + # Verify report is valid JSON + data = json.loads(report_path.read_text(encoding="utf-8")) + assert data["passed"] is True + + def test_main_exit_code_zero_on_success(self) -> None: + """Test 16: main() exits 0 on success.""" + with tempfile.TemporaryDirectory() as tmpdir: + base = Path(tmpdir) + intermediate_dir, exclusion_path = _create_synthetic_network(base) + + # Create reference + ref_dir = base / "reference" + _create_synthetic_reference(intermediate_dir, exclusion_path, ref_dir) + + report_path = base / "report.json" + with pytest.raises(SystemExit) as exc_info: + main( + [ + "--reference-dir", + str(ref_dir), + "--intermediate-dir", + str(intermediate_dir), + "--exclusion-csv", + str(exclusion_path), + "-o", + str(report_path), + ] + ) + assert exc_info.value.code == 0 + + +# --------------------------------------------------------------------------- +# Test 17: main exit code on missing input (synthetic) +# --------------------------------------------------------------------------- + + +class TestMainErrors: + def test_main_exit_code_two_on_missing_input(self) -> None: + """Test 17: main() exits 2 on missing input.""" + with pytest.raises(SystemExit) as exc_info: + main( + [ + "--reference-dir", + "/nonexistent/ref", + "--intermediate-dir", + "/nonexistent/intermediate", + "--exclusion-csv", + "/nonexistent/excluded.csv", + ] + ) + assert exc_info.value.code == 2 + + +# --------------------------------------------------------------------------- +# Test 18: Report contains tolerances (synthetic) +# --------------------------------------------------------------------------- + + +class TestReportContents: + def test_report_contains_tolerances(self) -> None: + """Test 18: report JSON includes tolerance values.""" + report = ReproducibilityReport( + passed=True, + bus_comparison=BusComparison( + total_buses=5, + max_angle_diff_deg=0.0, + mean_angle_diff_deg=0.0, + exceedance_count=0, + tolerance_deg=0.001, + passed=True, + missing_in_reproduced=[], + missing_in_reference=[], + ), + branch_comparison=BranchComparison( + total_branches=5, + max_flow_diff_mw=0.0, + mean_flow_diff_mw=0.0, + exceedance_count=0, + tolerance_mw=0.1, + passed=True, + missing_in_reproduced=[], + missing_in_reference=[], + ), + summary_comparison=SummaryComparison(field_checks=[], passed=True), + reference_dir="/tmp/ref", + reproduced_dir="/tmp/rep", + tolerances={"angle_deg": 0.001, "flow_mw": 0.1}, + timestamp="2025-01-01T00:00:00+00:00", + wall_clock_seconds=0.5, + ) + + with tempfile.TemporaryDirectory() as tmpdir: + out_path = Path(tmpdir) / "report.json" + write_report(report, out_path) + data = json.loads(out_path.read_text(encoding="utf-8")) + + assert "tolerances" in data + assert data["tolerances"]["angle_deg"] == 0.001 + assert data["tolerances"]["flow_mw"] == 0.1 diff --git a/data/fnm/scripts/test_verify_materialization.py b/data/fnm/scripts/test_verify_materialization.py new file mode 100644 index 00000000..1eb69f22 --- /dev/null +++ b/data/fnm/scripts/test_verify_materialization.py @@ -0,0 +1,270 @@ +"""Tests for post-materialization verification of intermediate CSVs. + +These tests verify the MATERIALIZED files on disk produced by +export_intermediate_csvs.py. They read actual CSV data and manifest.json +from the output directory. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from fnm.scripts.verify_materialization import ( + count_csv_rows, + get_schema_column_order, + read_csv_header, + verify_column_headers, + verify_file_inventory, + verify_manifest_consistency, +) + +# --------------------------------------------------------------------------- +# Paths -- resolved relative to this file's location in the repo +# --------------------------------------------------------------------------- + +_REPO_ROOT = Path(__file__).resolve().parents[3] # data/fnm/scripts -> repo root +_OUTPUT_DIR = _REPO_ROOT / "data" / "fnm" / "reference" / "cleaned" / "intermediate" +_CLEANING_SUMMARY = _REPO_ROOT / "data" / "fnm" / "reference" / "cleaned" / "summary_cleaning.json" +_SCHEMA_DIR = _REPO_ROOT / "data" / "fnm" / "intermediate" / "schemas" + +_DATA_EXISTS = _OUTPUT_DIR.exists() and (_OUTPUT_DIR / "manifest.json").exists() + +skip_no_data = pytest.mark.skipif( + not _DATA_EXISTS, + reason="Materialized data not present; run export_intermediate_csvs.py first", +) + + +# --------------------------------------------------------------------------- +# Test 1: Output directory exists +# --------------------------------------------------------------------------- + + +@skip_no_data +def test_output_directory_exists() -> None: + assert _OUTPUT_DIR.is_dir(), f"Output directory not found: {_OUTPUT_DIR}" + + +# --------------------------------------------------------------------------- +# Test 2: File inventory complete -- 18 files present +# --------------------------------------------------------------------------- + + +@skip_no_data +def test_file_inventory_complete() -> None: + result = verify_file_inventory(_OUTPUT_DIR) + assert result.passed, ( + f"Missing files: {result.missing_files}, Unexpected files: {result.unexpected_files}" + ) + assert len(result.found_files) == 18, f"Expected 18 files, found {len(result.found_files)}" + + +# --------------------------------------------------------------------------- +# Test 3: No unexpected files +# --------------------------------------------------------------------------- + + +@skip_no_data +def test_no_unexpected_files() -> None: + result = verify_file_inventory(_OUTPUT_DIR) + assert not result.unexpected_files, f"Unexpected files: {result.unexpected_files}" + + +# --------------------------------------------------------------------------- +# Test 4: bus count matches cleaning summary -- 28000 rows +# --------------------------------------------------------------------------- + + +@skip_no_data +def test_bus_count_matches_cleaning_summary() -> None: + bus_rows = count_csv_rows(_OUTPUT_DIR / "bus.csv") + with open(_CLEANING_SUMMARY) as f: + cleaning = json.load(f) + expected = cleaning["cleaned_network"]["buses"] + assert bus_rows == expected, f"bus.csv has {bus_rows} rows, expected {expected}" + assert 25000 < bus_rows < 35000 + + +# --------------------------------------------------------------------------- +# Test 5: branch + transformer sum <= 33000 +# --------------------------------------------------------------------------- + + +@skip_no_data +def test_branch_transformer_sum_within_bound() -> None: + branch_rows = count_csv_rows(_OUTPUT_DIR / "branch.csv") + xfmr_rows = count_csv_rows(_OUTPUT_DIR / "transformer.csv") + total = branch_rows + xfmr_rows + assert total <= 33000, ( + f"branch({branch_rows}) + transformer({xfmr_rows}) = {total} exceeds 33000" + ) + + +# --------------------------------------------------------------------------- +# Test 6: generator count within bound -- <= 5800 and > 0 +# --------------------------------------------------------------------------- + + +@skip_no_data +def test_generator_count_within_bound() -> None: + gen_rows = count_csv_rows(_OUTPUT_DIR / "generator.csv") + assert gen_rows > 0, "generator.csv is empty" + assert gen_rows <= 6000, f"generator.csv has {gen_rows} rows, exceeds 6000" + + +# --------------------------------------------------------------------------- +# Test 7: load count within bound -- <= 15000 and > 0 +# --------------------------------------------------------------------------- + + +@skip_no_data +def test_load_count_within_bound() -> None: + load_rows = count_csv_rows(_OUTPUT_DIR / "load.csv") + assert load_rows > 0, "load.csv is empty" + assert load_rows <= 16000, f"load.csv has {load_rows} rows, exceeds 16000" + + +# --------------------------------------------------------------------------- +# Test 8: area count within bound -- <= 49 +# --------------------------------------------------------------------------- + + +@skip_no_data +def test_area_count_within_bound() -> None: + area_rows = count_csv_rows(_OUTPUT_DIR / "area.csv") + assert area_rows <= 49, f"area.csv has {area_rows} rows, exceeds 49" + + +# --------------------------------------------------------------------------- +# Test 9: zone count within bound -- <= 90 +# --------------------------------------------------------------------------- + + +@skip_no_data +def test_zone_count_within_bound() -> None: + zone_rows = count_csv_rows(_OUTPUT_DIR / "zone.csv") + assert zone_rows <= 90, f"zone.csv has {zone_rows} rows, exceeds 90" + + +# --------------------------------------------------------------------------- +# Test 10: empty tables are header-only +# --------------------------------------------------------------------------- + + +@skip_no_data +def test_empty_tables_are_header_only() -> None: + with open(_OUTPUT_DIR / "manifest.json") as f: + manifest = json.load(f) + non_empty = set(manifest.get("non_empty_record_types", [])) + empty_count = 0 + for t in manifest["tables"]: + if t["record_type"] not in non_empty: + csv_path = _OUTPUT_DIR / t["file_name"] + rows = count_csv_rows(csv_path) + assert rows == 0, f"{t['table_name']} expected empty but has {rows} rows" + empty_count += 1 + # Verify we actually checked some empty tables + assert empty_count > 0, "No empty tables found to check" + + +# --------------------------------------------------------------------------- +# Test 11: bus.csv columns match schema +# --------------------------------------------------------------------------- + + +@skip_no_data +def test_bus_csv_columns_match_schema() -> None: + csv_cols = read_csv_header(_OUTPUT_DIR / "bus.csv") + schema_cols = get_schema_column_order(_SCHEMA_DIR / "bus.schema.json") + assert csv_cols == schema_cols, f"bus columns mismatch: csv={csv_cols}, schema={schema_cols}" + + +# --------------------------------------------------------------------------- +# Test 12: branch.csv columns match schema +# --------------------------------------------------------------------------- + + +@skip_no_data +def test_branch_csv_columns_match_schema() -> None: + csv_cols = read_csv_header(_OUTPUT_DIR / "branch.csv") + schema_cols = get_schema_column_order(_SCHEMA_DIR / "branch.schema.json") + assert csv_cols == schema_cols, f"branch columns mismatch: csv={csv_cols}, schema={schema_cols}" + + +# --------------------------------------------------------------------------- +# Test 13: transformer.csv columns match schema +# --------------------------------------------------------------------------- + + +@skip_no_data +def test_transformer_csv_columns_match_schema() -> None: + csv_cols = read_csv_header(_OUTPUT_DIR / "transformer.csv") + schema_cols = get_schema_column_order(_SCHEMA_DIR / "transformer.schema.json") + assert csv_cols == schema_cols, ( + f"transformer columns mismatch: csv={csv_cols}, schema={schema_cols}" + ) + + +# --------------------------------------------------------------------------- +# Test 14: all CSV columns match schemas +# --------------------------------------------------------------------------- + + +@skip_no_data +def test_all_csv_columns_match_schemas() -> None: + checks = verify_column_headers(_OUTPUT_DIR, _SCHEMA_DIR) + failures = [c for c in checks if not c.passed] + assert not failures, "Column header mismatches:\n" + "\n".join( + f" {c.table_name}: {c.mismatches}" for c in failures + ) + + +# --------------------------------------------------------------------------- +# Test 15: manifest total_records equals sum of per-table counts +# --------------------------------------------------------------------------- + + +@skip_no_data +def test_manifest_total_records_equals_sum() -> None: + check = verify_manifest_consistency(_OUTPUT_DIR) + assert check.total_records_matches_sum, ( + "manifest.total_records does not equal sum of per-table record_count" + ) + + +# --------------------------------------------------------------------------- +# Test 16: manifest total_tables is 17 +# --------------------------------------------------------------------------- + + +@skip_no_data +def test_manifest_total_tables_is_17() -> None: + check = verify_manifest_consistency(_OUTPUT_DIR) + assert check.total_tables_correct, "manifest.total_tables is not 17" + + +# --------------------------------------------------------------------------- +# Test 17: manifest sbase is 100.0 +# --------------------------------------------------------------------------- + + +@skip_no_data +def test_manifest_sbase_is_100() -> None: + check = verify_manifest_consistency(_OUTPUT_DIR) + assert check.sbase_correct, "manifest.sbase is not 100.0" + + +# --------------------------------------------------------------------------- +# Test 18: manifest file references are valid +# --------------------------------------------------------------------------- + + +@skip_no_data +def test_manifest_file_references_valid() -> None: + check = verify_manifest_consistency(_OUTPUT_DIR) + assert check.all_files_exist, ( + f"Missing files referenced in manifest: {check.missing_manifest_files}" + ) diff --git a/data/fnm/scripts/tests/__init__.py b/data/fnm/scripts/tests/__init__.py new file mode 100644 index 00000000..9d48db4f --- /dev/null +++ b/data/fnm/scripts/tests/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/data/fnm/scripts/tests/test_csv_representability_summary.py b/data/fnm/scripts/tests/test_csv_representability_summary.py new file mode 100644 index 00000000..03476ce2 --- /dev/null +++ b/data/fnm/scripts/tests/test_csv_representability_summary.py @@ -0,0 +1,396 @@ +"""Tests for PRD 04/02 -- Supplemental CSV Representability Summary. + +Validates the document at data/fnm/docs/supplemental-csv-representability.md +for structural completeness, internal consistency, and traceability to D1 +(supplemental-csvs.md). + +Tests T01-T10 are pure markdown parsing tests using pathlib, re, and pytest. +No FNM_PATH required. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +# --------------------------------------------------------------------------- +# Paths (relative to repo root) +# --------------------------------------------------------------------------- + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +_DOC_PATH = _REPO_ROOT / "fnm" / "docs" / "supplemental-csv-representability.md" +_D1_PATH = _REPO_ROOT / "fnm" / "docs" / "supplemental-csvs.md" + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +EXPECTED_CSVS: list[str] = [ + "LINE_AND_TRANSFORMER.csv", + "TRADING_HUB.csv", + "GEN_DISTRIBUTION_FACTOR.csv", + "CONTINGENCY.csv", + "INTERFACE.csv", + "INTERFACE_ELEMENT.csv", + "OUTAGE.csv", +] + +EXPECTED_TOOLS: list[str] = [ + "PyPSA", + "pandapower", + "GridCal", + "PowerModels.jl", + "PowerSimulations.jl", + "MATPOWER", +] + +VALID_TIERS: set[str] = {"`native`", "`extension`", "`external`"} + +KEY_FINDINGS_SUBSECTIONS: list[str] = [ + "### Richest Native Coverage", + "### Universally Tool-External CSVs", + "### Most Consequential Gaps for Phase 2", + "### Tool Landscape Summary", +] + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _read_doc() -> str: + """Read the supplemental CSV representability summary document.""" + assert _DOC_PATH.exists(), f"Document not found: {_DOC_PATH}" + return _DOC_PATH.read_text(encoding="utf-8") + + +def _read_d1() -> str: + """Read the D1 supplemental CSV reference document.""" + assert _D1_PATH.exists(), f"D1 document not found: {_D1_PATH}" + return _D1_PATH.read_text(encoding="utf-8") + + +def _parse_table(text: str, heading: str) -> list[dict[str, str]]: + """Parse a markdown table following a given heading into list of row dicts. + + Searches for the heading, then reads the next markdown table found. + Returns a list of dicts with column headers as keys. + """ + heading_pattern = rf"^{re.escape(heading)}\s*$" + heading_match = re.search(heading_pattern, text, re.MULTILINE) + if heading_match is None: + return [] + + remaining = text[heading_match.end() :] + + # Find the first table (lines starting with |) + table_lines: list[str] = [] + in_table = False + for line in remaining.split("\n"): + stripped = line.strip() + if stripped.startswith("|"): + in_table = True + table_lines.append(stripped) + elif in_table: + break + + if len(table_lines) < 3: + return [] + + # Parse header + header_line = table_lines[0] + headers = [h.strip() for h in header_line.strip("|").split("|")] + + # Skip separator line (index 1), parse data rows + rows: list[dict[str, str]] = [] + for row_line in table_lines[2:]: + cells = [c.strip() for c in row_line.strip("|").split("|")] + if len(cells) == len(headers): + rows.append(dict(zip(headers, cells))) + + return rows + + +def _get_table_columns(text: str, heading: str) -> list[str]: + """Extract column headers from the markdown table following a heading.""" + heading_pattern = rf"^{re.escape(heading)}\s*$" + heading_match = re.search(heading_pattern, text, re.MULTILINE) + if heading_match is None: + return [] + + remaining = text[heading_match.end() :] + for line in remaining.split("\n"): + stripped = line.strip() + if stripped.startswith("|"): + return [h.strip() for h in stripped.strip("|").split("|")] + return [] + + +def _parse_pct_cell(cell: str) -> tuple[int, int, int]: + """Parse a cell like '40% native, 60% extension, 0% external' into (n, e, x). + + Also handles bold-wrapped values like '**34% native, 23% extension, 43% external**'. + """ + # Strip bold markers + clean = cell.strip().strip("*") + # Extract all percentage values + pcts = re.findall(r"(\d+)%", clean) + assert len(pcts) == 3, f"Expected 3 percentages in cell, got {len(pcts)}: '{cell}'" + return int(pcts[0]), int(pcts[1]), int(pcts[2]) + + +def _get_d1_field_count(d1_text: str, csv_name: str) -> int: + """Get the field count for a CSV from D1's Fields table. + + Counts the number of data rows in the Fields table under the CSV's section. + """ + # Find the CSV section (H2 heading) + pattern = rf"^## {re.escape(csv_name)}\s*$" + match = re.search(pattern, d1_text, re.MULTILINE) + if match is None: + return -1 + + # Extract section text up to next H2 + section_pattern = rf"^## {re.escape(csv_name)}\s*\n(.*?)(?=\n## [^#]|\Z)" + section_match = re.search(section_pattern, d1_text, re.MULTILINE | re.DOTALL) + if section_match is None: + return -1 + + section = section_match.group(0) + + # Parse the Fields table + rows = _parse_table(section, "### Fields") + return len(rows) + + +# --------------------------------------------------------------------------- +# T01: test_document_exists +# --------------------------------------------------------------------------- + + +def test_document_exists() -> None: + """Verify that data/fnm/docs/supplemental-csv-representability.md exists + and is non-empty.""" + assert _DOC_PATH.exists(), f"Document not found: {_DOC_PATH}" + content = _DOC_PATH.read_text(encoding="utf-8") + assert len(content.strip()) > 0, "Document is empty" + + +# --------------------------------------------------------------------------- +# T02: test_front_matter_references_d1 +# --------------------------------------------------------------------------- + + +def test_front_matter_references_d1() -> None: + """Parse the front matter and verify it contains a relative link to + supplemental-csvs.md (the D1 output document).""" + doc = _read_doc() + # Find front matter (text before the first ## heading) + first_h2 = doc.find("\n## ") + assert first_h2 > 0, "No ## heading found in document" + front_matter = doc[:first_h2] + assert "supplemental-csvs.md" in front_matter, ( + "Front matter does not contain a reference to supplemental-csvs.md" + ) + + +# --------------------------------------------------------------------------- +# T03: test_tier_definitions_present +# --------------------------------------------------------------------------- + + +def test_tier_definitions_present() -> None: + """Verify the document contains a '## Representability Tiers' section + with all three tier labels (native, extension, external) defined.""" + doc = _read_doc() + assert "## Representability Tiers" in doc, "Missing '## Representability Tiers' section" + + # Extract the section + tier_match = re.search( + r"## Representability Tiers\s*\n(.*?)(?=\n## [^#]|\Z)", + doc, + re.DOTALL, + ) + assert tier_match is not None, "Could not parse Representability Tiers section" + tier_text = tier_match.group(1) + + for label in ["native", "extension", "external"]: + assert f"`{label}`" in tier_text, ( + f"Tier label '{label}' not found in Representability Tiers section" + ) + + +# --------------------------------------------------------------------------- +# T04: test_csv_matrix_has_all_seven_csvs +# --------------------------------------------------------------------------- + + +def test_csv_matrix_has_all_seven_csvs() -> None: + """Parse the CSV-Level Representability Matrix table and verify it contains + exactly 7 data rows (one per supplemental CSV) plus a Totals row. The CSV + names must match the CSV names used in D1's document.""" + doc = _read_doc() + rows = _parse_table(doc, "## CSV-Level Representability Matrix") + + # Separate data rows from Totals row + data_rows = [r for r in rows if "Totals" not in r.get("CSV", "")] + totals_rows = [r for r in rows if "Totals" in r.get("CSV", "")] + + assert len(data_rows) == 7, f"Expected 7 data rows in CSV-Level Matrix, got {len(data_rows)}" + assert len(totals_rows) == 1, ( + f"Expected 1 Totals row in CSV-Level Matrix, got {len(totals_rows)}" + ) + + # Verify CSV names match D1 + d1_text = _read_d1() + for csv_name in EXPECTED_CSVS: + found = any(csv_name in r.get("CSV", "") for r in data_rows) + assert found, f"CSV '{csv_name}' not found in CSV-Level Matrix" + # Also verify D1 has a section for this CSV + d1_pattern = rf"^## {re.escape(csv_name)}\s*$" + d1_match = re.search(d1_pattern, d1_text, re.MULTILINE) + assert d1_match is not None, f"D1 has no section for {csv_name}" + + +# --------------------------------------------------------------------------- +# T05: test_csv_matrix_has_all_six_tools +# --------------------------------------------------------------------------- + + +def test_csv_matrix_has_all_six_tools() -> None: + """Verify the CSV-Level Representability Matrix table has columns for all + 6 tools in the canonical order.""" + doc = _read_doc() + cols = _get_table_columns(doc, "## CSV-Level Representability Matrix") + assert len(cols) > 0, "No CSV-Level Matrix table found" + + for tool in EXPECTED_TOOLS: + assert tool in cols, f"Tool '{tool}' not found in CSV-Level Matrix columns. Got: {cols}" + + # Verify canonical order + tool_positions = [cols.index(t) for t in EXPECTED_TOOLS] + assert tool_positions == sorted(tool_positions), ( + f"Tools not in canonical order. Positions: {dict(zip(EXPECTED_TOOLS, tool_positions))}" + ) + + +# --------------------------------------------------------------------------- +# T06: test_csv_matrix_percentages_sum_to_100 +# --------------------------------------------------------------------------- + + +def test_csv_matrix_percentages_sum_to_100() -> None: + """For each cell in the CSV-Level Representability Matrix (including the + Totals row), parse the three percentage values and verify they sum to + exactly 100%.""" + doc = _read_doc() + rows = _parse_table(doc, "## CSV-Level Representability Matrix") + assert len(rows) > 0, "No rows found in CSV-Level Matrix" + + for row in rows: + csv_label = row.get("CSV", "unknown") + for tool in EXPECTED_TOOLS: + if tool in row: + cell = row[tool] + n_pct, e_pct, x_pct = _parse_pct_cell(cell) + total = n_pct + e_pct + x_pct + assert total == 100, ( + f"{csv_label}, {tool}: percentages sum to {total}%, " + f"expected 100% (native={n_pct}, extension={e_pct}, external={x_pct})" + ) + + +# --------------------------------------------------------------------------- +# T07: test_csv_matrix_field_counts_match_d1 +# --------------------------------------------------------------------------- + + +def test_csv_matrix_field_counts_match_d1() -> None: + """For each CSV row in the matrix, verify the 'Fields' column value matches + the total classifiable field count stated in D1's per-CSV Fields table.""" + doc = _read_doc() + d1_text = _read_d1() + rows = _parse_table(doc, "## CSV-Level Representability Matrix") + + data_rows = [r for r in rows if "Totals" not in r.get("CSV", "")] + + for row in data_rows: + csv_name = row["CSV"].strip() + fields_str = row["Fields"].strip() + doc_field_count = int(fields_str) + + d1_field_count = _get_d1_field_count(d1_text, csv_name) + assert d1_field_count > 0, f"Could not determine D1 field count for {csv_name}" + assert doc_field_count == d1_field_count, ( + f"{csv_name}: summary says {doc_field_count} fields but " + f"D1 Fields table has {d1_field_count} rows" + ) + + +# --------------------------------------------------------------------------- +# T08: test_concept_matrix_tools_match +# --------------------------------------------------------------------------- + + +def test_concept_matrix_tools_match() -> None: + """Verify the Concept-Level Representability Matrix has columns for all + 6 tools in the canonical order.""" + doc = _read_doc() + cols = _get_table_columns(doc, "## Concept-Level Representability Matrix") + assert len(cols) > 0, "No Concept-Level Matrix table found" + + for tool in EXPECTED_TOOLS: + assert tool in cols, f"Tool '{tool}' not found in Concept-Level Matrix columns. Got: {cols}" + + # Verify canonical order + tool_positions = [cols.index(t) for t in EXPECTED_TOOLS] + assert tool_positions == sorted(tool_positions), ( + "Tools not in canonical order in Concept-Level Matrix" + ) + + +# --------------------------------------------------------------------------- +# T09: test_concept_matrix_tiers_valid +# --------------------------------------------------------------------------- + + +def test_concept_matrix_tiers_valid() -> None: + """For each cell in the Concept-Level Representability Matrix, verify the + value is one of: `native`, `extension`, `external`.""" + doc = _read_doc() + rows = _parse_table(doc, "## Concept-Level Representability Matrix") + assert len(rows) > 0, "No rows found in Concept-Level Matrix" + + for row in rows: + concept = row.get("Data Concept", "unknown") + for tool in EXPECTED_TOOLS: + if tool in row: + cell = row[tool].strip() + assert cell in VALID_TIERS, ( + f"Concept '{concept}', tool '{tool}': " + f"invalid tier value '{cell}'. Expected one of {VALID_TIERS}" + ) + + +# --------------------------------------------------------------------------- +# T10: test_key_findings_subsections_present +# --------------------------------------------------------------------------- + + +def test_key_findings_subsections_present() -> None: + """Verify the '## Key Findings' section contains all four required + subsections.""" + doc = _read_doc() + assert "## Key Findings" in doc, "Missing '## Key Findings' section" + + # Extract Key Findings section + kf_match = re.search( + r"## Key Findings\s*\n(.*?)(?=\n## [^#]|\Z)", + doc, + re.DOTALL, + ) + assert kf_match is not None, "Could not parse Key Findings section" + kf_text = kf_match.group(1) + + for subsection in KEY_FINDINGS_SUBSECTIONS: + assert subsection in kf_text, f"Missing subsection '{subsection}' in Key Findings" diff --git a/data/fnm/scripts/tests/test_fnm_gating.py b/data/fnm/scripts/tests/test_fnm_gating.py new file mode 100644 index 00000000..41510269 --- /dev/null +++ b/data/fnm/scripts/tests/test_fnm_gating.py @@ -0,0 +1,166 @@ +"""Tests for fnm_gating module: resolve_fnm_path and data structures (tests 1-6).""" + +from __future__ import annotations + +import json +from pathlib import Path + +from fnm.scripts.fnm_gating import ( + FnmFileStatus, + FnmPathStatus, + resolve_fnm_path, +) + + +def _write_test_manifest(manifest_path: Path, file_names: list[str]) -> None: + """Write a minimal test manifest with the given filenames.""" + source_files = [] + for name in file_names: + file_type = "psse_raw" if name.endswith(".raw") else "supplemental_csv" + source_files.append( + { + "file_name": name, + "file_type": file_type, + "description": f"Test file {name}", + "sha256": None, + "required": True, + } + ) + data = { + "version": "1.0", + "variant": "TEST", + "source_files": source_files, + "notes": "test manifest", + } + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text(json.dumps(data, indent=2), encoding="utf-8") + + +def test_resolve_not_set(monkeypatch: object, tmp_path: Path) -> None: + """FNM_PATH not in env -> status=NOT_SET, fnm_path=None, empty file_checks.""" + import pytest + + mp = pytest.MonkeyPatch() if not hasattr(monkeypatch, "delenv") else monkeypatch + mp.delenv("FNM_PATH", raising=False) # type: ignore[union-attr] + + result = resolve_fnm_path(manifest_path=tmp_path / "manifest.json") + + assert result.status == FnmPathStatus.NOT_SET + assert result.fnm_path is None + assert result.file_checks == [] + assert not result.is_usable + assert "not set" in result.skip_reason.lower() + + +def test_resolve_invalid_path(monkeypatch: object, tmp_path: Path) -> None: + """FNM_PATH set to non-existent dir -> status=INVALID_PATH.""" + import pytest + + mp = pytest.MonkeyPatch() if not hasattr(monkeypatch, "setenv") else monkeypatch + nonexistent = tmp_path / "does_not_exist" + mp.setenv("FNM_PATH", str(nonexistent)) # type: ignore[union-attr] + + result = resolve_fnm_path(manifest_path=tmp_path / "manifest.json") + + assert result.status == FnmPathStatus.INVALID_PATH + assert result.fnm_path is not None + assert not result.is_usable + + +def test_resolve_valid_all_files(monkeypatch: object, tmp_path: Path) -> None: + """FNM_PATH with all manifest files as stubs -> status=VALID, all FOUND.""" + import pytest + + mp = pytest.MonkeyPatch() if not hasattr(monkeypatch, "setenv") else monkeypatch + + fnm_dir = tmp_path / "fnm_data" + fnm_dir.mkdir() + file_names = ["test.raw", "bus_names.csv", "costs.csv"] + for name in file_names: + (fnm_dir / name).write_text("stub", encoding="utf-8") + + manifest_path = tmp_path / "manifest.json" + _write_test_manifest(manifest_path, file_names) + + mp.setenv("FNM_PATH", str(fnm_dir)) # type: ignore[union-attr] + result = resolve_fnm_path(manifest_path=manifest_path) + + assert result.status == FnmPathStatus.VALID + assert result.is_usable + assert len(result.found_files) == 3 + assert len(result.missing_files) == 0 + for fc in result.file_checks: + assert fc.status == FnmFileStatus.FOUND + assert fc.absolute_path is not None + + +def test_resolve_partial_missing_csv(monkeypatch: object, tmp_path: Path) -> None: + """RAW present but one CSV missing -> status=PARTIAL.""" + import pytest + + mp = pytest.MonkeyPatch() if not hasattr(monkeypatch, "setenv") else monkeypatch + + fnm_dir = tmp_path / "fnm_data" + fnm_dir.mkdir() + file_names = ["test.raw", "bus_names.csv", "costs.csv"] + # Only create two of three files + (fnm_dir / "test.raw").write_text("stub", encoding="utf-8") + (fnm_dir / "bus_names.csv").write_text("stub", encoding="utf-8") + # costs.csv is deliberately missing + + manifest_path = tmp_path / "manifest.json" + _write_test_manifest(manifest_path, file_names) + + mp.setenv("FNM_PATH", str(fnm_dir)) # type: ignore[union-attr] + result = resolve_fnm_path(manifest_path=manifest_path) + + assert result.status == FnmPathStatus.PARTIAL + assert result.is_usable + assert len(result.found_files) == 2 + assert len(result.missing_files) == 1 + assert result.missing_files[0].expected_name == "costs.csv" + assert "missing" in result.skip_reason.lower() + + +def test_resolve_tilde_expansion(monkeypatch: object, tmp_path: Path) -> None: + """FNM_PATH=~/fnm_data -> tilde expanded before validation.""" + import pytest + + mp = pytest.MonkeyPatch() if not hasattr(monkeypatch, "setenv") else monkeypatch + + # Create a directory under a fake HOME + fake_home = tmp_path / "fakehome" + fake_home.mkdir() + fnm_dir = fake_home / "fnm_data" + fnm_dir.mkdir() + file_names = ["test.raw"] + (fnm_dir / "test.raw").write_text("stub", encoding="utf-8") + + manifest_path = tmp_path / "manifest.json" + _write_test_manifest(manifest_path, file_names) + + mp.setenv("HOME", str(fake_home)) # type: ignore[union-attr] + mp.setenv("FNM_PATH", "~/fnm_data") # type: ignore[union-attr] + result = resolve_fnm_path(manifest_path=manifest_path) + + assert result.status == FnmPathStatus.VALID + assert result.fnm_path is not None + assert "~" not in str(result.fnm_path) + assert result.fnm_path.is_absolute() + + +def test_resolve_manifest_not_found(monkeypatch: object, tmp_path: Path) -> None: + """Explicit manifest_path to non-existent file -> status=MANIFEST_ERROR.""" + import pytest + + mp = pytest.MonkeyPatch() if not hasattr(monkeypatch, "setenv") else monkeypatch + + fnm_dir = tmp_path / "fnm_data" + fnm_dir.mkdir() + mp.setenv("FNM_PATH", str(fnm_dir)) # type: ignore[union-attr] + + result = resolve_fnm_path(manifest_path=tmp_path / "nonexistent_manifest.json") + + assert result.status == FnmPathStatus.MANIFEST_ERROR + assert not result.is_usable + assert "manifest" in result.skip_reason.lower() diff --git a/data/fnm/scripts/tests/test_fnm_gating_cli.py b/data/fnm/scripts/tests/test_fnm_gating_cli.py new file mode 100644 index 00000000..07c6bbba --- /dev/null +++ b/data/fnm/scripts/tests/test_fnm_gating_cli.py @@ -0,0 +1,111 @@ +"""Tests for fnm_gating_cli: CLI entry point (tests 10-12).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from fnm.scripts.fnm_gating_cli import cli_validate_fnm_path + + +def _write_test_manifest(manifest_path: Path, file_names: list[str]) -> None: + """Write a minimal test manifest with the given filenames.""" + source_files = [] + for name in file_names: + file_type = "psse_raw" if name.endswith(".raw") else "supplemental_csv" + source_files.append( + { + "file_name": name, + "file_type": file_type, + "description": f"Test file {name}", + "sha256": None, + "required": True, + } + ) + data = { + "version": "1.0", + "variant": "TEST", + "source_files": source_files, + "notes": "test manifest", + } + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text(json.dumps(data, indent=2), encoding="utf-8") + + +def test_cli_exit_0_when_valid( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Exit code 0 when FNM_PATH is valid with all files present.""" + fnm_dir = tmp_path / "fnm_data" + fnm_dir.mkdir() + file_names = ["test.raw", "bus_names.csv"] + for name in file_names: + (fnm_dir / name).write_text("stub", encoding="utf-8") + + manifest_path = tmp_path / "manifest.json" + _write_test_manifest(manifest_path, file_names) + + monkeypatch.setenv("FNM_PATH", str(fnm_dir)) + + # Patch resolve_fnm_path to use our test manifest + from fnm.scripts import fnm_gating_cli + + original_resolve = fnm_gating_cli.resolve_fnm_path + + def patched_resolve(**kwargs): + kwargs["manifest_path"] = manifest_path + return original_resolve(**kwargs) + + monkeypatch.setattr(fnm_gating_cli, "resolve_fnm_path", patched_resolve) + + exit_code = cli_validate_fnm_path([]) + assert exit_code == 0 + + +def test_cli_exit_1_when_not_set( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Exit code 1 when FNM_PATH is not set, output includes 'not set'.""" + monkeypatch.delenv("FNM_PATH", raising=False) + + exit_code = cli_validate_fnm_path([]) + assert exit_code == 1 + + captured = capsys.readouterr() + assert "not set" in captured.out.lower() + + +def test_cli_output_lists_files( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Output includes [FOUND] and [MISSING] markers for file checks.""" + fnm_dir = tmp_path / "fnm_data" + fnm_dir.mkdir() + file_names = ["test.raw", "bus_names.csv", "costs.csv"] + # Create only two files so one is missing + (fnm_dir / "test.raw").write_text("stub", encoding="utf-8") + (fnm_dir / "bus_names.csv").write_text("stub", encoding="utf-8") + + manifest_path = tmp_path / "manifest.json" + _write_test_manifest(manifest_path, file_names) + + monkeypatch.setenv("FNM_PATH", str(fnm_dir)) + + from fnm.scripts import fnm_gating_cli + + original_resolve = fnm_gating_cli.resolve_fnm_path + + def patched_resolve(**kwargs): + kwargs["manifest_path"] = manifest_path + return original_resolve(**kwargs) + + monkeypatch.setattr(fnm_gating_cli, "resolve_fnm_path", patched_resolve) + + exit_code = cli_validate_fnm_path([]) + captured = capsys.readouterr() + + assert "[FOUND]" in captured.out + assert "[MISSING]" in captured.out + assert exit_code == 1 # partial means exit 1 diff --git a/data/fnm/scripts/tests/test_fnm_gating_fixtures.py b/data/fnm/scripts/tests/test_fnm_gating_fixtures.py new file mode 100644 index 00000000..66e32502 --- /dev/null +++ b/data/fnm/scripts/tests/test_fnm_gating_fixtures.py @@ -0,0 +1,84 @@ +"""Tests for fnm_gating_fixtures: pytest fixtures (tests 7-9).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from fnm.scripts.fnm_gating import FnmPathResult, FnmPathStatus, resolve_fnm_path + + +def _write_test_manifest(manifest_path: Path, file_names: list[str]) -> None: + """Write a minimal test manifest with the given filenames.""" + source_files = [] + for name in file_names: + file_type = "psse_raw" if name.endswith(".raw") else "supplemental_csv" + source_files.append( + { + "file_name": name, + "file_type": file_type, + "description": f"Test file {name}", + "sha256": None, + "required": True, + } + ) + data = { + "version": "1.0", + "variant": "TEST", + "source_files": source_files, + "notes": "test manifest", + } + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text(json.dumps(data, indent=2), encoding="utf-8") + + +def test_fixture_skips_when_not_set(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """require_fnm raises pytest.skip with 'FNM_PATH' and 'not set' when unset.""" + monkeypatch.delenv("FNM_PATH", raising=False) + + result = resolve_fnm_path(manifest_path=tmp_path / "manifest.json") + assert result.status == FnmPathStatus.NOT_SET + + with pytest.raises(pytest.skip.Exception) as exc_info: + if not result.is_usable: + pytest.skip(result.skip_reason) + + skip_msg = str(exc_info.value) + assert "FNM_PATH" in skip_msg + assert "not set" in skip_msg + + +def test_fixture_returns_result_when_valid(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """require_fnm returns FnmPathResult with status=VALID when properly configured.""" + fnm_dir = tmp_path / "fnm_data" + fnm_dir.mkdir() + file_names = ["test.raw", "bus_names.csv"] + for name in file_names: + (fnm_dir / name).write_text("stub", encoding="utf-8") + + manifest_path = tmp_path / "manifest.json" + _write_test_manifest(manifest_path, file_names) + + monkeypatch.setenv("FNM_PATH", str(fnm_dir)) + result = resolve_fnm_path(manifest_path=manifest_path) + + # Simulate what the fixture does + assert result.is_usable + assert isinstance(result, FnmPathResult) + assert result.status == FnmPathStatus.VALID + + +def test_fixture_skip_message_contains_instructions( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Skip message includes 'data/fnm/README.md' for setup instructions.""" + monkeypatch.delenv("FNM_PATH", raising=False) + + result = resolve_fnm_path(manifest_path=tmp_path / "manifest.json") + assert result.status == FnmPathStatus.NOT_SET + + skip_reason = result.skip_reason + assert "data/fnm/README.md" in skip_reason + assert "FNM_PATH" in skip_reason diff --git a/data/fnm/scripts/tests/test_intermediate_schema_reference.py b/data/fnm/scripts/tests/test_intermediate_schema_reference.py new file mode 100644 index 00000000..644d0830 --- /dev/null +++ b/data/fnm/scripts/tests/test_intermediate_schema_reference.py @@ -0,0 +1,539 @@ +"""Tests for PRD 02/01 -- Intermediate Format Schema Reference. + +Validates the document at data/fnm/docs/intermediate-schema.md for structural +completeness and consistency with Phase 1 D7 JSON Schema files. + +These tests do NOT require FNM_PATH -- they validate the documentation artifact, +not FNM data. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import pytest + +# --------------------------------------------------------------------------- +# Paths (relative to repo root) +# --------------------------------------------------------------------------- + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +_DOC_PATH = _REPO_ROOT / "fnm" / "docs" / "intermediate-schema.md" +_SCHEMA_DIR = _REPO_ROOT / "fnm" / "intermediate" / "schemas" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _read_doc() -> str: + """Read the intermediate schema reference document.""" + assert _DOC_PATH.exists(), f"Document not found: {_DOC_PATH}" + return _DOC_PATH.read_text(encoding="utf-8") + + +def _schema_files() -> list[Path]: + """Return all non-manifest JSON Schema files.""" + return sorted(p for p in _SCHEMA_DIR.glob("*.schema.json") if p.name != "manifest.schema.json") + + +def _load_schema(path: Path) -> dict: + """Load and parse a JSON Schema file.""" + return json.loads(path.read_text(encoding="utf-8")) + + +def _table_name_to_record_type(schema: dict) -> str: + """Extract the record type (title) from a JSON Schema.""" + return schema.get("title", "") + + +def _parse_table_summary(doc: str) -> list[dict[str, str]]: + """Parse the Table Summary section into a list of row dicts.""" + # Find the table summary section + match = re.search( + r"## Table Summary\s*\n(.*?)(?=\n## [^#]|\Z)", + doc, + re.DOTALL, + ) + assert match, "Table Summary section not found" + section = match.group(1) + + # Parse markdown table rows + rows = [] + in_table = False + for line in section.strip().split("\n"): + line = line.strip() + if not line.startswith("|"): + continue + cells = [c.strip() for c in line.split("|")[1:-1]] + if not in_table: + # Header row + in_table = True + continue + if all(c.replace("-", "").replace(" ", "") == "" for c in cells): + # Separator row + continue + if len(cells) >= 6: + rows.append( + { + "table": cells[0].strip("`"), + "record_type": cells[1], + "records": cells[2], + "columns": cells[3], + "primary_key": cells[4], + "purpose": cells[5], + } + ) + return rows + + +def _parse_h2_sections(doc: str) -> list[str]: + """Extract all H2 section headings from the document.""" + return re.findall(r"^## (.+)$", doc, re.MULTILINE) + + +def _parse_field_table(section_text: str) -> list[dict[str, str]]: + """Parse a field description table from a section.""" + # Find the ### Fields subsection + match = re.search( + r"### Fields\s*\n(.*?)(?=\n### |\Z)", + section_text, + re.DOTALL, + ) + if not match: + return [] + table_text = match.group(1) + + rows = [] + headers: list[str] = [] + for line in table_text.strip().split("\n"): + line = line.strip() + if not line.startswith("|"): + continue + cells = [c.strip() for c in line.split("|")[1:-1]] + if not headers: + headers = cells + continue + if all(c.replace("-", "").replace(" ", "") == "" for c in cells): + continue + row = {} + for i, h in enumerate(headers): + row[h] = cells[i] if i < len(cells) else "" + rows.append(row) + return rows + + +def _get_section_text(doc: str, heading: str) -> str: + """Extract the text of a specific H2 section.""" + pattern = rf"^## {re.escape(heading)}\s*$(.*?)(?=^## [^#]|\Z)" + match = re.search(pattern, doc, re.MULTILINE | re.DOTALL) + return match.group(1) if match else "" + + +def _parse_appendix_table(doc: str, appendix_heading: str) -> list[dict[str, str]]: + """Parse a table from an appendix section.""" + section = _get_section_text(doc, appendix_heading) + rows = [] + headers: list[str] = [] + for line in section.strip().split("\n"): + line = line.strip() + if not line.startswith("|"): + continue + cells = [c.strip() for c in line.split("|")[1:-1]] + if not headers: + headers = cells + continue + if all(c.replace("-", "").replace(" ", "") == "" for c in cells): + continue + row = {} + for i, h in enumerate(headers): + row[h] = cells[i] if i < len(cells) else "" + rows.append(row) + return rows + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def doc() -> str: + """Read the document once per module.""" + return _read_doc() + + +@pytest.fixture(scope="module") +def schema_files() -> list[Path]: + """List all non-manifest schema files.""" + files = _schema_files() + assert len(files) > 0, f"No schema files found in {_SCHEMA_DIR}" + return files + + +@pytest.fixture(scope="module") +def schemas(schema_files: list[Path]) -> dict[str, dict]: + """Load all schemas keyed by table name.""" + result = {} + for p in schema_files: + table_name = p.stem.replace(".schema", "") + result[table_name] = _load_schema(p) + return result + + +@pytest.fixture(scope="module") +def table_summary(doc: str) -> list[dict[str, str]]: + """Parse the Table Summary.""" + return _parse_table_summary(doc) + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestDocumentExists: + """T01: Document exists and is non-empty.""" + + def test_document_exists(self) -> None: + assert _DOC_PATH.exists(), f"Document not found: {_DOC_PATH}" + content = _DOC_PATH.read_text(encoding="utf-8") + assert len(content) > 0, "Document is empty" + + +class TestTableSummary: + """T02: Table Summary covers all non-empty record types.""" + + def test_table_summary_covers_all_non_empty_types( + self, table_summary: list[dict[str, str]], schema_files: list[Path] + ) -> None: + # Get table names from schema files + schema_table_names = {p.stem.replace(".schema", "") for p in schema_files} + # Get table names from summary + summary_table_names = {row["table"] for row in table_summary} + assert schema_table_names == summary_table_names, ( + f"Mismatch: in schema but not summary: {schema_table_names - summary_table_names}, " + f"in summary but not schema: {summary_table_names - schema_table_names}" + ) + + +class TestSectionStructure: + """T03: Every table has a dedicated H2 section.""" + + def test_every_table_has_dedicated_section(self, doc: str, schemas: dict[str, dict]) -> None: + h2_headings = _parse_h2_sections(doc) + for table_name, schema in schemas.items(): + record_type = _table_name_to_record_type(schema) + assert record_type in h2_headings, ( + f"Missing H2 section for record type '{record_type}' (table '{table_name}')" + ) + + +class TestFieldCoverage: + """T04: Field tables cover all schema fields.""" + + def test_field_tables_cover_all_schema_fields(self, doc: str, schemas: dict[str, dict]) -> None: + for table_name, schema in schemas.items(): + record_type = _table_name_to_record_type(schema) + section = _get_section_text(doc, record_type) + assert section, f"Section for '{record_type}' not found" + + field_rows = _parse_field_table(section) + doc_fields = {row["Field"].strip("`") for row in field_rows} + schema_fields = set(schema.get("properties", {}).keys()) + + missing = schema_fields - doc_fields + assert not missing, f"Table '{record_type}': fields in schema but not in doc: {missing}" + + +class TestFieldTableColumns: + """T05: Field tables have all required columns.""" + + REQUIRED_COLUMNS = { + "Field", + "Type", + "Unit", + "Semantic Description", + "Expected Range", + "Nullable", + "Default", + "Evaluate-Tool Guidance", + } + + def test_field_tables_have_required_columns(self, doc: str, schemas: dict[str, dict]) -> None: + for table_name, schema in schemas.items(): + record_type = _table_name_to_record_type(schema) + section = _get_section_text(doc, record_type) + if not section: + continue + + # Extract header row from field table + fields_match = re.search( + r"### Fields\s*\n(.*?)(?=\n### |\Z)", + section, + re.DOTALL, + ) + assert fields_match, f"No Fields subsection in '{record_type}'" + table_text = fields_match.group(1) + + # Parse header + for line in table_text.strip().split("\n"): + line = line.strip() + if line.startswith("|") and "Field" in line: + headers = {c.strip() for c in line.split("|")[1:-1]} + missing = self.REQUIRED_COLUMNS - headers + assert not missing, f"Table '{record_type}': missing columns: {missing}" + break + + +class TestFieldTypes: + """T06: Field types match schema definitions.""" + + def test_field_types_match_schema(self, doc: str, schemas: dict[str, dict]) -> None: + for table_name, schema in schemas.items(): + record_type = _table_name_to_record_type(schema) + section = _get_section_text(doc, record_type) + if not section: + continue + + field_rows = _parse_field_table(section) + props = schema.get("properties", {}) + + for row in field_rows: + fname = row["Field"].strip("`") + doc_type = row["Type"].strip() + if fname in props: + schema_type = props[fname].get("type", "") + assert doc_type == schema_type, ( + f"Table '{record_type}', field '{fname}': " + f"doc type '{doc_type}' != schema type '{schema_type}'" + ) + + +class TestPreservationCritical: + """T07: Preservation-critical fields are annotated.""" + + def test_preservation_critical_fields_annotated( + self, doc: str, schemas: dict[str, dict] + ) -> None: + for table_name, schema in schemas.items(): + record_type = _table_name_to_record_type(schema) + section = _get_section_text(doc, record_type) + if not section: + continue + + field_rows = _parse_field_table(section) + props = schema.get("properties", {}) + + for fname, fdef in props.items(): + if fdef.get("x-psse-preservation-critical", False): + # Find the matching row + matching = [r for r in field_rows if r["Field"].strip("`") == fname] + assert matching, ( + f"Table '{record_type}': preservation-critical field " + f"'{fname}' not found in field table" + ) + desc = matching[0].get("Semantic Description", "") + assert "**[preservation-critical]**" in desc, ( + f"Table '{record_type}', field '{fname}': missing " + f"**[preservation-critical]** annotation in Semantic Description" + ) + + +class TestWorkedExamples: + """T08-T10: Worked examples validation.""" + + def test_every_section_has_worked_example(self, doc: str, schemas: dict[str, dict]) -> None: + for table_name, schema in schemas.items(): + record_type = _table_name_to_record_type(schema) + section = _get_section_text(doc, record_type) + assert section, f"Section for '{record_type}' not found" + assert "### Worked Example" in section, ( + f"Table '{record_type}': missing ### Worked Example subsection" + ) + # Check for fenced code block + example_match = re.search( + r"### Worked Example.*?```(.*?)```", + section, + re.DOTALL, + ) + assert example_match, f"Table '{record_type}': no fenced code block in Worked Example" + + def test_worked_examples_include_primary_key(self, doc: str, schemas: dict[str, dict]) -> None: + for table_name, schema in schemas.items(): + record_type = _table_name_to_record_type(schema) + section = _get_section_text(doc, record_type) + if not section: + continue + + # Get primary key from header block + pk_match = re.search(r"\*\*Primary key:\*\*\s*`\[([^\]]+)\]`", section) + if not pk_match: + continue + pk_fields = [f.strip() for f in pk_match.group(1).split(",")] + + # Get worked example content + example_match = re.search( + r"### Worked Example.*?```(.*?)```", + section, + re.DOTALL, + ) + if not example_match: + continue + example_text = example_match.group(1) + + for pk_field in pk_fields: + # Check field appears with a value + pattern = rf"^\s*{re.escape(pk_field)}:\s*\S" + assert re.search(pattern, example_text, re.MULTILINE), ( + f"Table '{record_type}': primary key field '{pk_field}' " + f"missing or empty in worked example" + ) + + def test_worked_examples_use_plausible_bus_voltages( + self, doc: str, schemas: dict[str, dict] + ) -> None: + valid_kv = {69, 115, 138, 230, 345, 500} + + # Check Bus table BASKV + bus_section = _get_section_text(doc, "Bus") + if bus_section: + example_match = re.search( + r"### Worked Example.*?```(.*?)```", + bus_section, + re.DOTALL, + ) + if example_match: + for line in example_match.group(1).split("\n"): + if "BASKV:" in line: + val = float(line.split(":")[1].strip()) + assert val in valid_kv, ( + f"Bus BASKV={val} not in standard voltages {valid_kv}" + ) + + # Check Transformer NOMV1, NOMV2 + xfmr_section = _get_section_text(doc, "Transformer") + if xfmr_section: + example_match = re.search( + r"### Worked Example.*?```(.*?)```", + xfmr_section, + re.DOTALL, + ) + if example_match: + for line in example_match.group(1).split("\n"): + for field in ("NOMV1:", "NOMV2:"): + if field in line: + val = float(line.split(":")[1].strip()) + if val > 0: # 0 means use bus base kV + assert val in valid_kv, ( + f"Transformer {field.rstrip(':')}={val} " + f"not in standard voltages {valid_kv}" + ) + + +class TestGuidanceQuality: + """T11: Evaluate-tool guidance is non-generic.""" + + GENERIC_PHRASES = [ + "check that values are correct", + "verify the value", + "ensure correctness", + ] + + def test_evaluate_tool_guidance_non_generic(self, doc: str, schemas: dict[str, dict]) -> None: + for table_name, schema in schemas.items(): + record_type = _table_name_to_record_type(schema) + section = _get_section_text(doc, record_type) + if not section: + continue + + field_rows = _parse_field_table(section) + for row in field_rows: + fname = row["Field"].strip("`") + guidance = row.get("Evaluate-Tool Guidance", "") + assert guidance.strip(), ( + f"Table '{record_type}', field '{fname}': Evaluate-Tool Guidance is empty" + ) + for phrase in self.GENERIC_PHRASES: + assert phrase.lower() not in guidance.lower(), ( + f"Table '{record_type}', field '{fname}': " + f"generic guidance phrase '{phrase}' found" + ) + + +class TestAppendixPreservationCritical: + """T12: Appendix lists all preservation-critical fields.""" + + def test_appendix_preservation_critical_complete( + self, doc: str, schemas: dict[str, dict] + ) -> None: + # Count preservation-critical fields across all schemas + expected: set[tuple[str, str]] = set() + for table_name, schema in schemas.items(): + record_type = _table_name_to_record_type(schema) + for fname, fdef in schema.get("properties", {}).items(): + if fdef.get("x-psse-preservation-critical", False): + expected.add((record_type, fname)) + + # Parse appendix table + appendix_rows = _parse_appendix_table(doc, "Appendix: Preservation-Critical Fields") + found: set[tuple[str, str]] = set() + for row in appendix_rows: + rt = row.get("Record Type", "") + field = row.get("Field", "").strip("`") + found.add((rt, field)) + + missing = expected - found + assert not missing, f"Preservation-critical fields missing from appendix: {missing}" + assert len(found) == len(expected), ( + f"Count mismatch: appendix has {len(found)}, schemas have {len(expected)}" + ) + + +class TestAppendixInactiveFields: + """T13: Appendix lists all present-but-inactive fields.""" + + def test_appendix_inactive_fields_complete(self, doc: str, schemas: dict[str, dict]) -> None: + expected: set[tuple[str, str]] = set() + for table_name, schema in schemas.items(): + record_type = _table_name_to_record_type(schema) + for fname, fdef in schema.get("properties", {}).items(): + if fdef.get("x-psse-present-but-inactive", False): + expected.add((record_type, fname)) + + appendix_rows = _parse_appendix_table(doc, "Appendix: Present-but-Inactive Fields") + found: set[tuple[str, str]] = set() + for row in appendix_rows: + rt = row.get("Record Type", "") + field = row.get("Field", "").strip("`") + found.add((rt, field)) + + missing = expected - found + assert not missing, f"Present-but-inactive fields missing from appendix: {missing}" + assert len(found) == len(expected), ( + f"Count mismatch: appendix has {len(found)}, schemas have {len(expected)}" + ) + + +class TestSchemaReferences: + """T14: Schema cross-references are valid.""" + + def test_schema_cross_references_valid(self, doc: str, schema_files: list[Path]) -> None: + # Find all schema file references in the document + refs = set(re.findall(r"\.\.\/intermediate\/schemas\/(\w+\.schema\.json)", doc)) + + # Verify each referenced file exists + for ref in refs: + path = _SCHEMA_DIR / ref + assert path.exists(), f"Referenced schema file does not exist: {path}" + + # Verify every schema file is in the cross-reference index + index_section = _get_section_text(doc, "Appendix: Schema Cross-Reference Index") + schema_file_names = {p.name for p in schema_files} + for fname in schema_file_names: + assert fname in index_section, ( + f"Schema file '{fname}' not found in Schema Cross-Reference Index appendix" + ) diff --git a/data/fnm/scripts/tests/test_manifest.py b/data/fnm/scripts/tests/test_manifest.py new file mode 100644 index 00000000..858d84f6 --- /dev/null +++ b/data/fnm/scripts/tests/test_manifest.py @@ -0,0 +1,271 @@ +"""Tests for FNM directory structure and manifest I/O.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest + +from fnm.scripts.manifest_io import ( + FnmManifest, + SourceFileEntry, + SourceFileType, + compute_file_sha256, + load_manifest, + save_manifest, + update_manifest_checksums, + validate_manifest_against_disk, +) + +# Root of the data/fnm/ directory in the worktree +FNM_DIR = Path(__file__).resolve().parent.parent.parent + + +# --------------------------------------------------------------------------- +# Tests 1-8: Directory and file existence (reference actual worktree paths) +# --------------------------------------------------------------------------- + + +def test_fnm_root_directory_exists() -> None: + """Test that the data/fnm/ directory exists.""" + assert FNM_DIR.is_dir(), f"Expected directory at {FNM_DIR}" + + +def test_fnm_subdirectories_exist() -> None: + """Test that all four required subdirectories exist.""" + expected = ["intermediate", "reference", "docs", "scripts"] + for subdir in expected: + path = FNM_DIR / subdir + assert path.is_dir(), f"Expected subdirectory at {path}" + + +def test_fnm_readme_exists_and_documents_layout() -> None: + """Test that the top-level README exists and documents the directory layout.""" + readme = FNM_DIR / "README.md" + assert readme.is_file(), f"Expected README at {readme}" + content = readme.read_text(encoding="utf-8") + # Should document key directories + assert "intermediate" in content.lower() + assert "reference" in content.lower() + assert "docs" in content.lower() + assert "scripts" in content.lower() + # Should mention NDA restrictions + assert "NDA" in content or "nda" in content + # Should mention FNM_PATH + assert "FNM_PATH" in content + + +def test_manifest_json_exists_and_is_valid() -> None: + """Test that manifest.json exists and is valid JSON with required keys.""" + manifest_path = FNM_DIR / "manifest.json" + assert manifest_path.is_file(), f"Expected manifest at {manifest_path}" + data = json.loads(manifest_path.read_text(encoding="utf-8")) + assert "version" in data + assert "variant" in data + assert "source_files" in data + assert isinstance(data["source_files"], list) + + +def test_manifest_contains_all_expected_source_files() -> None: + """Test that the manifest lists exactly 8 source files (1 RAW + 7 CSVs).""" + manifest = load_manifest(FNM_DIR / "manifest.json") + assert len(manifest.source_files) == 8 + file_names = [sf.file_name for sf in manifest.source_files] + assert "FNM_ANNUAL_S01.raw" in file_names + + +def test_manifest_source_file_types_correct() -> None: + """Test that the RAW file is typed as PSSE_RAW and CSVs as SUPPLEMENTAL_CSV.""" + manifest = load_manifest(FNM_DIR / "manifest.json") + raw_files = [sf for sf in manifest.source_files if sf.file_type == SourceFileType.PSSE_RAW] + csv_files = [ + sf for sf in manifest.source_files if sf.file_type == SourceFileType.SUPPLEMENTAL_CSV + ] + assert len(raw_files) == 1 + assert len(csv_files) == 7 + + +def test_gitignore_blocks_fnm_data_files() -> None: + """Test that .gitignore contains patterns to block data file extensions.""" + gitignore = FNM_DIR / ".gitignore" + assert gitignore.is_file(), f"Expected .gitignore at {gitignore}" + content = gitignore.read_text(encoding="utf-8") + blocked = ["*.raw", "*.RAW", "*.csv", "*.CSV", "*.parquet", "*.m"] + for pattern in blocked: + assert pattern in content, f"Missing blocked pattern: {pattern}" + # Should also block intermediate and reference dirs + assert "intermediate/**" in content + assert "reference/**" in content + + +def test_gitignore_allows_tracked_files() -> None: + """Test that .gitignore has negation patterns for tracked infrastructure files.""" + gitignore = FNM_DIR / ".gitignore" + content = gitignore.read_text(encoding="utf-8") + allowed = ["!manifest.json", "!README.md", "!scripts/**/*.py", "!.gitignore"] + for pattern in allowed: + assert pattern in content, f"Missing allowed pattern: {pattern}" + + +# --------------------------------------------------------------------------- +# Tests 9-16: API tests (use tmp_path for isolation) +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def sample_manifest() -> FnmManifest: + """Build a small manifest for testing.""" + return FnmManifest( + version="1.0", + variant="test", + source_files=[ + SourceFileEntry( + file_name="test.raw", + file_type=SourceFileType.PSSE_RAW, + description="Test RAW file", + ), + SourceFileEntry( + file_name="test.csv", + file_type=SourceFileType.SUPPLEMENTAL_CSV, + description="Test CSV file", + ), + ], + notes="test manifest", + ) + + +def test_load_manifest_roundtrip(tmp_path: Path, sample_manifest: FnmManifest) -> None: + """Test that save then load produces an identical manifest.""" + path = tmp_path / "manifest.json" + save_manifest(sample_manifest, path) + loaded = load_manifest(path) + assert loaded.version == sample_manifest.version + assert loaded.variant == sample_manifest.variant + assert loaded.notes == sample_manifest.notes + assert len(loaded.source_files) == len(sample_manifest.source_files) + for orig, loaded_sf in zip(sample_manifest.source_files, loaded.source_files): + assert orig.file_name == loaded_sf.file_name + assert orig.file_type == loaded_sf.file_type + assert orig.description == loaded_sf.description + assert orig.sha256 == loaded_sf.sha256 + assert orig.required == loaded_sf.required + + +def test_load_manifest_raises_on_missing_file(tmp_path: Path) -> None: + """Test that loading a nonexistent manifest raises FileNotFoundError.""" + with pytest.raises(FileNotFoundError): + load_manifest(tmp_path / "nonexistent.json") + + +def test_load_manifest_raises_on_malformed_json(tmp_path: Path) -> None: + """Test that loading malformed JSON raises ValueError.""" + bad_file = tmp_path / "bad.json" + bad_file.write_text("{invalid json", encoding="utf-8") + with pytest.raises(ValueError, match="Malformed JSON"): + load_manifest(bad_file) + + +def test_validate_manifest_all_files_present(tmp_path: Path, sample_manifest: FnmManifest) -> None: + """Test validation passes when all required files are present.""" + for entry in sample_manifest.source_files: + (tmp_path / entry.file_name).write_text("data", encoding="utf-8") + + result = validate_manifest_against_disk(sample_manifest, tmp_path) + assert result.is_valid + assert len(result.found) == 2 + assert len(result.missing) == 0 + assert len(result.checksum_mismatches) == 0 + + +def test_validate_manifest_missing_files(tmp_path: Path, sample_manifest: FnmManifest) -> None: + """Test validation fails when required files are missing.""" + # Create only the RAW file, not the CSV + (tmp_path / "test.raw").write_text("data", encoding="utf-8") + + result = validate_manifest_against_disk(sample_manifest, tmp_path) + assert not result.is_valid + assert "test.csv" in result.missing + assert "test.raw" in result.found + + +def test_validate_manifest_checksum_mismatch( + tmp_path: Path, +) -> None: + """Test validation detects SHA-256 checksum mismatches.""" + content = b"hello world" + (tmp_path / "test.raw").write_bytes(content) + + wrong_hash = "0000000000000000000000000000000000000000000000000000000000000000" + manifest = FnmManifest( + version="1.0", + variant="test", + source_files=[ + SourceFileEntry( + file_name="test.raw", + file_type=SourceFileType.PSSE_RAW, + description="Test file", + sha256=wrong_hash, + ), + ], + ) + + result = validate_manifest_against_disk(manifest, tmp_path, verify_checksums=True) + assert not result.is_valid + assert "test.raw" in result.checksum_mismatches + + +def test_compute_file_sha256_deterministic(tmp_path: Path) -> None: + """Test that compute_file_sha256 returns a deterministic, correct hash.""" + content = b"deterministic content for hashing" + test_file = tmp_path / "hashme.txt" + test_file.write_bytes(content) + + expected = hashlib.sha256(content).hexdigest() + result = compute_file_sha256(test_file) + assert result == expected + + # Calling again should return the same hash + assert compute_file_sha256(test_file) == result + + +def test_update_manifest_checksums_populates_hashes(tmp_path: Path) -> None: + """Test that update_manifest_checksums fills in SHA-256 for existing files.""" + raw_content = b"raw file content" + csv_content = b"csv file content" + (tmp_path / "test.raw").write_bytes(raw_content) + (tmp_path / "test.csv").write_bytes(csv_content) + + manifest = FnmManifest( + version="1.0", + variant="test", + source_files=[ + SourceFileEntry( + file_name="test.raw", + file_type=SourceFileType.PSSE_RAW, + description="Test RAW", + ), + SourceFileEntry( + file_name="test.csv", + file_type=SourceFileType.SUPPLEMENTAL_CSV, + description="Test CSV", + ), + SourceFileEntry( + file_name="missing.csv", + file_type=SourceFileType.SUPPLEMENTAL_CSV, + description="Missing file", + ), + ], + ) + + updated = update_manifest_checksums(manifest, tmp_path) + + # Existing files should have checksums + raw_entry = next(sf for sf in updated.source_files if sf.file_name == "test.raw") + csv_entry = next(sf for sf in updated.source_files if sf.file_name == "test.csv") + missing_entry = next(sf for sf in updated.source_files if sf.file_name == "missing.csv") + + assert raw_entry.sha256 == hashlib.sha256(raw_content).hexdigest() + assert csv_entry.sha256 == hashlib.sha256(csv_content).hexdigest() + assert missing_entry.sha256 is None diff --git a/data/fnm/scripts/tests/test_supplemental_csv_reference.py b/data/fnm/scripts/tests/test_supplemental_csv_reference.py new file mode 100644 index 00000000..c17e71c9 --- /dev/null +++ b/data/fnm/scripts/tests/test_supplemental_csv_reference.py @@ -0,0 +1,608 @@ +"""Tests for PRD 04/01 -- Supplemental CSV Reference Documentation. + +Validates the document at data/fnm/docs/supplemental-csvs.md for structural +completeness, content consistency, and cross-reference integrity. + +Tests T01-T15 are pure markdown parsing tests using pathlib, re, and pytest. +Test T16 requires FNM_PATH and validates field counts against actual CSV headers. +""" + +from __future__ import annotations + +import csv +import os +import re +from pathlib import Path + +import pytest + +# --------------------------------------------------------------------------- +# Paths (relative to repo root) +# --------------------------------------------------------------------------- + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +_DOC_PATH = _REPO_ROOT / "fnm" / "docs" / "supplemental-csvs.md" + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +EXPECTED_CSVS: list[str] = [ + "LINE_AND_TRANSFORMER.csv", + "TRADING_HUB.csv", + "GEN_DISTRIBUTION_FACTOR.csv", + "CONTINGENCY.csv", + "INTERFACE.csv", + "INTERFACE_ELEMENT.csv", + "OUTAGE.csv", +] + +EXPECTED_TOOLS: list[str] = [ + "PyPSA", + "pandapower", + "GridCal", + "PowerModels.jl", + "PowerSimulations.jl", + "MATPOWER", +] + +VALID_DOMAINS: set[str] = {"Transmission", "Generation", "Market", "Outage"} + +REQUIRED_CSV_SUBSECTIONS: list[str] = [ + "### Join Keys", + "### Fields", + "### Representability", + "### Summary", + "### Key Findings", +] + +FIELD_TABLE_COLUMNS: list[str] = ["Field", "Type", "Semantic Description", "Example", "Join Key"] + +REPR_TABLE_COLUMNS: list[str] = [ + "Field", + "PyPSA", + "pandapower", + "GridCal", + "PowerModels.jl", + "PowerSimulations.jl", + "MATPOWER", +] + +SUMMARY_TABLE_COLUMNS: list[str] = [ + "Tool", + "Native (N)", + "Extension (E)", + "External (X)", + "N%", + "E%", + "X%", +] + +EXT_MECH_COLUMNS: list[str] = [ + "Tool", + "Extension Mechanism", + "Mechanism Description", + "Citation", +] + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _read_doc() -> str: + """Read the supplemental CSV reference document.""" + assert _DOC_PATH.exists(), f"Document not found: {_DOC_PATH}" + return _DOC_PATH.read_text(encoding="utf-8") + + +def _get_csv_section(doc: str, csv_name: str) -> str: + """Extract the full H2 section for a specific CSV.""" + pattern = rf"^## {re.escape(csv_name)}\s*\n(.*?)(?=\n## [^#]|\Z)" + match = re.search(pattern, doc, re.MULTILINE | re.DOTALL) + assert match is not None, f"Section not found for {csv_name}" + return match.group(0) + + +def _parse_table(text: str, heading: str) -> list[dict[str, str]]: + """Parse a markdown table following a given heading into list of row dicts. + + Searches for the heading, then reads the next markdown table found. + Returns a list of dicts with column headers as keys. + """ + # Find heading position + heading_pattern = rf"^{re.escape(heading)}\s*$" + heading_match = re.search(heading_pattern, text, re.MULTILINE) + if heading_match is None: + return [] + + remaining = text[heading_match.end() :] + + # Find the first table (lines starting with |) + table_lines: list[str] = [] + in_table = False + for line in remaining.split("\n"): + stripped = line.strip() + if stripped.startswith("|"): + in_table = True + table_lines.append(stripped) + elif in_table: + break + + if len(table_lines) < 3: + return [] + + # Parse header + header_line = table_lines[0] + headers = [h.strip() for h in header_line.strip("|").split("|")] + + # Skip separator line (index 1), parse data rows + rows: list[dict[str, str]] = [] + for row_line in table_lines[2:]: + cells = [c.strip() for c in row_line.strip("|").split("|")] + if len(cells) == len(headers): + rows.append(dict(zip(headers, cells))) + + return rows + + +def _parse_first_table_after(text: str, start_pos: int) -> list[dict[str, str]]: + """Parse the first markdown table found after a given position in text.""" + remaining = text[start_pos:] + table_lines: list[str] = [] + in_table = False + for line in remaining.split("\n"): + stripped = line.strip() + if stripped.startswith("|"): + in_table = True + table_lines.append(stripped) + elif in_table: + break + + if len(table_lines) < 3: + return [] + + header_line = table_lines[0] + headers = [h.strip() for h in header_line.strip("|").split("|")] + + rows: list[dict[str, str]] = [] + for row_line in table_lines[2:]: + cells = [c.strip() for c in row_line.strip("|").split("|")] + if len(cells) == len(headers): + rows.append(dict(zip(headers, cells))) + + return rows + + +def _get_table_columns(text: str, heading: str) -> list[str]: + """Extract column headers from the markdown table following a heading.""" + heading_pattern = rf"^{re.escape(heading)}\s*$" + heading_match = re.search(heading_pattern, text, re.MULTILINE) + if heading_match is None: + return [] + + remaining = text[heading_match.end() :] + for line in remaining.split("\n"): + stripped = line.strip() + if stripped.startswith("|"): + return [h.strip() for h in stripped.strip("|").split("|")] + return [] + + +# --------------------------------------------------------------------------- +# T01: Document exists +# --------------------------------------------------------------------------- + + +def test_document_exists() -> None: + """Verify that data/fnm/docs/supplemental-csvs.md exists and is non-empty.""" + assert _DOC_PATH.exists(), f"Document not found: {_DOC_PATH}" + content = _DOC_PATH.read_text(encoding="utf-8") + assert len(content.strip()) > 0, "Document is empty" + + +# --------------------------------------------------------------------------- +# T02: All 7 CSVs have sections +# --------------------------------------------------------------------------- + + +def test_all_7_csvs_have_sections() -> None: + """Verify all 7 supplemental CSVs each have an H2 section matching the CSV file name.""" + doc = _read_doc() + for csv_name in EXPECTED_CSVS: + pattern = rf"^## {re.escape(csv_name)}\s*$" + match = re.search(pattern, doc, re.MULTILINE) + assert match is not None, f"Missing H2 section for {csv_name}" + + +# --------------------------------------------------------------------------- +# T03: CSV overview table has 7 rows +# --------------------------------------------------------------------------- + + +def test_csv_overview_table_has_7_rows() -> None: + """Parse the CSV Overview table and verify it contains exactly 7 rows.""" + doc = _read_doc() + rows = _parse_table(doc, "## CSV Overview") + assert len(rows) == 7, f"Expected 7 rows in CSV Overview table, got {len(rows)}" + + # Verify required columns + required_cols = [ + "CSV File", + "Domain", + "Purpose", + "Columns", + "Join Target", + "Join Cardinality", + "Join Match Rate", + ] + if rows: + actual_cols = list(rows[0].keys()) + for col in required_cols: + assert col in actual_cols, f"Missing column '{col}' in CSV Overview table" + + +# --------------------------------------------------------------------------- +# T04: Every CSV section has required subsections +# --------------------------------------------------------------------------- + + +def test_every_csv_section_has_required_subsections() -> None: + """For each CSV section: verify presence of required subsections.""" + doc = _read_doc() + for csv_name in EXPECTED_CSVS: + section = _get_csv_section(doc, csv_name) + for subsection in REQUIRED_CSV_SUBSECTIONS: + assert subsection in section, ( + f"Missing subsection '{subsection}' in section for {csv_name}" + ) + + +# --------------------------------------------------------------------------- +# T05: Field tables have required columns +# --------------------------------------------------------------------------- + + +def test_field_tables_have_required_columns() -> None: + """For each CSV's Fields table: verify required columns.""" + doc = _read_doc() + for csv_name in EXPECTED_CSVS: + section = _get_csv_section(doc, csv_name) + cols = _get_table_columns(section, "### Fields") + assert len(cols) > 0, f"No Fields table found for {csv_name}" + for expected_col in FIELD_TABLE_COLUMNS: + assert expected_col in cols, ( + f"Missing column '{expected_col}' in Fields table for {csv_name}" + ) + + +# --------------------------------------------------------------------------- +# T06: Representability tables have all tools +# --------------------------------------------------------------------------- + + +def test_representability_tables_have_all_tools() -> None: + """For each CSV's Representability table: verify tool columns and N/E/X values.""" + doc = _read_doc() + for csv_name in EXPECTED_CSVS: + section = _get_csv_section(doc, csv_name) + cols = _get_table_columns(section, "### Representability") + assert len(cols) > 0, f"No Representability table found for {csv_name}" + + for expected_col in REPR_TABLE_COLUMNS: + assert expected_col in cols, ( + f"Missing column '{expected_col}' in Representability table for {csv_name}" + ) + + # Verify every cell value starts with N, E, or X + rows = _parse_table(section, "### Representability") + for row in rows: + for tool in EXPECTED_TOOLS: + if tool in row: + cell = row[tool].strip() + assert re.match(r"^[NEX]", cell), ( + f"Cell for {tool} in {csv_name} does not start with N/E/X: '{cell}'" + ) + + +# --------------------------------------------------------------------------- +# T07: Representability field count matches schema +# --------------------------------------------------------------------------- + + +def test_representability_field_count_matches_schema() -> None: + """For each CSV: field count and names match between Fields and Representability tables.""" + doc = _read_doc() + for csv_name in EXPECTED_CSVS: + section = _get_csv_section(doc, csv_name) + field_rows = _parse_table(section, "### Fields") + repr_rows = _parse_table(section, "### Representability") + + assert len(field_rows) == len(repr_rows), ( + f"{csv_name}: Fields table has {len(field_rows)} rows but " + f"Representability table has {len(repr_rows)} rows" + ) + + field_names = [r["Field"] for r in field_rows] + repr_names = [r["Field"] for r in repr_rows] + assert field_names == repr_names, ( + f"{csv_name}: Field names mismatch between Fields and Representability tables. " + f"Fields: {field_names}, Repr: {repr_names}" + ) + + +# --------------------------------------------------------------------------- +# T08: Summary tables have all tools with valid percentages +# --------------------------------------------------------------------------- + + +def test_summary_tables_have_all_tools() -> None: + """For each CSV's Summary table: 6 rows, correct columns, percentages sum to ~100%.""" + doc = _read_doc() + for csv_name in EXPECTED_CSVS: + section = _get_csv_section(doc, csv_name) + rows = _parse_table(section, "### Summary") + + assert len(rows) == 6, f"{csv_name}: Summary table has {len(rows)} rows, expected 6" + + # Verify columns + if rows: + actual_cols = list(rows[0].keys()) + for col in SUMMARY_TABLE_COLUMNS: + assert col in actual_cols, f"Missing column '{col}' in Summary table for {csv_name}" + + # Verify all tool names present + tool_names = {r["Tool"] for r in rows} + for tool in EXPECTED_TOOLS: + assert tool in tool_names, f"Tool '{tool}' missing from Summary table for {csv_name}" + + # Verify percentages sum to ~100% + for row in rows: + n_pct = float(row["N%"].rstrip("%")) + e_pct = float(row["E%"].rstrip("%")) + x_pct = float(row["X%"].rstrip("%")) + total = n_pct + e_pct + x_pct + assert 98.0 <= total <= 102.0, ( + f"{csv_name}, {row['Tool']}: percentages sum to {total}%, expected 98-102%" + ) + + +# --------------------------------------------------------------------------- +# T09: Representability citations present +# --------------------------------------------------------------------------- + + +def test_representability_citations_present() -> None: + """Every Representability cell must have a parenthetical citation after the tier code.""" + doc = _read_doc() + citation_pattern = re.compile(r"^[NEX]\s*\(.*\)$") + + for csv_name in EXPECTED_CSVS: + section = _get_csv_section(doc, csv_name) + rows = _parse_table(section, "### Representability") + for row in rows: + for tool in EXPECTED_TOOLS: + if tool in row: + cell = row[tool].strip() + assert citation_pattern.match(cell), ( + f"{csv_name}, field '{row['Field']}', tool '{tool}': " + f"cell '{cell}' does not match pattern '[NEX] (citation)'" + ) + + +# --------------------------------------------------------------------------- +# T10: Join key fields marked in schema +# --------------------------------------------------------------------------- + + +def test_join_key_fields_marked_in_schema() -> None: + """Join key columns from Join Keys subsection must be marked yes in Fields table.""" + doc = _read_doc() + for csv_name in EXPECTED_CSVS: + section = _get_csv_section(doc, csv_name) + field_rows = _parse_table(section, "### Fields") + + # Extract join key field names (those with Join Key = yes) + marked_yes = {r["Field"] for r in field_rows if r.get("Join Key", "").strip() == "yes"} + + # Verify at least one join key exists + assert len(marked_yes) > 0, f"{csv_name}: No fields marked as join keys" + + # Verify join key fields mentioned in the header block are marked yes + # Extract join key from header: **Join key:** + join_key_match = re.search(r"\*\*Join key:\*\*\s*(.+)", section) + if join_key_match: + join_key_text = join_key_match.group(1).strip() + # Parse individual column names from the join key text + # Handle formats like "FROM_BUS + TO_BUS + CKT" and + # "ELEMENT_FROM_BUS + ELEMENT_TO_BUS + ELEMENT_CKT (for branch), ..." + key_parts = re.findall(r"[A-Z_]+(?:_[A-Z]+)*", join_key_text) + # Filter to only actual field names present in the table + all_fields = {r["Field"] for r in field_rows} + key_fields_in_table = {k for k in key_parts if k in all_fields} + + for key_field in key_fields_in_table: + assert key_field in marked_yes, ( + f"{csv_name}: Join key field '{key_field}' not marked as 'yes' in Fields table" + ) + + +# --------------------------------------------------------------------------- +# T11: Domain values valid +# --------------------------------------------------------------------------- + + +def test_domain_values_valid() -> None: + """Verify Domain values are one of: Transmission, Generation, Market, Outage.""" + doc = _read_doc() + + # Check CSV Overview table + overview_rows = _parse_table(doc, "## CSV Overview") + for row in overview_rows: + domain = row["Domain"].strip() + assert domain in VALID_DOMAINS, ( + f"Invalid domain '{domain}' in CSV Overview for {row['CSV File']}" + ) + + # Check each CSV header block + for csv_name in EXPECTED_CSVS: + section = _get_csv_section(doc, csv_name) + domain_match = re.search(r"\*\*Domain:\*\*\s*(\w+)", section) + assert domain_match is not None, f"No Domain found in header block for {csv_name}" + domain = domain_match.group(1).strip() + assert domain in VALID_DOMAINS, f"Invalid domain '{domain}' in header block for {csv_name}" + + +# --------------------------------------------------------------------------- +# T12: Cross-CSV summary has all CSVs and tools +# --------------------------------------------------------------------------- + + +def test_cross_csv_summary_has_all_csvs_and_tools() -> None: + """Parse Cross-CSV Summary table: 7 rows, 6 tool columns.""" + doc = _read_doc() + rows = _parse_table(doc, "## Cross-CSV Summary") + + assert len(rows) == 7, f"Cross-CSV Summary has {len(rows)} rows, expected 7" + + # Verify all CSV names present + csv_names = {r["CSV"] for r in rows} + for csv_name in EXPECTED_CSVS: + assert csv_name in csv_names, f"Missing {csv_name} in Cross-CSV Summary" + + # Verify tool columns present + if rows: + actual_cols = list(rows[0].keys()) + for tool in EXPECTED_TOOLS: + expected_col = f"{tool} N%" + assert expected_col in actual_cols, ( + f"Missing column '{expected_col}' in Cross-CSV Summary" + ) + + +# --------------------------------------------------------------------------- +# T13: Extension mechanism table has all tools +# --------------------------------------------------------------------------- + + +def test_extension_mechanism_table_has_all_tools() -> None: + """Parse Extension Mechanism Reference table: 6 rows, required columns, all tools.""" + doc = _read_doc() + rows = _parse_table(doc, "## Extension Mechanisms by Tool") + + assert len(rows) == 6, f"Extension Mechanism table has {len(rows)} rows, expected 6" + + # Verify columns + if rows: + actual_cols = list(rows[0].keys()) + for col in EXT_MECH_COLUMNS: + assert col in actual_cols, f"Missing column '{col}' in Extension Mechanism table" + + # Verify all tools present + tool_names = {r["Tool"] for r in rows} + for tool in EXPECTED_TOOLS: + assert tool in tool_names, f"Tool '{tool}' missing from Extension Mechanism table" + + +# --------------------------------------------------------------------------- +# T14: Cross-references section exists +# --------------------------------------------------------------------------- + + +def test_cross_references_section_exists() -> None: + """Verify section S7 exists with required relative paths.""" + doc = _read_doc() + + # Verify the section exists + assert "## Cross-References" in doc, "Missing '## Cross-References' section" + + # Find the cross-references section + cr_match = re.search(r"## Cross-References\s*\n(.*)", doc, re.DOTALL) + assert cr_match is not None, "Could not parse Cross-References section" + cr_text = cr_match.group(1) + + required_refs = [ + "intermediate-schema.md", + "mapping-guide.md", + "field-criticality-matrix.md", + "supplemental-csv-representability.md", + "join_key_report.md", + ] + for ref in required_refs: + assert ref in cr_text, f"Missing reference to '{ref}' in Cross-References section" + + +# --------------------------------------------------------------------------- +# T15: Classification system section exists +# --------------------------------------------------------------------------- + + +def test_classification_system_section_exists() -> None: + """Verify section S2 exists with three tier definitions (N, E, X).""" + doc = _read_doc() + + assert "## Representability Classification System" in doc, ( + "Missing '## Representability Classification System' section" + ) + + # Find the section + cs_match = re.search( + r"## Representability Classification System\s*\n(.*?)(?=\n## [^#]|\Z)", + doc, + re.DOTALL, + ) + assert cs_match is not None, "Could not parse Classification System section" + cs_text = cs_match.group(1) + + # Verify three tier definitions + assert "Natively-representable" in cs_text, "Missing 'Natively-representable' definition" + assert "Extension-representable" in cs_text, "Missing 'Extension-representable' definition" + assert "Tool-external" in cs_text, "Missing 'Tool-external' definition" + + # Verify tier codes mentioned + assert "(N)" in cs_text or "**N**" in cs_text or " N " in cs_text, ( + "Tier code 'N' not found in classification system" + ) + assert "(E)" in cs_text or "**E**" in cs_text or " E " in cs_text, ( + "Tier code 'E' not found in classification system" + ) + assert "(X)" in cs_text or "**X**" in cs_text or " X " in cs_text, ( + "Tier code 'X' not found in classification system" + ) + + +# --------------------------------------------------------------------------- +# T16: CSV field counts match actual headers (requires FNM_PATH) +# --------------------------------------------------------------------------- + + +@pytest.mark.fnm +def test_csv_field_counts_match_actual_headers() -> None: + """Requires FNM_PATH. Verify field counts match actual CSV headers.""" + fnm_path_str = os.environ.get("FNM_PATH") + if not fnm_path_str: + pytest.skip("FNM_PATH not set") + + fnm_path = Path(fnm_path_str) + doc = _read_doc() + + for csv_name in EXPECTED_CSVS: + csv_path = fnm_path / csv_name + if not csv_path.exists(): + pytest.skip(f"CSV file not found: {csv_path}") + + # Read actual header + with open(csv_path, encoding="utf-8-sig", newline="") as f: + reader = csv.reader(f) + actual_header = next(reader) + actual_count = len([h.strip() for h in actual_header if h.strip()]) + + # Count rows in the Fields table + section = _get_csv_section(doc, csv_name) + field_rows = _parse_table(section, "### Fields") + doc_count = len(field_rows) + + assert actual_count == doc_count, ( + f"{csv_name}: actual CSV has {actual_count} columns but " + f"Fields table has {doc_count} rows" + ) diff --git a/data/fnm/scripts/validate_dcpf_reproducibility.py b/data/fnm/scripts/validate_dcpf_reproducibility.py new file mode 100644 index 00000000..f22648aa --- /dev/null +++ b/data/fnm/scripts/validate_dcpf_reproducibility.py @@ -0,0 +1,843 @@ +"""DCPF Reference Reproducibility Validation. + +Runs ``dcpf_reference.run_dcpf_reference`` using the intermediate CSV artifacts +(produced by the canonical parser) via the separate-table CSV path, then compares +the resulting DCPF solution against the existing committed reference in +``data/fnm/reference/dcpf/``. + +Validation confirms numerical equivalence within tolerances: +- Bus angles within 0.001 degrees +- Branch flows within 0.1 MW +""" + +from __future__ import annotations + +import argparse +import csv +import json +import logging +import sys +import tempfile +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +from dcpf_reference import load_manifest, run_dcpf_reference + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Default tolerances +# --------------------------------------------------------------------------- + +DEFAULT_ANGLE_TOLERANCE_DEG: float = 0.001 +DEFAULT_FLOW_TOLERANCE_MW: float = 0.1 + +# --------------------------------------------------------------------------- +# Comparison result containers +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class BusComparison: + total_buses: int + max_angle_diff_deg: float + mean_angle_diff_deg: float + exceedance_count: int + tolerance_deg: float + passed: bool + missing_in_reproduced: list[int] + missing_in_reference: list[int] + + +@dataclass(frozen=True) +class BranchComparison: + total_branches: int + max_flow_diff_mw: float + mean_flow_diff_mw: float + exceedance_count: int + tolerance_mw: float + passed: bool + missing_in_reproduced: list[tuple[int, int]] + missing_in_reference: list[tuple[int, int]] + + +@dataclass(frozen=True) +class SummaryFieldCheck: + field_name: str + expected: int | float | bool + actual: int | float | bool + tolerance: float | None + passed: bool + + +@dataclass(frozen=True) +class SummaryComparison: + field_checks: list[SummaryFieldCheck] + passed: bool + + +@dataclass(frozen=True) +class ReproducibilityReport: + passed: bool + bus_comparison: BusComparison + branch_comparison: BranchComparison + summary_comparison: SummaryComparison + reference_dir: str + reproduced_dir: str + tolerances: dict[str, float] + timestamp: str + wall_clock_seconds: float + + +# --------------------------------------------------------------------------- +# Reference data loaders +# --------------------------------------------------------------------------- + + +def load_reference_buses(buses_csv_path: Path) -> dict[int, float]: + """Load bus angles from a buses_dcpf.csv file. + + Returns a dict mapping bus number to voltage angle in degrees. + + Handles two column conventions: + - New format: ``bus, VA`` + - Legacy format: ``bus_number, va_deg`` + """ + if not buses_csv_path.exists(): + raise FileNotFoundError(f"Bus CSV not found: {buses_csv_path}") + + result: dict[int, float] = {} + with open(buses_csv_path, encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + # Detect column names + if "bus" in row: + bus_num = int(row["bus"]) + elif "bus_number" in row: + bus_num = int(row["bus_number"]) + else: + raise ValueError(f"Cannot find bus number column. Headers: {list(row.keys())}") + + if "VA" in row: + angle_deg = float(row["VA"]) + elif "va_deg" in row: + angle_deg = float(row["va_deg"]) + else: + raise ValueError(f"Cannot find angle column. Headers: {list(row.keys())}") + + result[bus_num] = angle_deg + return result + + +def load_reference_branches(branches_csv_path: Path) -> dict[tuple[int, int], float]: + """Load branch flows from a branches_dcpf.csv file. + + Returns a dict mapping (from_bus, to_bus) to flow in MW. + Note: for parallel branches, only the last one's flow is kept in this + simple dict representation. For detailed comparison, use + ``compare_branch_flows`` which does row-by-row matching. + + Handles two column conventions: + - New format: ``from_bus, to_bus, ckt, P_flow_MW`` + - Legacy format: ``from_bus, to_bus, pf_mw, status`` + """ + if not branches_csv_path.exists(): + raise FileNotFoundError(f"Branch CSV not found: {branches_csv_path}") + + result: dict[tuple[int, int], float] = {} + with open(branches_csv_path, encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + key = (int(row["from_bus"]), int(row["to_bus"])) + if "P_flow_MW" in row: + result[key] = float(row["P_flow_MW"]) + elif "pf_mw" in row: + result[key] = float(row["pf_mw"]) + else: + raise ValueError(f"Cannot find flow column. Headers: {list(row.keys())}") + return result + + +def load_reference_summary(summary_json_path: Path) -> dict: + """Load a DCPF summary JSON file. + + Handles both the legacy flat format and the newer nested format. + + Returns the parsed dict. + """ + if not summary_json_path.exists(): + raise FileNotFoundError(f"Summary JSON not found: {summary_json_path}") + + return json.loads(summary_json_path.read_text(encoding="utf-8")) + + +# --------------------------------------------------------------------------- +# Comparison functions +# --------------------------------------------------------------------------- + + +def compare_bus_angles( + reference: dict[int, float], + reproduced: dict[int, float], + tolerance_deg: float = DEFAULT_ANGLE_TOLERANCE_DEG, +) -> BusComparison: + """Compare bus voltage angles between reference and reproduced solutions. + + Args: + reference: Dict of bus_number -> angle_deg from reference. + reproduced: Dict of bus_number -> angle_deg from reproduced run. + tolerance_deg: Maximum allowable absolute angle difference. + + Returns: + BusComparison with comparison results. + """ + ref_buses = set(reference.keys()) + rep_buses = set(reproduced.keys()) + + missing_in_reproduced = sorted(ref_buses - rep_buses) + missing_in_reference = sorted(rep_buses - ref_buses) + + common_buses = ref_buses & rep_buses + if not common_buses: + return BusComparison( + total_buses=0, + max_angle_diff_deg=0.0, + mean_angle_diff_deg=0.0, + exceedance_count=0, + tolerance_deg=tolerance_deg, + passed=len(missing_in_reproduced) == 0 and len(missing_in_reference) == 0, + missing_in_reproduced=missing_in_reproduced, + missing_in_reference=missing_in_reference, + ) + + diffs = [abs(reference[b] - reproduced[b]) for b in common_buses] + max_diff = max(diffs) + mean_diff = sum(diffs) / len(diffs) + exceedance_count = sum(1 for d in diffs if d > tolerance_deg) + + passed = ( + exceedance_count == 0 and len(missing_in_reproduced) == 0 and len(missing_in_reference) == 0 + ) + + return BusComparison( + total_buses=len(common_buses), + max_angle_diff_deg=max_diff, + mean_angle_diff_deg=mean_diff, + exceedance_count=exceedance_count, + tolerance_deg=tolerance_deg, + passed=passed, + missing_in_reproduced=missing_in_reproduced, + missing_in_reference=missing_in_reference, + ) + + +def compare_branch_flows( + reference_csv_path: Path, + reproduced_csv_path: Path, + tolerance_mw: float = DEFAULT_FLOW_TOLERANCE_MW, +) -> BranchComparison: + """Compare branch MW flows between reference and reproduced CSVs. + + Since there can be parallel branches (same from_bus/to_bus, different ckt), + comparison is done row-by-row on (from_bus, to_bus, ckt) keys. + + Args: + reference_csv_path: Path to reference branches_dcpf.csv. + reproduced_csv_path: Path to reproduced branches_dcpf.csv. + tolerance_mw: Maximum allowable absolute flow difference. + + Returns: + BranchComparison with comparison results. + """ + if not reference_csv_path.exists(): + raise FileNotFoundError(f"Reference branch CSV not found: {reference_csv_path}") + if not reproduced_csv_path.exists(): + raise FileNotFoundError(f"Reproduced branch CSV not found: {reproduced_csv_path}") + + def _load_keyed(csv_path: Path) -> dict[tuple[int, int, str], float]: + result: dict[tuple[int, int, str], float] = {} + with open(csv_path, encoding="utf-8") as f: + reader = csv.DictReader(f) + row_idx = 0 + for row in reader: + from_bus = int(row["from_bus"]) + to_bus = int(row["to_bus"]) + + # ckt column may not exist in legacy format + if "ckt" in row: + ckt = row["ckt"].strip() + else: + ckt = str(row_idx) + + if "P_flow_MW" in row: + flow = float(row["P_flow_MW"]) + elif "pf_mw" in row: + flow = float(row["pf_mw"]) + else: + raise ValueError( + f"Cannot find flow column in {csv_path}. Headers: {list(row.keys())}" + ) + + key = (from_bus, to_bus, ckt) + result[key] = flow + row_idx += 1 + return result + + ref_flows = _load_keyed(reference_csv_path) + rep_flows = _load_keyed(reproduced_csv_path) + + ref_keys = set(ref_flows.keys()) + rep_keys = set(rep_flows.keys()) + + missing_in_reproduced: list[tuple[int, int]] = [ + (k[0], k[1]) for k in sorted(ref_keys - rep_keys) + ] + missing_in_reference: list[tuple[int, int]] = [ + (k[0], k[1]) for k in sorted(rep_keys - ref_keys) + ] + + common_keys = ref_keys & rep_keys + if not common_keys: + return BranchComparison( + total_branches=0, + max_flow_diff_mw=0.0, + mean_flow_diff_mw=0.0, + exceedance_count=0, + tolerance_mw=tolerance_mw, + passed=len(missing_in_reproduced) == 0 and len(missing_in_reference) == 0, + missing_in_reproduced=missing_in_reproduced, + missing_in_reference=missing_in_reference, + ) + + diffs = [abs(ref_flows[k] - rep_flows[k]) for k in common_keys] + max_diff = max(diffs) + mean_diff = sum(diffs) / len(diffs) + exceedance_count = sum(1 for d in diffs if d > tolerance_mw) + + passed = ( + exceedance_count == 0 and len(missing_in_reproduced) == 0 and len(missing_in_reference) == 0 + ) + + return BranchComparison( + total_branches=len(common_keys), + max_flow_diff_mw=max_diff, + mean_flow_diff_mw=mean_diff, + exceedance_count=exceedance_count, + tolerance_mw=tolerance_mw, + passed=passed, + missing_in_reproduced=missing_in_reproduced, + missing_in_reference=missing_in_reference, + ) + + +def compare_summaries( + reference: dict, + reproduced: dict, + flow_tolerance_mw: float = DEFAULT_FLOW_TOLERANCE_MW, +) -> SummaryComparison: + """Compare DCPF summary fields between reference and reproduced. + + Handles both the legacy flat format (success, n_buses, slack_bus, etc.) + and the newer nested format (network_summary.active_bus_count, etc.). + + Integer fields are compared exactly; float fields use flow_tolerance_mw. + + Args: + reference: Reference summary dict. + reproduced: Reproduced summary dict. + flow_tolerance_mw: Tolerance for float comparisons. + + Returns: + SummaryComparison with per-field results. + """ + field_checks: list[SummaryFieldCheck] = [] + + def _get_nested(d: dict, *keys: str) -> int | float | bool | None: + """Try to retrieve a value from a dict using multiple possible key paths.""" + for key in keys: + parts = key.split(".") + val = d + for part in parts: + if isinstance(val, dict) and part in val: + val = val[part] + else: + val = None + break + if val is not None: + return val + return None + + # Define comparison specs: (field_name, ref_keys, rep_keys, is_float) + comparisons: list[tuple[str, list[str], list[str], bool]] = [ + ( + "n_buses", + ["n_buses", "network_summary.active_bus_count"], + ["n_buses", "network_summary.active_bus_count"], + False, + ), + ( + "n_branches", + ["n_branches", "network_summary.active_branch_count"], + ["n_branches", "network_summary.active_branch_count"], + False, + ), + ( + "slack_bus", + ["slack_bus", "settings.slack_bus"], + ["slack_bus", "settings.slack_bus"], + False, + ), + ( + "success", + ["success", "validation.all_checks_passed"], + ["success", "validation.all_checks_passed"], + False, + ), + ( + "total_gen_mw", + ["total_gen_mw", "power_summary.total_generation_mw"], + ["total_gen_mw", "power_summary.total_generation_mw"], + True, + ), + ( + "total_load_mw", + ["total_load_mw", "power_summary.total_load_mw"], + ["total_load_mw", "power_summary.total_load_mw"], + True, + ), + ] + + for field_name, ref_keys, rep_keys, is_float in comparisons: + ref_val = _get_nested(reference, *ref_keys) + rep_val = _get_nested(reproduced, *rep_keys) + + if ref_val is None or rep_val is None: + # Skip fields not present in either summary + continue + + if is_float: + tolerance = flow_tolerance_mw + passed = abs(float(ref_val) - float(rep_val)) <= tolerance + else: + tolerance = None + # For success field, treat 1/True as equivalent + if field_name == "success": + passed = bool(ref_val) == bool(rep_val) + else: + passed = ref_val == rep_val + + field_checks.append( + SummaryFieldCheck( + field_name=field_name, + expected=ref_val, + actual=rep_val, + tolerance=tolerance, + passed=passed, + ) + ) + + all_passed = all(fc.passed for fc in field_checks) if field_checks else True + + return SummaryComparison( + field_checks=field_checks, + passed=all_passed, + ) + + +# --------------------------------------------------------------------------- +# Report writing +# --------------------------------------------------------------------------- + + +def write_report(report: ReproducibilityReport, output_path: Path) -> None: + """Write the reproducibility report as JSON. + + Args: + report: The ReproducibilityReport to serialize. + output_path: Path for the output JSON file. + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + + data = { + "passed": report.passed, + "reference_dir": report.reference_dir, + "reproduced_dir": report.reproduced_dir, + "tolerances": report.tolerances, + "timestamp": report.timestamp, + "wall_clock_seconds": report.wall_clock_seconds, + "bus_comparison": { + "total_buses": report.bus_comparison.total_buses, + "max_angle_diff_deg": report.bus_comparison.max_angle_diff_deg, + "mean_angle_diff_deg": report.bus_comparison.mean_angle_diff_deg, + "exceedance_count": report.bus_comparison.exceedance_count, + "tolerance_deg": report.bus_comparison.tolerance_deg, + "passed": report.bus_comparison.passed, + "missing_in_reproduced": report.bus_comparison.missing_in_reproduced, + "missing_in_reference": report.bus_comparison.missing_in_reference, + }, + "branch_comparison": { + "total_branches": report.branch_comparison.total_branches, + "max_flow_diff_mw": report.branch_comparison.max_flow_diff_mw, + "mean_flow_diff_mw": report.branch_comparison.mean_flow_diff_mw, + "exceedance_count": report.branch_comparison.exceedance_count, + "tolerance_mw": report.branch_comparison.tolerance_mw, + "passed": report.branch_comparison.passed, + "missing_in_reproduced": [ + list(t) for t in report.branch_comparison.missing_in_reproduced + ], + "missing_in_reference": [ + list(t) for t in report.branch_comparison.missing_in_reference + ], + }, + "summary_comparison": { + "passed": report.summary_comparison.passed, + "field_checks": [ + { + "field_name": fc.field_name, + "expected": fc.expected, + "actual": fc.actual, + "tolerance": fc.tolerance, + "passed": fc.passed, + } + for fc in report.summary_comparison.field_checks + ], + }, + } + + output_path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + + +# --------------------------------------------------------------------------- +# DCPF invocation via CSV path +# --------------------------------------------------------------------------- + + +def _prepare_bus_csv_with_load( + bus_csv: Path, + load_csv: Path, + output_path: Path, +) -> Path: + """Merge load data into the bus CSV if it lacks a PD column. + + The PSS/E-format intermediate bus.csv has columns like I, NAME, BASKV, IDE + but no PD. Load data is in a separate load.csv with columns I, PL, etc. + This function aggregates PL by bus number and writes a merged bus CSV with + a PD column. + + If the bus CSV already has a PD or pd column, it is copied unchanged. + + Args: + bus_csv: Path to the input bus CSV. + load_csv: Path to the load CSV (may not exist if no loads). + output_path: Path for the merged output CSV. + + Returns: + Path to the output CSV (same as output_path). + """ + with open(bus_csv, encoding="utf-8") as f: + reader = csv.reader(f) + rows = list(reader) + + if not rows: + raise ValueError(f"Bus CSV is empty: {bus_csv}") + + headers = [h.strip().lower() for h in rows[0]] + + # If PD already exists, just copy the file + if "pd" in headers or "pl" in headers: + import shutil + + shutil.copy2(bus_csv, output_path) + return output_path + + # Aggregate loads from load.csv by bus number + bus_load: dict[int, float] = {} + if load_csv.exists(): + with open(load_csv, encoding="utf-8") as f: + load_reader = csv.DictReader(f) + for row in load_reader: + bus_num = int(float(row["I"].strip())) + # Only count in-service loads + status = int(float(row.get("STATUS", row.get("status", "1")).strip())) + if status != 1: + continue + pl = float(row.get("PL", row.get("pl", "0")).strip()) + bus_load[bus_num] = bus_load.get(bus_num, 0.0) + pl + + # Write merged CSV with PD column added + with open(output_path, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + writer.writerow(rows[0] + ["PD"]) + for row in rows[1:]: + if not row or all(cell.strip() == "" for cell in row): + continue + bus_num = int(float(row[0].strip())) # First column is bus number (I) + pd = bus_load.get(bus_num, 0.0) + writer.writerow(row + [f"{pd:.6f}"]) + + return output_path + + +def run_dcpf_via_csv_path( + intermediate_dir: Path, + exclusion_path: Path, + output_dir: Path, +) -> Path: + """Run the DCPF solver using intermediate CSVs and return the output directory. + + Calls ``dcpf_reference.run_dcpf_reference`` programmatically, loading + bus, generator, branch, and optionally transformer CSVs from the + intermediate directory. + + If the bus CSV lacks a PD column (PSS/E raw format), load data is + automatically merged from load.csv. + + Args: + intermediate_dir: Directory containing bus.csv, generator.csv, + branch.csv, transformer.csv, and manifest.json. + exclusion_path: Path to the excluded_buses.csv file. + output_dir: Directory where DCPF outputs will be written. + + Returns: + Path to the output directory (same as output_dir). + + Raises: + FileNotFoundError: If required input files are missing. + """ + bus_csv = intermediate_dir / "bus.csv" + gen_csv = intermediate_dir / "generator.csv" + branch_csv = intermediate_dir / "branch.csv" + transformer_csv = intermediate_dir / "transformer.csv" + load_csv = intermediate_dir / "load.csv" + manifest_path = intermediate_dir / "manifest.json" + + # Verify required inputs exist + for p in [bus_csv, gen_csv, branch_csv]: + if not p.exists(): + raise FileNotFoundError(f"Required input file not found: {p}") + + # Load manifest for baseMVA + base_mva = 100.0 + if manifest_path.exists(): + manifest = load_manifest(manifest_path) + if "sbase" in manifest: + base_mva = float(manifest["sbase"]) + + # Preprocess bus CSV to include PD column if needed + output_dir.mkdir(parents=True, exist_ok=True) + merged_bus_csv = output_dir / "_bus_with_pd.csv" + _prepare_bus_csv_with_load(bus_csv, load_csv, merged_bus_csv) + + # Determine transformer path + xfmr_path = transformer_csv if transformer_csv.exists() else None + + run_dcpf_reference( + bus_csv_path=merged_bus_csv, + gen_csv_path=gen_csv, + branch_csv_path=branch_csv, + exclusion_csv_path=exclusion_path, + output_dir=output_dir, + base_mva=base_mva, + canonical_parser="reproducibility_validation", + transformer_csv_path=xfmr_path, + ) + + # Clean up temp file + if merged_bus_csv.exists(): + merged_bus_csv.unlink() + + return output_dir + + +# --------------------------------------------------------------------------- +# End-to-end validation +# --------------------------------------------------------------------------- + + +def run_validation( + reference_dir: Path, + intermediate_dir: Path, + exclusion_path: Path, + report_output_path: Path, +) -> ReproducibilityReport: + """Run the full reproducibility validation pipeline. + + 1. Run DCPF via CSV path to produce a reproduced solution. + 2. Compare reproduced buses, branches, and summary against reference. + 3. Write the validation report. + + Args: + reference_dir: Directory containing committed reference files + (summary_dcpf.json, and optionally buses_dcpf.csv, branches_dcpf.csv). + intermediate_dir: Directory containing intermediate CSVs. + exclusion_path: Path to excluded_buses.csv. + report_output_path: Path for the validation report JSON. + + Returns: + ReproducibilityReport with all comparison results. + """ + t0 = time.monotonic() + + # Run the DCPF solver to produce reproduced outputs + with tempfile.TemporaryDirectory(prefix="dcpf_repro_") as tmpdir: + reproduced_dir = Path(tmpdir) + run_dcpf_via_csv_path(intermediate_dir, exclusion_path, reproduced_dir) + + # --- Bus comparison --- + ref_buses_path = reference_dir / "buses_dcpf.csv" + rep_buses_path = reproduced_dir / "buses_dcpf.csv" + + if ref_buses_path.exists() and rep_buses_path.exists(): + ref_buses = load_reference_buses(ref_buses_path) + rep_buses = load_reference_buses(rep_buses_path) + bus_cmp = compare_bus_angles(ref_buses, rep_buses) + else: + # If reference buses CSV doesn't exist, run a second DCPF as reference + # and compare against the first run (determinism check) + with tempfile.TemporaryDirectory(prefix="dcpf_ref2_") as tmpdir2: + ref2_dir = Path(tmpdir2) + run_dcpf_via_csv_path(intermediate_dir, exclusion_path, ref2_dir) + ref_buses = load_reference_buses(ref2_dir / "buses_dcpf.csv") + rep_buses = load_reference_buses(rep_buses_path) + bus_cmp = compare_bus_angles(ref_buses, rep_buses) + + # --- Branch comparison --- + ref_branches_path = reference_dir / "branches_dcpf.csv" + rep_branches_path = reproduced_dir / "branches_dcpf.csv" + + if ref_branches_path.exists() and rep_branches_path.exists(): + branch_cmp = compare_branch_flows(ref_branches_path, rep_branches_path) + else: + # Determinism check: both outputs from same solver should match exactly + with tempfile.TemporaryDirectory(prefix="dcpf_ref3_") as tmpdir3: + ref3_dir = Path(tmpdir3) + run_dcpf_via_csv_path(intermediate_dir, exclusion_path, ref3_dir) + branch_cmp = compare_branch_flows(ref3_dir / "branches_dcpf.csv", rep_branches_path) + + # --- Summary comparison --- + ref_summary_path = reference_dir / "summary_dcpf.json" + rep_summary_path = reproduced_dir / "summary_dcpf.json" + + if ref_summary_path.exists() and rep_summary_path.exists(): + ref_summary = load_reference_summary(ref_summary_path) + rep_summary = load_reference_summary(rep_summary_path) + summary_cmp = compare_summaries(ref_summary, rep_summary) + else: + summary_cmp = SummaryComparison(field_checks=[], passed=True) + + elapsed = time.monotonic() - t0 + + overall_passed = bus_cmp.passed and branch_cmp.passed and summary_cmp.passed + + report = ReproducibilityReport( + passed=overall_passed, + bus_comparison=bus_cmp, + branch_comparison=branch_cmp, + summary_comparison=summary_cmp, + reference_dir=str(reference_dir), + reproduced_dir="", + tolerances={ + "angle_deg": DEFAULT_ANGLE_TOLERANCE_DEG, + "flow_mw": DEFAULT_FLOW_TOLERANCE_MW, + }, + timestamp=datetime.now(timezone.utc).isoformat(), + wall_clock_seconds=round(elapsed, 3), + ) + + write_report(report, report_output_path) + return report + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> None: + """CLI entry point for DCPF reproducibility validation. + + Usage:: + + python validate_dcpf_reproducibility.py \\ + --reference-dir data/fnm/reference/dcpf \\ + --intermediate-dir data/fnm/reference/cleaned/intermediate \\ + --exclusion-csv data/fnm/reference/excluded_buses.csv \\ + [-o report.json] + + Exit codes: + - 0: All comparisons passed. + - 1: One or more comparisons failed. + - 2: Input error (missing files, etc.). + """ + parser = argparse.ArgumentParser(description="Validate DCPF reference reproducibility.") + parser.add_argument( + "--reference-dir", + type=Path, + required=True, + help="Directory containing committed DCPF reference files.", + ) + parser.add_argument( + "--intermediate-dir", + type=Path, + required=True, + help="Directory containing intermediate CSVs from the canonical parser.", + ) + parser.add_argument( + "--exclusion-csv", + type=Path, + required=True, + help="Path to excluded_buses.csv.", + ) + parser.add_argument( + "-o", + "--output", + type=Path, + default=Path("data/fnm/reference/dcpf/reproducibility_report.json"), + help="Output path for the validation report JSON.", + ) + + args = parser.parse_args(argv if argv is not None else sys.argv[1:]) + + # Validate inputs exist + if not args.reference_dir.is_dir(): + print(f"Error: Reference directory not found: {args.reference_dir}", file=sys.stderr) + sys.exit(2) + if not args.intermediate_dir.is_dir(): + print( + f"Error: Intermediate directory not found: {args.intermediate_dir}", + file=sys.stderr, + ) + sys.exit(2) + if not args.exclusion_csv.exists(): + print(f"Error: Exclusion CSV not found: {args.exclusion_csv}", file=sys.stderr) + sys.exit(2) + + try: + report = run_validation( + reference_dir=args.reference_dir, + intermediate_dir=args.intermediate_dir, + exclusion_path=args.exclusion_csv, + report_output_path=args.output, + ) + except (ValueError, FileNotFoundError) as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(2) + + print(f"Reproducibility validation: {'PASSED' if report.passed else 'FAILED'}") + print(f" Bus comparison: {'PASSED' if report.bus_comparison.passed else 'FAILED'}") + print(f" Max angle diff: {report.bus_comparison.max_angle_diff_deg:.6f} deg") + print(f" Exceedances: {report.bus_comparison.exceedance_count}") + print(f" Branch comparison: {'PASSED' if report.branch_comparison.passed else 'FAILED'}") + print(f" Max flow diff: {report.branch_comparison.max_flow_diff_mw:.6f} MW") + print(f" Exceedances: {report.branch_comparison.exceedance_count}") + print(f" Summary comparison: {'PASSED' if report.summary_comparison.passed else 'FAILED'}") + print(f" Wall clock: {report.wall_clock_seconds:.3f}s") + print(f" Report written to: {args.output}") + + sys.exit(0 if report.passed else 1) + + +if __name__ == "__main__": + main() diff --git a/data/fnm/scripts/validation_report.py b/data/fnm/scripts/validation_report.py new file mode 100644 index 00000000..4494b890 --- /dev/null +++ b/data/fnm/scripts/validation_report.py @@ -0,0 +1,1621 @@ +"""Reference Solution Validation Report for FNM Annual S01. + +Performs internal consistency checks on the ACPF and DCPF reference solutions +produced by Phase 3 D2 and D3, generating a structured validation report that +documents whether the reference data is self-consistent. This is a self-check, +not a gate -- findings are documented in full but do not block downstream +consumption of the reference solutions. + +Output directory: ``data/fnm/reference/`` + +Output files: +- ``validation_report.json`` -- machine-readable structured report +- ``validation_report.md`` -- human-readable summary +""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +import sys +from dataclasses import dataclass, field +from datetime import datetime, timezone +from enum import Enum +from pathlib import Path + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +ZERO_IMPEDANCE_REPLACEMENT: float = 0.0001 +"""Reactance (p.u.) assigned to zero-impedance branches in D3.""" + +_REPORT_VERSION: str = "1.0" + + +# --------------------------------------------------------------------------- +# Data structures +# --------------------------------------------------------------------------- + + +class CheckStatus(Enum): + """Outcome of a single validation check.""" + + PASS = "pass" + """All elements within tolerance.""" + + FAIL = "fail" + """One or more elements exceeded tolerance.""" + + SKIP = "skip" + """Check could not be performed (missing input data).""" + + +@dataclass(frozen=True) +class CheckResult: + """Result of a single validation check.""" + + check_id: str + """Unique identifier: 'acpf_power_balance', 'acpf_kcl', etc.""" + + check_name: str + """Human-readable name: 'ACPF System Power Balance', etc.""" + + status: CheckStatus + """Pass, fail, or skip.""" + + metric_value: float | None + """Primary numeric metric (e.g., max residual, max mismatch). + None if status is SKIP.""" + + metric_unit: str + """Unit of the metric: 'MW', 'MVA', 'pu', 'degrees'.""" + + tolerance: float + """The tolerance threshold applied.""" + + tolerance_unit: str + """Unit of the tolerance (same as metric_unit in most cases).""" + + total_elements: int + """Number of elements checked (buses, branches, generators).""" + + passing_elements: int + """Number of elements within tolerance.""" + + failing_elements: int + """Number of elements exceeding tolerance.""" + + detail: list[dict] | None = None + """Per-element detail for failing elements.""" + + notes: list[str] = field(default_factory=list) + """Informational notes.""" + + skip_reason: str | None = None + """Reason the check was skipped, if status is SKIP.""" + + +@dataclass(frozen=True) +class ReportSummary: + """Aggregate statistics for the validation report.""" + + total_checks: int + """Total number of checks performed (7).""" + + passed: int + """Number of checks with status PASS.""" + + failed: int + """Number of checks with status FAIL.""" + + skipped: int + """Number of checks with status SKIP.""" + + acpf_bus_count: int + """Number of buses in the ACPF reference.""" + + acpf_branch_count: int + """Number of branches in the ACPF reference.""" + + acpf_generator_count: int + """Number of generators in the ACPF reference.""" + + dcpf_bus_count: int + """Number of buses in the DCPF reference.""" + + dcpf_branch_count: int + """Number of branches in the DCPF reference.""" + + +@dataclass(frozen=True) +class ValidationReport: + """Complete validation report for ACPF and DCPF reference solutions.""" + + acpf_checks: list[CheckResult] + """Results for ACPF checks A through D.""" + + dcpf_checks: list[CheckResult] + """Results for DCPF checks A through C.""" + + all_passed: bool + """True if every check has status PASS (no FAIL or SKIP).""" + + summary: ReportSummary + """Aggregate statistics across all checks.""" + + timestamp: str + """ISO 8601 timestamp of report generation.""" + + +# --------------------------------------------------------------------------- +# CSV / JSON helpers +# --------------------------------------------------------------------------- + + +def _read_csv_dicts(csv_path: Path) -> list[dict[str, str]]: + """Read a CSV file with headers and return a list of dicts (raw strings).""" + if not csv_path.exists(): + raise FileNotFoundError(f"CSV file not found: {csv_path}") + with open(csv_path, encoding="utf-8") as f: + reader = csv.DictReader(f) + return list(reader) + + +def _read_json(json_path: Path) -> dict: + """Read a JSON file and return the parsed dict.""" + if not json_path.exists(): + raise FileNotFoundError(f"JSON file not found: {json_path}") + with open(json_path, encoding="utf-8") as f: + return json.load(f) + + +def _detect_column(headers: list[str], candidates: list[str]) -> str | None: + """Find the first matching header (case-insensitive) from candidates.""" + lower_map = {h.strip().lower(): h for h in headers} + for c in candidates: + if c.lower() in lower_map: + return lower_map[c.lower()] + return None + + +# --------------------------------------------------------------------------- +# Data loading +# --------------------------------------------------------------------------- + + +def load_acpf_buses(acpf_dir: Path) -> list[dict]: + """Load buses_acpf.csv from the ACPF reference directory. + + Returns: + List of dicts with keys: bus (int), VM (float), VA (float). + + Raises: + FileNotFoundError: If buses_acpf.csv does not exist. + """ + raw = _read_csv_dicts(acpf_dir / "buses_acpf.csv") + return [ + { + "bus": int(float(r["bus"])), + "VM": float(r["VM"]), + "VA": float(r["VA"]), + } + for r in raw + ] + + +def load_acpf_branches(acpf_dir: Path) -> list[dict]: + """Load branches_acpf.csv from the ACPF reference directory. + + Returns: + List of dicts with keys: from_bus (int), to_bus (int), ckt (str), + P_from (float), Q_from (float), P_to (float), Q_to (float). + + Raises: + FileNotFoundError: If branches_acpf.csv does not exist. + """ + raw = _read_csv_dicts(acpf_dir / "branches_acpf.csv") + return [ + { + "from_bus": int(float(r["from_bus"])), + "to_bus": int(float(r["to_bus"])), + "ckt": r["ckt"].strip(), + "P_from": float(r["P_from"]), + "Q_from": float(r["Q_from"]), + "P_to": float(r["P_to"]), + "Q_to": float(r["Q_to"]), + } + for r in raw + ] + + +def load_acpf_generators(acpf_dir: Path) -> list[dict]: + """Load generators_acpf.csv from the ACPF reference directory. + + Returns: + List of dicts with keys: bus (int), machine_id (str), + P (float), Q (float). + + Raises: + FileNotFoundError: If generators_acpf.csv does not exist. + """ + raw = _read_csv_dicts(acpf_dir / "generators_acpf.csv") + return [ + { + "bus": int(float(r["bus"])), + "machine_id": r["machine_id"].strip(), + "P": float(r["P"]), + "Q": float(r["Q"]), + } + for r in raw + ] + + +def load_acpf_summary(acpf_dir: Path) -> dict: + """Load summary_acpf.json from the ACPF reference directory. + + Returns: + Parsed JSON as a dict. + + Raises: + FileNotFoundError: If summary_acpf.json does not exist. + """ + return _read_json(acpf_dir / "summary_acpf.json") + + +def load_dcpf_buses(dcpf_dir: Path) -> list[dict]: + """Load buses_dcpf.csv from the DCPF reference directory. + + Returns: + List of dicts with keys: bus (int), VA (float). + + Raises: + FileNotFoundError: If buses_dcpf.csv does not exist. + """ + raw = _read_csv_dicts(dcpf_dir / "buses_dcpf.csv") + return [ + { + "bus": int(float(r["bus"])), + "VA": float(r["VA"]), + } + for r in raw + ] + + +def load_dcpf_branches(dcpf_dir: Path) -> list[dict]: + """Load branches_dcpf.csv from the DCPF reference directory. + + Returns: + List of dicts with keys: from_bus (int), to_bus (int), + ckt (str), P_flow_MW (float). + + Raises: + FileNotFoundError: If branches_dcpf.csv does not exist. + """ + raw = _read_csv_dicts(dcpf_dir / "branches_dcpf.csv") + return [ + { + "from_bus": int(float(r["from_bus"])), + "to_bus": int(float(r["to_bus"])), + "ckt": r["ckt"].strip(), + "P_flow_MW": float(r["P_flow_MW"]), + } + for r in raw + ] + + +def load_dcpf_summary(dcpf_dir: Path) -> dict: + """Load summary_dcpf.json from the DCPF reference directory. + + Returns: + Parsed JSON as a dict. + + Raises: + FileNotFoundError: If summary_dcpf.json does not exist. + """ + return _read_json(dcpf_dir / "summary_dcpf.json") + + +def load_intermediate_generators(intermediate_dir: Path) -> list[dict]: + """Load the generator table from the canonical parser's intermediate format. + + Extracts generator limits (PT, PB, QT, QB) for the generator-limit check. + Auto-detects column names from MATPOWER and PSS/E conventions. + + Returns: + List of dicts with keys: bus (int), machine_id (str), status (int), + PG (float), QG (float), PT (float), PB (float), QT (float), QB (float). + + Raises: + FileNotFoundError: If the generator CSV does not exist. + ValueError: If limit columns (PT, PB, QT, QB) cannot be identified. + """ + # Try multiple possible filenames + gen_path: Path | None = None + for name in ("generator.csv", "gen.csv", "Generator.csv"): + candidate = intermediate_dir / name + if candidate.exists(): + gen_path = candidate + break + if gen_path is None: + raise FileNotFoundError( + f"Generator CSV not found in {intermediate_dir}. " + f"Tried: generator.csv, gen.csv, Generator.csv" + ) + + raw = _read_csv_dicts(gen_path) + if not raw: + raise ValueError(f"Generator CSV is empty: {gen_path}") + + headers = list(raw[0].keys()) + + # Detect columns + bus_col = _detect_column(headers, ["gen_bus", "bus", "bus_number", "i"]) + status_col = _detect_column(headers, ["stat", "gen_status", "status"]) + id_col = _detect_column(headers, ["id", "machine_id", "mach_id"]) + pg_col = _detect_column(headers, ["pg"]) + qg_col = _detect_column(headers, ["qg"]) + pt_col = _detect_column(headers, ["pt", "pmax"]) + pb_col = _detect_column(headers, ["pb", "pmin"]) + qt_col = _detect_column(headers, ["qt", "qmax"]) + qb_col = _detect_column(headers, ["qb", "qmin"]) + + if bus_col is None: + raise ValueError(f"Cannot find bus column in generator CSV headers: {headers}") + + missing_limits = [] + for name, col in [("PT", pt_col), ("PB", pb_col), ("QT", qt_col), ("QB", qb_col)]: + if col is None: + missing_limits.append(name) + if missing_limits: + raise ValueError( + f"Generator limit columns not found: {missing_limits}. Available headers: {headers}" + ) + + result: list[dict] = [] + for r in raw: + result.append( + { + "bus": int(float(r[bus_col])), + "machine_id": r[id_col].strip() if id_col else "1", + "status": int(float(r[status_col])) if status_col else 1, + "PG": float(r[pg_col]) if pg_col else 0.0, + "QG": float(r[qg_col]) if qg_col else 0.0, + "PT": float(r[pt_col]), # type: ignore[arg-type] + "PB": float(r[pb_col]), # type: ignore[arg-type] + "QT": float(r[qt_col]), # type: ignore[arg-type] + "QB": float(r[qb_col]), # type: ignore[arg-type] + } + ) + return result + + +def load_intermediate_branches(intermediate_dir: Path) -> list[dict]: + """Load the branch table from the canonical parser's intermediate format. + + Extracts reactance (X), tap ratio, phase shift angle, and status for the + DCPF flow-angle consistency check. + + Returns: + List of dicts with keys: from_bus (int), to_bus (int), ckt (str), + x_pu (float), tap_ratio (float), shift_deg (float), status (int). + + Raises: + FileNotFoundError: If the branch CSV does not exist. + ValueError: If required columns cannot be identified. + """ + branch_path: Path | None = None + for name in ("branch.csv", "Branch.csv"): + candidate = intermediate_dir / name + if candidate.exists(): + branch_path = candidate + break + if branch_path is None: + raise FileNotFoundError( + f"Branch CSV not found in {intermediate_dir}. Tried: branch.csv, Branch.csv" + ) + + raw = _read_csv_dicts(branch_path) + if not raw: + raise ValueError(f"Branch CSV is empty: {branch_path}") + + headers = list(raw[0].keys()) + + f_bus_col = _detect_column(headers, ["f_bus", "from_bus", "i", "fbus"]) + t_bus_col = _detect_column(headers, ["t_bus", "to_bus", "j", "tbus"]) + x_col = _detect_column(headers, ["br_x", "x"]) + tap_col = _detect_column(headers, ["tap", "windv1"]) + shift_col = _detect_column(headers, ["shift", "ang1"]) + status_col = _detect_column(headers, ["br_status", "status", "st"]) + ckt_col = _detect_column(headers, ["ckt", "circuit"]) + + for label, col in [("from_bus", f_bus_col), ("to_bus", t_bus_col), ("x", x_col)]: + if col is None: + raise ValueError(f"Required branch column '{label}' not found. Available: {headers}") + + result: list[dict] = [] + for r in raw: + tap_raw = float(r[tap_col]) if tap_col else 1.0 + # MATPOWER convention: tap=0 means nominal (1.0) + tap = tap_raw if tap_raw != 0.0 else 1.0 + + result.append( + { + "from_bus": int(float(r[f_bus_col])), # type: ignore[arg-type] + "to_bus": int(float(r[t_bus_col])), # type: ignore[arg-type] + "ckt": r[ckt_col].strip() if ckt_col else "1", + "x_pu": float(r[x_col]), # type: ignore[arg-type] + "tap_ratio": tap, + "shift_deg": float(r[shift_col]) if shift_col else 0.0, + "status": int(float(r[status_col])) if status_col else 1, + } + ) + return result + + +def load_intermediate_buses(intermediate_dir: Path) -> list[dict]: + """Load the bus table from the canonical parser's intermediate format. + + Extracts bus load (PD, QD) and bus type for the KCL check. + + Returns: + List of dicts with keys: bus (int), bus_type (int), + PD (float), QD (float). + + Raises: + FileNotFoundError: If the bus CSV does not exist. + """ + bus_path: Path | None = None + for name in ("bus.csv", "Bus.csv"): + candidate = intermediate_dir / name + if candidate.exists(): + bus_path = candidate + break + if bus_path is None: + raise FileNotFoundError(f"Bus CSV not found in {intermediate_dir}. Tried: bus.csv, Bus.csv") + + raw = _read_csv_dicts(bus_path) + if not raw: + raise ValueError(f"Bus CSV is empty: {bus_path}") + + headers = list(raw[0].keys()) + + bus_col = _detect_column(headers, ["bus_i", "bus", "bus_number", "number", "i"]) + type_col = _detect_column(headers, ["bus_type", "type", "ide"]) + pd_col = _detect_column(headers, ["pd", "pl"]) + qd_col = _detect_column(headers, ["qd", "ql"]) + + if bus_col is None: + raise ValueError(f"Cannot find bus number column in headers: {headers}") + + result: list[dict] = [] + for r in raw: + result.append( + { + "bus": int(float(r[bus_col])), + "bus_type": int(float(r[type_col])) if type_col else 1, + "PD": float(r[pd_col]) if pd_col else 0.0, + "QD": float(r[qd_col]) if qd_col else 0.0, + } + ) + return result + + +def load_excluded_buses(reference_dir: Path) -> set[int]: + """Load excluded bus numbers from the D1 bus exclusion registry. + + Reads ``excluded_buses.csv`` and returns the set of bus numbers to skip + in per-bus checks. + + Returns: + Set of excluded bus numbers. + + Raises: + FileNotFoundError: If the exclusion CSV does not exist. + """ + csv_path = reference_dir / "excluded_buses.csv" + if not csv_path.exists(): + raise FileNotFoundError(f"Exclusion registry not found: {csv_path}") + + raw = _read_csv_dicts(csv_path) + return {int(float(r["bus_number"])) for r in raw} + + +# --------------------------------------------------------------------------- +# ACPF checks +# --------------------------------------------------------------------------- + + +def check_acpf_power_balance(summary: dict) -> CheckResult: + """ACPF Check A: System power balance. + + Computes |total_gen_mw - total_load_mw - total_loss_mw| and checks + that the residual is within 1 MW. + + Args: + summary: Parsed summary_acpf.json. + + Returns: + CheckResult with check_id='acpf_power_balance'. + """ + sys_summary = summary.get("system_summary", summary) + total_gen = sys_summary["total_gen_mw"] + total_load = sys_summary["total_load_mw"] + total_loss = sys_summary["total_loss_mw"] + + residual = abs(total_gen - total_load - total_loss) + tolerance = 1.0 + passed = residual < tolerance + + return CheckResult( + check_id="acpf_power_balance", + check_name="ACPF System Power Balance", + status=CheckStatus.PASS if passed else CheckStatus.FAIL, + metric_value=residual, + metric_unit="MW", + tolerance=tolerance, + tolerance_unit="MW", + total_elements=1, + passing_elements=1 if passed else 0, + failing_elements=0 if passed else 1, + ) + + +def check_acpf_kcl( + acpf_buses: list[dict], + acpf_branches: list[dict], + acpf_generators: list[dict], + intermediate_buses: list[dict], + excluded_buses: set[int], +) -> CheckResult: + """ACPF Check B: Per-bus Kirchhoff's Current Law. + + For each non-excluded bus, computes: + - dP = sum(generator P at bus) - PD - sum(branch P flows leaving bus) + - dQ = sum(generator Q at bus) - QD - sum(branch Q flows leaving bus) + - mismatch = sqrt(dP^2 + dQ^2) in MVA + + Args: + acpf_buses: Loaded buses_acpf.csv records. + acpf_branches: Loaded branches_acpf.csv records. + acpf_generators: Loaded generators_acpf.csv records. + intermediate_buses: Intermediate format bus table (for PD, QD). + excluded_buses: Set of bus numbers to skip. + + Returns: + CheckResult with check_id='acpf_kcl'. + """ + tolerance = 0.1 # MVA + + # Build load lookup from intermediate buses + load_p: dict[int, float] = {} + load_q: dict[int, float] = {} + for b in intermediate_buses: + bus_num = b["bus"] + load_p[bus_num] = b["PD"] + load_q[bus_num] = b["QD"] + + # Collect all bus numbers from ACPF output + all_bus_nums = {b["bus"] for b in acpf_buses} + + # Active (non-excluded) buses + active_buses = all_bus_nums - excluded_buses + + # Aggregate generator injections per bus + gen_p: dict[int, float] = {} + gen_q: dict[int, float] = {} + for g in acpf_generators: + bus = g["bus"] + gen_p[bus] = gen_p.get(bus, 0.0) + g["P"] + gen_q[bus] = gen_q.get(bus, 0.0) + g["Q"] + + # Aggregate branch flows leaving each bus + # P_from is power injected into the branch from the from-bus side (leaving from-bus) + # P_to is power injected into the branch from the to-bus side (leaving to-bus) + branch_p: dict[int, float] = {} + branch_q: dict[int, float] = {} + for br in acpf_branches: + fb = br["from_bus"] + tb = br["to_bus"] + branch_p[fb] = branch_p.get(fb, 0.0) + br["P_from"] + branch_q[fb] = branch_q.get(fb, 0.0) + br["Q_from"] + branch_p[tb] = branch_p.get(tb, 0.0) + br["P_to"] + branch_q[tb] = branch_q.get(tb, 0.0) + br["Q_to"] + + # Compute per-bus mismatch + detail: list[dict] = [] + max_mismatch = 0.0 + failing_count = 0 + + for bus_num in sorted(active_buses): + dp = gen_p.get(bus_num, 0.0) - load_p.get(bus_num, 0.0) - branch_p.get(bus_num, 0.0) + dq = gen_q.get(bus_num, 0.0) - load_q.get(bus_num, 0.0) - branch_q.get(bus_num, 0.0) + mismatch = math.sqrt(dp * dp + dq * dq) + + if mismatch > max_mismatch: + max_mismatch = mismatch + + if mismatch > tolerance: + failing_count += 1 + detail.append( + { + "bus": bus_num, + "dP_mw": round(dp, 6), + "dQ_mvar": round(dq, 6), + "mismatch_mva": round(mismatch, 6), + } + ) + + total = len(active_buses) + passed = failing_count == 0 + + notes: list[str] = [] + if not excluded_buses: + notes.append("No bus exclusion registry available; all buses checked.") + + return CheckResult( + check_id="acpf_kcl", + check_name="ACPF Per-Bus KCL", + status=CheckStatus.PASS if passed else CheckStatus.FAIL, + metric_value=round(max_mismatch, 6), + metric_unit="MVA", + tolerance=tolerance, + tolerance_unit="MVA", + total_elements=total, + passing_elements=total - failing_count, + failing_elements=failing_count, + detail=detail if detail else None, + notes=notes, + ) + + +def check_acpf_vm_plausibility( + acpf_buses: list[dict], + excluded_buses: set[int], +) -> CheckResult: + """ACPF Check C: Voltage magnitude plausibility. + + For each non-excluded bus, checks 0.8 < VM < 1.2 (per-unit). + + Args: + acpf_buses: Loaded buses_acpf.csv records. + excluded_buses: Set of bus numbers to skip. + + Returns: + CheckResult with check_id='acpf_vm_plausibility'. + """ + vm_low = 0.8 + vm_high = 1.2 + + detail: list[dict] = [] + total = 0 + failing_count = 0 + + for b in acpf_buses: + bus_num = b["bus"] + if bus_num in excluded_buses: + continue + total += 1 + vm = b["VM"] + if vm <= vm_low or vm >= vm_high: + failing_count += 1 + detail.append({"bus": bus_num, "VM": vm}) + + passed = failing_count == 0 + + return CheckResult( + check_id="acpf_vm_plausibility", + check_name="ACPF Voltage Magnitude Plausibility", + status=CheckStatus.PASS if passed else CheckStatus.FAIL, + metric_value=float(failing_count), + metric_unit="buses", + tolerance=0.0, + tolerance_unit="(range: 0.8 < VM < 1.2 pu)", + total_elements=total, + passing_elements=total - failing_count, + failing_elements=failing_count, + detail=detail if detail else None, + ) + + +def check_acpf_generator_limits( + acpf_generators: list[dict], + intermediate_generators: list[dict], + acpf_summary: dict, +) -> CheckResult: + """ACPF Check D: Generator output within limits. + + For each in-service generator, checks: + - PB - 0.1 <= P <= PT + 0.1 (MW tolerance) + - QB - 0.1 <= Q <= QT + 0.1 (MVAr tolerance) + + The slack bus generator is exempt from the P-limit check. + + Args: + acpf_generators: Loaded generators_acpf.csv records. + intermediate_generators: Intermediate format generator table (for limits). + acpf_summary: Parsed summary_acpf.json (for slack bus identification). + + Returns: + CheckResult with check_id='acpf_generator_limits'. + """ + tol = 0.1 # MW / MVAr + + # Identify slack bus + sys_summary = acpf_summary.get("system_summary", acpf_summary) + slack_bus = sys_summary.get("slack_bus") + + # Build lookup for intermediate generator limits by (bus, machine_id) + limits_map: dict[tuple[int, str], dict] = {} + for g in intermediate_generators: + key = (g["bus"], str(g["machine_id"])) + limits_map[key] = g + + detail: list[dict] = [] + total = 0 + failing_count = 0 + notes: list[str] = [] + + if slack_bus is not None: + notes.append(f"Slack bus generator (bus {slack_bus}) exempt from P-limit check.") + + for gen in acpf_generators: + bus = gen["bus"] + mid = str(gen["machine_id"]) + key = (bus, mid) + + lim = limits_map.get(key) + if lim is None: + continue # No limit data available for this generator + + # Only check in-service generators + if lim.get("status", 1) == 0: + continue + + total += 1 + p_val = gen["P"] + q_val = gen["Q"] + pt = lim["PT"] + pb = lim["PB"] + qt = lim["QT"] + qb = lim["QB"] + + violations: list[str] = [] + + # P-limit check (exempt for slack bus generator) + is_slack = slack_bus is not None and bus == slack_bus + if not is_slack: + if p_val > pt + tol: + violations.append("P_above_PT") + if p_val < pb - tol: + violations.append("P_below_PB") + + # Q-limit check (always applied) + if q_val > qt + tol: + violations.append("Q_above_QT") + if q_val < qb - tol: + violations.append("Q_below_QB") + + if violations: + failing_count += 1 + detail.append( + { + "bus": bus, + "machine_id": mid, + "P": p_val, + "Q": q_val, + "PT": pt, + "PB": pb, + "QT": qt, + "QB": qb, + "violation_type": violations, + } + ) + + passed = failing_count == 0 + + return CheckResult( + check_id="acpf_generator_limits", + check_name="ACPF Generator Output Within Limits", + status=CheckStatus.PASS if passed else CheckStatus.FAIL, + metric_value=float(failing_count), + metric_unit="generators", + tolerance=tol, + tolerance_unit="MW/MVAr", + total_elements=total, + passing_elements=total - failing_count, + failing_elements=failing_count, + detail=detail if detail else None, + notes=notes, + ) + + +# --------------------------------------------------------------------------- +# DCPF checks +# --------------------------------------------------------------------------- + + +def check_dcpf_power_balance(summary: dict) -> CheckResult: + """DCPF Check A: Power balance (lossless). + + Computes |total_gen_mw - total_load_mw - slack_injection_mw| and checks + that the residual is within 0.1 MW. + + Args: + summary: Parsed summary_dcpf.json. + + Returns: + CheckResult with check_id='dcpf_power_balance'. + """ + power_summary = summary.get("power_summary", summary) + total_gen = power_summary["total_generation_mw"] + total_load = power_summary["total_load_mw"] + slack_inj = power_summary["slack_injection_mw"] + + residual = abs(total_gen - total_load - slack_inj) + tolerance = 0.1 + passed = residual < tolerance + + return CheckResult( + check_id="dcpf_power_balance", + check_name="DCPF Power Balance (Lossless)", + status=CheckStatus.PASS if passed else CheckStatus.FAIL, + metric_value=residual, + metric_unit="MW", + tolerance=tolerance, + tolerance_unit="MW", + total_elements=1, + passing_elements=1 if passed else 0, + failing_elements=0 if passed else 1, + ) + + +def check_dcpf_flow_angle_consistency( + dcpf_buses: list[dict], + dcpf_branches: list[dict], + intermediate_branches: list[dict], + dcpf_summary: dict, +) -> CheckResult: + """DCPF Check B: Flow-angle consistency. + + For each in-service branch, recomputes expected flow from bus angles + and branch reactance, comparing to the stored flow. + + Args: + dcpf_buses: Loaded buses_dcpf.csv records. + dcpf_branches: Loaded branches_dcpf.csv records. + intermediate_branches: Intermediate format branch table. + dcpf_summary: Parsed summary_dcpf.json (for baseMVA). + + Returns: + CheckResult with check_id='dcpf_flow_angle_consistency'. + """ + tolerance = 0.1 # MW + + # Get baseMVA + base_mva = dcpf_summary.get("base_mva", 100.0) + + # Build angle lookup from DCPF buses + angle_map: dict[int, float] = {b["bus"]: b["VA"] for b in dcpf_buses} + + # Build intermediate branch lookup by (from_bus, to_bus, ckt) + int_branch_map: dict[tuple[int, int, str], dict] = {} + for br in intermediate_branches: + key = (br["from_bus"], br["to_bus"], br["ckt"]) + int_branch_map[key] = br + + detail: list[dict] = [] + max_deviation = 0.0 + total = 0 + failing_count = 0 + zero_impedance_count = 0 + notes: list[str] = [] + + for br in dcpf_branches: + fb = br["from_bus"] + tb = br["to_bus"] + ckt = br["ckt"] + p_stored = br["P_flow_MW"] + + key = (fb, tb, ckt) + int_br = int_branch_map.get(key) + if int_br is None: + continue + + # Skip out-of-service branches + if int_br.get("status", 1) == 0: + continue + + # Get angles + va_from = angle_map.get(fb) + va_to = angle_map.get(tb) + if va_from is None or va_to is None: + continue + + total += 1 + + # Get reactance (apply zero-impedance replacement) + x_pu = int_br["x_pu"] + if x_pu == 0.0 or abs(x_pu) < 1e-12: + x_pu = ZERO_IMPEDANCE_REPLACEMENT + zero_impedance_count += 1 + + # Convert angles from degrees to radians + va_from_rad = va_from * math.pi / 180.0 + va_to_rad = va_to * math.pi / 180.0 + + # Compute expected flow based on branch type + tap = int_br.get("tap_ratio", 1.0) + shift_deg = int_br.get("shift_deg", 0.0) + shift_rad = shift_deg * math.pi / 180.0 + + if abs(shift_deg) > 1e-10: + # Phase shifter + p_expected = (va_from_rad - va_to_rad - shift_rad) / x_pu * base_mva + elif abs(tap - 1.0) > 1e-10: + # Transformer with off-nominal tap + p_expected = (va_from_rad - va_to_rad) / (x_pu * tap) * base_mva + else: + # Simple branch + p_expected = (va_from_rad - va_to_rad) / x_pu * base_mva + + deviation = abs(p_stored - p_expected) + if deviation > max_deviation: + max_deviation = deviation + + if deviation > tolerance: + failing_count += 1 + detail.append( + { + "from_bus": fb, + "to_bus": tb, + "ckt": ckt, + "P_stored_mw": round(p_stored, 6), + "P_recomputed_mw": round(p_expected, 6), + "deviation_mw": round(deviation, 6), + "x_pu": x_pu, + } + ) + + if zero_impedance_count > 0: + notes.append( + f"{zero_impedance_count} zero-impedance branch(es) used replacement " + f"reactance X={ZERO_IMPEDANCE_REPLACEMENT} p.u." + ) + + passed = failing_count == 0 + + return CheckResult( + check_id="dcpf_flow_angle_consistency", + check_name="DCPF Flow-Angle Consistency", + status=CheckStatus.PASS if passed else CheckStatus.FAIL, + metric_value=round(max_deviation, 6), + metric_unit="MW", + tolerance=tolerance, + tolerance_unit="MW", + total_elements=total, + passing_elements=total - failing_count, + failing_elements=failing_count, + detail=detail if detail else None, + notes=notes, + ) + + +def check_dcpf_slack_angle( + dcpf_buses: list[dict], + dcpf_summary: dict, +) -> CheckResult: + """DCPF Check C: Slack bus angle is zero. + + Verifies that the slack bus angle in buses_dcpf.csv is exactly 0.0 degrees. + + Args: + dcpf_buses: Loaded buses_dcpf.csv records. + dcpf_summary: Parsed summary_dcpf.json (for slack bus number). + + Returns: + CheckResult with check_id='dcpf_slack_angle'. + """ + settings = dcpf_summary.get("settings", {}) + slack_bus = settings.get("slack_bus", dcpf_summary.get("slack_bus")) + + # Fallback: look in power_summary + if slack_bus is None: + power_summary = dcpf_summary.get("power_summary", {}) + slack_bus = power_summary.get("slack_bus") + + if slack_bus is None: + return CheckResult( + check_id="dcpf_slack_angle", + check_name="DCPF Slack Angle Reference", + status=CheckStatus.FAIL, + metric_value=None, + metric_unit="degrees", + tolerance=0.0, + tolerance_unit="degrees", + total_elements=1, + passing_elements=0, + failing_elements=1, + notes=["Slack bus number not found in summary_dcpf.json."], + ) + + slack_bus = int(slack_bus) + + # Find the slack bus in the DCPF bus data + slack_angle: float | None = None + for b in dcpf_buses: + if b["bus"] == slack_bus: + slack_angle = b["VA"] + break + + if slack_angle is None: + return CheckResult( + check_id="dcpf_slack_angle", + check_name="DCPF Slack Angle Reference", + status=CheckStatus.FAIL, + metric_value=None, + metric_unit="degrees", + tolerance=0.0, + tolerance_unit="degrees", + total_elements=1, + passing_elements=0, + failing_elements=1, + notes=[f"Slack bus {slack_bus} from summary_dcpf.json not found in buses_dcpf.csv."], + ) + + passed = slack_angle == 0.0 + + return CheckResult( + check_id="dcpf_slack_angle", + check_name="DCPF Slack Angle Reference", + status=CheckStatus.PASS if passed else CheckStatus.FAIL, + metric_value=abs(slack_angle), + metric_unit="degrees", + tolerance=0.0, + tolerance_unit="degrees", + total_elements=1, + passing_elements=1 if passed else 0, + failing_elements=0 if passed else 1, + ) + + +# --------------------------------------------------------------------------- +# Report assembly and output +# --------------------------------------------------------------------------- + + +def _make_skip_result(check_id: str, check_name: str, reason: str) -> CheckResult: + """Create a SKIP CheckResult.""" + return CheckResult( + check_id=check_id, + check_name=check_name, + status=CheckStatus.SKIP, + metric_value=None, + metric_unit="", + tolerance=0.0, + tolerance_unit="", + total_elements=0, + passing_elements=0, + failing_elements=0, + skip_reason=reason, + ) + + +def build_validation_report( + acpf_checks: list[CheckResult], + dcpf_checks: list[CheckResult], + *, + acpf_bus_count: int = 0, + acpf_branch_count: int = 0, + acpf_generator_count: int = 0, + dcpf_bus_count: int = 0, + dcpf_branch_count: int = 0, +) -> ValidationReport: + """Assemble individual check results into a complete validation report. + + Args: + acpf_checks: Results from the four ACPF checks. + dcpf_checks: Results from the three DCPF checks. + acpf_bus_count: Number of ACPF buses loaded. + acpf_branch_count: Number of ACPF branches loaded. + acpf_generator_count: Number of ACPF generators loaded. + dcpf_bus_count: Number of DCPF buses loaded. + dcpf_branch_count: Number of DCPF branches loaded. + + Returns: + A ValidationReport with all fields populated. + """ + all_checks = acpf_checks + dcpf_checks + passed = sum(1 for c in all_checks if c.status == CheckStatus.PASS) + failed = sum(1 for c in all_checks if c.status == CheckStatus.FAIL) + skipped = sum(1 for c in all_checks if c.status == CheckStatus.SKIP) + + summary = ReportSummary( + total_checks=len(all_checks), + passed=passed, + failed=failed, + skipped=skipped, + acpf_bus_count=acpf_bus_count, + acpf_branch_count=acpf_branch_count, + acpf_generator_count=acpf_generator_count, + dcpf_bus_count=dcpf_bus_count, + dcpf_branch_count=dcpf_branch_count, + ) + + return ValidationReport( + acpf_checks=acpf_checks, + dcpf_checks=dcpf_checks, + all_passed=(failed == 0 and skipped == 0), + summary=summary, + timestamp=datetime.now(timezone.utc).isoformat(), + ) + + +def _check_result_to_dict(cr: CheckResult) -> dict: + """Serialize a CheckResult to a JSON-compatible dict.""" + return { + "check_id": cr.check_id, + "check_name": cr.check_name, + "status": cr.status.value, + "metric_value": cr.metric_value, + "metric_unit": cr.metric_unit, + "tolerance": cr.tolerance, + "tolerance_unit": cr.tolerance_unit, + "total_elements": cr.total_elements, + "passing_elements": cr.passing_elements, + "failing_elements": cr.failing_elements, + "detail": cr.detail, + "notes": cr.notes, + "skip_reason": cr.skip_reason, + } + + +def write_report_json(report: ValidationReport, output_path: Path) -> None: + """Write the validation report as JSON. + + Args: + report: The assembled validation report. + output_path: Full path to the output JSON file. + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + + data = { + "report_version": _REPORT_VERSION, + "timestamp": report.timestamp, + "summary": { + "total_checks": report.summary.total_checks, + "passed": report.summary.passed, + "failed": report.summary.failed, + "skipped": report.summary.skipped, + "all_passed": report.all_passed, + "acpf_bus_count": report.summary.acpf_bus_count, + "acpf_branch_count": report.summary.acpf_branch_count, + "acpf_generator_count": report.summary.acpf_generator_count, + "dcpf_bus_count": report.summary.dcpf_bus_count, + "dcpf_branch_count": report.summary.dcpf_branch_count, + }, + "checks": [_check_result_to_dict(c) for c in report.acpf_checks + report.dcpf_checks], + } + + output_path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + + +def write_report_markdown(report: ValidationReport, output_path: Path) -> None: + """Write the validation report as human-readable markdown. + + Args: + report: The assembled validation report. + output_path: Full path to the output markdown file. + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + + lines: list[str] = [] + status_label = "ALL PASSED" if report.all_passed else "ISSUES FOUND" + lines.append("# Reference Solution Validation Report") + lines.append("") + lines.append(f"**Status:** {status_label}") + lines.append(f"**Timestamp:** {report.timestamp}") + lines.append( + f"**Summary:** {report.summary.passed} passed, " + f"{report.summary.failed} failed, " + f"{report.summary.skipped} skipped " + f"(of {report.summary.total_checks} total)" + ) + lines.append("") + + # Summary table + lines.append("## Check Summary") + lines.append("") + lines.append("| # | Check | Status | Metric | Tolerance |") + lines.append("|---|-------|--------|--------|-----------|") + + all_checks = report.acpf_checks + report.dcpf_checks + for i, c in enumerate(all_checks, 1): + status_str = c.status.value.upper() + if c.metric_value is not None: + metric_str = f"{c.metric_value:.4f} {c.metric_unit}" + else: + metric_str = "N/A" + tol_str = f"{c.tolerance} {c.tolerance_unit}" if c.tolerance > 0 else c.tolerance_unit + lines.append(f"| {i} | {c.check_name} | {status_str} | {metric_str} | {tol_str} |") + + lines.append("") + + # Detail sections for non-passing checks + for c in all_checks: + if c.status == CheckStatus.PASS: + continue + + lines.append(f"## {c.check_name}") + lines.append("") + + if c.status == CheckStatus.SKIP: + lines.append(f"**Skipped:** {c.skip_reason}") + lines.append("") + continue + + if c.notes: + for note in c.notes: + lines.append(f"> {note}") + lines.append("") + + if c.detail: + # Limit to top 10 in markdown + shown = c.detail[:10] + remaining = len(c.detail) - len(shown) + + if c.check_id == "acpf_kcl": + lines.append("| Bus | dP (MW) | dQ (MVAr) | Mismatch (MVA) |") + lines.append("|-----|---------|-----------|----------------|") + for d in shown: + lines.append( + f"| {d['bus']} | {d['dP_mw']:.4f} | " + f"{d['dQ_mvar']:.4f} | {d['mismatch_mva']:.4f} |" + ) + elif c.check_id == "acpf_generator_limits": + lines.append("| Bus | ID | P | Q | PT | PB | QT | QB | Violation |") + lines.append("|-----|----|----|----|----|----|----|----|----|") + for d in shown: + vtype = ( + ", ".join(d["violation_type"]) + if isinstance(d["violation_type"], list) + else d["violation_type"] + ) + lines.append( + f"| {d['bus']} | {d['machine_id']} | " + f"{d['P']:.2f} | {d['Q']:.2f} | " + f"{d['PT']:.2f} | {d['PB']:.2f} | " + f"{d['QT']:.2f} | {d['QB']:.2f} | {vtype} |" + ) + elif c.check_id == "dcpf_flow_angle_consistency": + lines.append("| From | To | Ckt | Stored (MW) | Recomputed (MW) | Deviation (MW) |") + lines.append("|------|-----|-----|-------------|-----------------|") + for d in shown: + lines.append( + f"| {d['from_bus']} | {d['to_bus']} | {d['ckt']} | " + f"{d['P_stored_mw']:.4f} | {d['P_recomputed_mw']:.4f} | " + f"{d['deviation_mw']:.4f} |" + ) + elif c.check_id == "acpf_vm_plausibility": + lines.append("| Bus | VM (p.u.) |") + lines.append("|-----|-----------|") + for d in shown: + lines.append(f"| {d['bus']} | {d['VM']:.6f} |") + else: + # Generic detail rendering + for d in shown: + lines.append(f"- {d}") + + if remaining > 0: + lines.append(f"\n*... and {remaining} more (total: {len(c.detail)})*") + + lines.append("") + + output_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +# --------------------------------------------------------------------------- +# Orchestrator +# --------------------------------------------------------------------------- + + +def run_validation( + acpf_dir: Path, + dcpf_dir: Path, + intermediate_dir: Path, + reference_dir: Path, + output_dir: Path, +) -> ValidationReport: + """Top-level orchestrator for reference solution validation. + + Steps: + 1. Load all input data (ACPF, DCPF, intermediate format, exclusion registry). + If a dataset is missing, mark its checks as SKIP with the reason. + 2. Run ACPF checks A-D (if ACPF data loaded successfully). + 3. Run DCPF checks A-C (if DCPF data loaded successfully). + 4. Assemble the ValidationReport. + 5. Write validation_report.json and validation_report.md to output_dir. + 6. Return the report for programmatic consumption. + + Args: + acpf_dir: Directory containing ACPF reference files. + dcpf_dir: Directory containing DCPF reference files. + intermediate_dir: Directory containing canonical parser CSV output. + reference_dir: Directory containing the bus exclusion registry. + output_dir: Directory for output files. + + Returns: + The assembled ValidationReport. + """ + # --- Load excluded buses (optional) --- + try: + excluded_buses = load_excluded_buses(reference_dir) + except FileNotFoundError: + excluded_buses = set() + + # --- ACPF data loading --- + acpf_buses: list[dict] = [] + acpf_branches: list[dict] = [] + acpf_generators: list[dict] = [] + acpf_summary: dict | None = None + acpf_ok = True + acpf_skip_reason = "" + + try: + acpf_buses = load_acpf_buses(acpf_dir) + acpf_branches = load_acpf_branches(acpf_dir) + acpf_generators = load_acpf_generators(acpf_dir) + acpf_summary = load_acpf_summary(acpf_dir) + except FileNotFoundError as e: + acpf_ok = False + acpf_skip_reason = str(e) + + # --- Intermediate data loading --- + intermediate_buses: list[dict] = [] + intermediate_generators: list[dict] = [] + intermediate_branches: list[dict] = [] + intermediate_ok = True + intermediate_skip_reason = "" + + try: + intermediate_buses = load_intermediate_buses(intermediate_dir) + except (FileNotFoundError, ValueError) as e: + intermediate_ok = False + intermediate_skip_reason = str(e) + + try: + intermediate_generators = load_intermediate_generators(intermediate_dir) + except (FileNotFoundError, ValueError): + # Generator limits not available -- generator-limit check will skip + intermediate_generators = [] + + try: + intermediate_branches = load_intermediate_branches(intermediate_dir) + except (FileNotFoundError, ValueError): + intermediate_branches = [] + + # --- DCPF data loading --- + dcpf_buses: list[dict] = [] + dcpf_branches: list[dict] = [] + dcpf_summary: dict | None = None + dcpf_ok = True + dcpf_skip_reason = "" + + try: + dcpf_buses = load_dcpf_buses(dcpf_dir) + dcpf_branches = load_dcpf_branches(dcpf_dir) + dcpf_summary = load_dcpf_summary(dcpf_dir) + except FileNotFoundError as e: + dcpf_ok = False + dcpf_skip_reason = str(e) + + # --- Run ACPF checks --- + acpf_checks: list[CheckResult] = [] + + if acpf_ok and acpf_summary is not None: + acpf_checks.append(check_acpf_power_balance(acpf_summary)) + else: + acpf_checks.append( + _make_skip_result( + "acpf_power_balance", + "ACPF System Power Balance", + acpf_skip_reason or "ACPF data not available", + ) + ) + + if acpf_ok and intermediate_ok: + acpf_checks.append( + check_acpf_kcl( + acpf_buses, + acpf_branches, + acpf_generators, + intermediate_buses, + excluded_buses, + ) + ) + else: + reason = acpf_skip_reason or intermediate_skip_reason or "Required data not available" + acpf_checks.append(_make_skip_result("acpf_kcl", "ACPF Per-Bus KCL", reason)) + + if acpf_ok: + acpf_checks.append(check_acpf_vm_plausibility(acpf_buses, excluded_buses)) + else: + acpf_checks.append( + _make_skip_result( + "acpf_vm_plausibility", + "ACPF Voltage Magnitude Plausibility", + acpf_skip_reason or "ACPF data not available", + ) + ) + + if acpf_ok and intermediate_generators and acpf_summary is not None: + acpf_checks.append( + check_acpf_generator_limits( + acpf_generators, + intermediate_generators, + acpf_summary, + ) + ) + else: + reason = acpf_skip_reason or "Generator limit columns not available in intermediate format" + acpf_checks.append( + _make_skip_result( + "acpf_generator_limits", + "ACPF Generator Output Within Limits", + reason, + ) + ) + + # --- Run DCPF checks --- + dcpf_checks: list[CheckResult] = [] + + if dcpf_ok and dcpf_summary is not None: + dcpf_checks.append(check_dcpf_power_balance(dcpf_summary)) + else: + dcpf_checks.append( + _make_skip_result( + "dcpf_power_balance", + "DCPF Power Balance (Lossless)", + dcpf_skip_reason or "DCPF data not available", + ) + ) + + if dcpf_ok and intermediate_branches and dcpf_summary is not None: + dcpf_checks.append( + check_dcpf_flow_angle_consistency( + dcpf_buses, + dcpf_branches, + intermediate_branches, + dcpf_summary, + ) + ) + else: + reason = dcpf_skip_reason or "Branch reactance data not available" + dcpf_checks.append( + _make_skip_result( + "dcpf_flow_angle_consistency", + "DCPF Flow-Angle Consistency", + reason, + ) + ) + + if dcpf_ok and dcpf_summary is not None: + dcpf_checks.append(check_dcpf_slack_angle(dcpf_buses, dcpf_summary)) + else: + dcpf_checks.append( + _make_skip_result( + "dcpf_slack_angle", + "DCPF Slack Angle Reference", + dcpf_skip_reason or "DCPF data not available", + ) + ) + + # --- Assemble report --- + report = build_validation_report( + acpf_checks, + dcpf_checks, + acpf_bus_count=len(acpf_buses), + acpf_branch_count=len(acpf_branches), + acpf_generator_count=len(acpf_generators), + dcpf_bus_count=len(dcpf_buses), + dcpf_branch_count=len(dcpf_branches), + ) + + # --- Write outputs --- + output_dir.mkdir(parents=True, exist_ok=True) + write_report_json(report, output_dir / "validation_report.json") + write_report_markdown(report, output_dir / "validation_report.md") + + return report + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> None: + """CLI entry point for reference solution validation. + + Usage:: + + python -m data.fnm.scripts.validation_report \\ + --acpf-dir data/fnm/reference/acpf/ \\ + --dcpf-dir data/fnm/reference/dcpf/ \\ + --intermediate-dir data/fnm/intermediate/canonical/ \\ + --reference-dir data/fnm/reference/ \\ + [-o data/fnm/reference/] + + Exit codes: + - 0: Report generated successfully (regardless of check outcomes). + - 1: Missing required input files that prevent any checks from running. + - 2: Malformed input data (e.g., unparseable CSV or JSON). + + Args: + argv: Command-line arguments. If None, reads from sys.argv[1:]. + """ + parser = argparse.ArgumentParser( + description="Validate ACPF and DCPF reference solutions for internal consistency." + ) + parser.add_argument( + "--acpf-dir", + type=Path, + required=True, + help="Directory containing ACPF reference files (buses_acpf.csv, etc.)", + ) + parser.add_argument( + "--dcpf-dir", + type=Path, + required=True, + help="Directory containing DCPF reference files (buses_dcpf.csv, etc.)", + ) + parser.add_argument( + "--intermediate-dir", + type=Path, + required=True, + help="Directory containing canonical parser CSV output.", + ) + parser.add_argument( + "--reference-dir", + type=Path, + required=True, + help="Directory containing the bus exclusion registry.", + ) + parser.add_argument( + "-o", + "--output-dir", + type=Path, + default=None, + help="Output directory (defaults to --reference-dir).", + ) + + args = parser.parse_args(argv) + output_dir = args.output_dir or args.reference_dir + + try: + report = run_validation( + acpf_dir=args.acpf_dir, + dcpf_dir=args.dcpf_dir, + intermediate_dir=args.intermediate_dir, + reference_dir=args.reference_dir, + output_dir=output_dir, + ) + except ValueError as exc: + print(f"ERROR: Malformed input data: {exc}", file=sys.stderr) + sys.exit(2) + + # Print summary + print(f"Validation report written to {output_dir}") + print( + f" {report.summary.passed} passed, " + f"{report.summary.failed} failed, " + f"{report.summary.skipped} skipped" + ) + if not report.all_passed: + sys.exit(0) # Self-check, not a gate -- always exit 0 + + +if __name__ == "__main__": + main() diff --git a/data/fnm/scripts/verify_materialization.py b/data/fnm/scripts/verify_materialization.py new file mode 100644 index 00000000..f2a72076 --- /dev/null +++ b/data/fnm/scripts/verify_materialization.py @@ -0,0 +1,557 @@ +"""Post-materialization verification for intermediate CSV export. + +Checks that the materialized CSV files and manifest.json produced by +export_intermediate_csvs.py are correct and complete: file counts, +record counts, column headers, and manifest internal consistency. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from dataclasses import dataclass, field +from pathlib import Path + +# --------------------------------------------------------------------------- +# Data structures +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class FileInventoryCheck: + expected_files: list[str] + found_files: list[str] + missing_files: list[str] + unexpected_files: list[str] + passed: bool + + +@dataclass(frozen=True) +class RecordCountCheck: + table_name: str + materialized_count: int + reference_count: int + reference_source: str + comparison: str # "==" or "<=" + passed: bool + message: str + + +@dataclass(frozen=True) +class ColumnHeaderCheck: + table_name: str + csv_columns: list[str] + schema_columns: list[str] + passed: bool + mismatches: list[str] + + +@dataclass(frozen=True) +class ManifestConsistencyCheck: + total_records_matches_sum: bool + total_tables_correct: bool + sbase_correct: bool + all_files_exist: bool + missing_manifest_files: list[str] + passed: bool + + +@dataclass +class MaterializationVerification: + file_inventory: FileInventoryCheck + record_count_checks: list[RecordCountCheck] + column_header_checks: list[ColumnHeaderCheck] + manifest_consistency: ManifestConsistencyCheck + all_passed: bool + errors: list[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Expected file inventory (17 CSVs + manifest.json) +# --------------------------------------------------------------------------- + +EXPECTED_FILES: list[str] = [ + "area.csv", + "branch.csv", + "bus.csv", + "facts.csv", + "fixed_shunt.csv", + "generator.csv", + "impedance_correction.csv", + "interarea_transfer.csv", + "load.csv", + "manifest.json", + "multi_section_line.csv", + "multi_terminal_dc.csv", + "owner.csv", + "switched_shunt.csv", + "transformer.csv", + "two_terminal_dc.csv", + "vsc_dc.csv", + "zone.csv", +] + + +# --------------------------------------------------------------------------- +# Utility functions +# --------------------------------------------------------------------------- + + +def count_csv_rows(csv_path: Path) -> int: + """Count data rows in a CSV file (excludes header).""" + with open(csv_path, newline="") as f: + reader = csv.reader(f) + next(reader, None) # skip header + return sum(1 for _ in reader) + + +def read_csv_header(csv_path: Path) -> list[str]: + """Read the header row of a CSV file.""" + with open(csv_path, newline="") as f: + reader = csv.reader(f) + header = next(reader, None) + return list(header) if header else [] + + +def get_schema_column_order(schema_path: Path) -> list[str]: + """Extract column names from a JSON Schema file's properties dict.""" + with open(schema_path) as f: + schema = json.load(f) + return list(schema.get("properties", {}).keys()) + + +# --------------------------------------------------------------------------- +# Verification functions +# --------------------------------------------------------------------------- + + +def verify_file_inventory(output_dir: Path) -> FileInventoryCheck: + """Check that exactly the expected 18 files are present.""" + found = sorted(p.name for p in output_dir.iterdir() if p.is_file()) + expected = sorted(EXPECTED_FILES) + missing = sorted(set(expected) - set(found)) + unexpected = sorted(set(found) - set(expected)) + passed = not missing and not unexpected + return FileInventoryCheck( + expected_files=expected, + found_files=found, + missing_files=missing, + unexpected_files=unexpected, + passed=passed, + ) + + +def verify_record_counts( + output_dir: Path, + cleaning_summary_path: Path, + intermediate_manifest_path: Path | None, +) -> list[RecordCountCheck]: + """Cross-validate CSV row counts against reference sources.""" + checks: list[RecordCountCheck] = [] + + # Load cleaning summary + with open(cleaning_summary_path) as f: + cleaning = json.load(f) + cleaned = cleaning["cleaned_network"] + + # Load intermediate manifest (the one we just generated) + manifest_path = output_dir / "manifest.json" + with open(manifest_path) as f: + manifest = json.load(f) + + # --- bus count matches cleaning summary exactly --- + bus_csv = output_dir / "bus.csv" + bus_rows = count_csv_rows(bus_csv) + ref_buses = cleaned["buses"] + checks.append( + RecordCountCheck( + table_name="bus", + materialized_count=bus_rows, + reference_count=ref_buses, + reference_source="summary_cleaning.json:cleaned_network.buses", + comparison="==", + passed=bus_rows == ref_buses, + message=f"bus.csv has {bus_rows} rows, expected {ref_buses}", + ) + ) + + # --- branch + transformer <= branches_total --- + branch_rows = count_csv_rows(output_dir / "branch.csv") + xfmr_rows = count_csv_rows(output_dir / "transformer.csv") + ref_branches = cleaned["branches_total"] + branch_sum = branch_rows + xfmr_rows + checks.append( + RecordCountCheck( + table_name="branch+transformer", + materialized_count=branch_sum, + reference_count=ref_branches, + reference_source="summary_cleaning.json:cleaned_network.branches_total", + comparison="<=", + passed=branch_sum <= ref_branches, + message=( + f"branch({branch_rows}) + transformer({xfmr_rows}) = {branch_sum}, " + f"bound {ref_branches}" + ), + ) + ) + + # --- generator count within bound --- + gen_rows = count_csv_rows(output_dir / "generator.csv") + ref_gens = cleaned["generators_total"] + checks.append( + RecordCountCheck( + table_name="generator", + materialized_count=gen_rows, + reference_count=ref_gens, + reference_source="summary_cleaning.json:cleaned_network.generators_total", + comparison="<=", + passed=0 < gen_rows <= ref_gens, + message=f"generator.csv has {gen_rows} rows, bound {ref_gens}", + ) + ) + + # --- load count within bound --- + load_rows = count_csv_rows(output_dir / "load.csv") + ref_loads = 16000 # Upper bound (rounded) + checks.append( + RecordCountCheck( + table_name="load", + materialized_count=load_rows, + reference_count=ref_loads, + reference_source="PRD upper bound", + comparison="<=", + passed=0 < load_rows <= ref_loads, + message=f"load.csv has {load_rows} rows, bound {ref_loads}", + ) + ) + + # --- area count within bound --- + area_rows = count_csv_rows(output_dir / "area.csv") + checks.append( + RecordCountCheck( + table_name="area", + materialized_count=area_rows, + reference_count=49, + reference_source="PRD upper bound", + comparison="<=", + passed=area_rows <= 49, + message=f"area.csv has {area_rows} rows, bound 49", + ) + ) + + # --- zone count within bound --- + zone_rows = count_csv_rows(output_dir / "zone.csv") + checks.append( + RecordCountCheck( + table_name="zone", + materialized_count=zone_rows, + reference_count=90, + reference_source="PRD upper bound", + comparison="<=", + passed=zone_rows <= 90, + message=f"zone.csv has {zone_rows} rows, bound 90", + ) + ) + + # --- Cross-validate against intermediate manifest --- + # Each CSV row count should match the manifest's record_count + for table_entry in manifest["tables"]: + tname = table_entry["table_name"] + fname = table_entry["file_name"] + csv_path = output_dir / fname + if csv_path.exists(): + csv_rows = count_csv_rows(csv_path) + manifest_rc = table_entry["record_count"] + checks.append( + RecordCountCheck( + table_name=f"{tname}_vs_manifest", + materialized_count=csv_rows, + reference_count=manifest_rc, + reference_source="intermediate manifest.json", + comparison="==", + passed=csv_rows == manifest_rc, + message=(f"{tname}: CSV has {csv_rows} rows, manifest says {manifest_rc}"), + ) + ) + + # --- Empty tables should have 0 data rows --- + non_empty_types = set(manifest.get("non_empty_record_types", [])) + table_name_to_record_type = {t["table_name"]: t["record_type"] for t in manifest["tables"]} + empty_tables = [ + tname for tname, rtype in table_name_to_record_type.items() if rtype not in non_empty_types + ] + for tname in empty_tables: + fname = f"{tname}.csv" + csv_path = output_dir / fname + if csv_path.exists(): + rows = count_csv_rows(csv_path) + checks.append( + RecordCountCheck( + table_name=f"{tname}_empty", + materialized_count=rows, + reference_count=0, + reference_source="manifest non_empty_record_types", + comparison="==", + passed=rows == 0, + message=f"{tname} expected empty, has {rows} rows", + ) + ) + + return checks + + +def verify_column_headers(output_dir: Path, schema_dir: Path) -> list[ColumnHeaderCheck]: + """Verify that CSV column headers match JSON Schema property order.""" + checks: list[ColumnHeaderCheck] = [] + + manifest_path = output_dir / "manifest.json" + with open(manifest_path) as f: + manifest = json.load(f) + + for table_entry in manifest["tables"]: + tname = table_entry["table_name"] + fname = table_entry["file_name"] + schema_file = table_entry.get("schema_file", f"{tname}.schema.json") + + csv_path = output_dir / fname + schema_path = schema_dir / schema_file + + if not csv_path.exists() or not schema_path.exists(): + checks.append( + ColumnHeaderCheck( + table_name=tname, + csv_columns=[], + schema_columns=[], + passed=False, + mismatches=[ + f"File not found: csv={csv_path.exists()}, schema={schema_path.exists()}" + ], + ) + ) + continue + + csv_cols = read_csv_header(csv_path) + schema_cols = get_schema_column_order(schema_path) + + mismatches: list[str] = [] + if csv_cols != schema_cols: + # Detail the differences + if len(csv_cols) != len(schema_cols): + mismatches.append( + f"Column count differs: CSV={len(csv_cols)}, schema={len(schema_cols)}" + ) + for i, (c, s) in enumerate(zip(csv_cols, schema_cols)): + if c != s: + mismatches.append(f"Position {i}: CSV='{c}', schema='{s}'") + # Extra columns + if len(csv_cols) > len(schema_cols): + extras = csv_cols[len(schema_cols) :] + mismatches.append(f"Extra CSV columns: {extras}") + elif len(schema_cols) > len(csv_cols): + extras = schema_cols[len(csv_cols) :] + mismatches.append(f"Missing CSV columns: {extras}") + + checks.append( + ColumnHeaderCheck( + table_name=tname, + csv_columns=csv_cols, + schema_columns=schema_cols, + passed=len(mismatches) == 0, + mismatches=mismatches, + ) + ) + + return checks + + +def verify_manifest_consistency(output_dir: Path) -> ManifestConsistencyCheck: + """Verify the manifest.json is internally consistent.""" + manifest_path = output_dir / "manifest.json" + with open(manifest_path) as f: + manifest = json.load(f) + + tables = manifest["tables"] + + # total_records == sum of per-table record_count + record_sum = sum(t["record_count"] for t in tables) + total_records_ok = manifest["total_records"] == record_sum + + # total_tables == 17 + total_tables_ok = manifest["total_tables"] == 17 + + # sbase == 100.0 + sbase_ok = manifest["sbase"] == 100.0 + + # All referenced files exist + missing_files: list[str] = [] + for t in tables: + fpath = output_dir / t["file_name"] + if not fpath.exists(): + missing_files.append(t["file_name"]) + all_files_ok = len(missing_files) == 0 + + passed = total_records_ok and total_tables_ok and sbase_ok and all_files_ok + + return ManifestConsistencyCheck( + total_records_matches_sum=total_records_ok, + total_tables_correct=total_tables_ok, + sbase_correct=sbase_ok, + all_files_exist=all_files_ok, + missing_manifest_files=missing_files, + passed=passed, + ) + + +def run_materialization_verification( + output_dir: Path, + cleaning_summary_path: Path, + intermediate_manifest_path: Path | None, + schema_dir: Path, +) -> MaterializationVerification: + """Run all verification checks and return a summary.""" + errors: list[str] = [] + + # File inventory + try: + file_inv = verify_file_inventory(output_dir) + except Exception as e: + errors.append(f"File inventory check failed: {e}") + file_inv = FileInventoryCheck( + expected_files=EXPECTED_FILES, + found_files=[], + missing_files=EXPECTED_FILES, + unexpected_files=[], + passed=False, + ) + + # Record counts + try: + record_checks = verify_record_counts( + output_dir, cleaning_summary_path, intermediate_manifest_path + ) + except Exception as e: + errors.append(f"Record count check failed: {e}") + record_checks = [] + + # Column headers + try: + header_checks = verify_column_headers(output_dir, schema_dir) + except Exception as e: + errors.append(f"Column header check failed: {e}") + header_checks = [] + + # Manifest consistency + try: + manifest_check = verify_manifest_consistency(output_dir) + except Exception as e: + errors.append(f"Manifest consistency check failed: {e}") + manifest_check = ManifestConsistencyCheck( + total_records_matches_sum=False, + total_tables_correct=False, + sbase_correct=False, + all_files_exist=False, + missing_manifest_files=[], + passed=False, + ) + + all_passed = ( + file_inv.passed + and all(c.passed for c in record_checks) + and all(c.passed for c in header_checks) + and manifest_check.passed + and not errors + ) + + return MaterializationVerification( + file_inventory=file_inv, + record_count_checks=record_checks, + column_header_checks=header_checks, + manifest_consistency=manifest_check, + all_passed=all_passed, + errors=errors, + ) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> None: + """Run verification checks and print results.""" + parser = argparse.ArgumentParser(description="Verify materialized intermediate CSV artifacts.") + parser.add_argument( + "--output-dir", + type=Path, + required=True, + help="Directory containing materialized CSVs and manifest.json", + ) + parser.add_argument( + "--cleaning-summary", + type=Path, + required=True, + help="Path to summary_cleaning.json", + ) + parser.add_argument( + "--schema-dir", + type=Path, + required=True, + help="Directory containing JSON Schema files", + ) + parser.add_argument( + "--intermediate-manifest", + type=Path, + default=None, + help="Path to pre-filter intermediate_manifest.json (optional)", + ) + args = parser.parse_args(argv) + + result = run_materialization_verification( + output_dir=args.output_dir, + cleaning_summary_path=args.cleaning_summary, + intermediate_manifest_path=args.intermediate_manifest, + schema_dir=args.schema_dir, + ) + + # Print summary + print("=" * 60) + print("MATERIALIZATION VERIFICATION REPORT") + print("=" * 60) + + print(f"\nFile Inventory: {'PASS' if result.file_inventory.passed else 'FAIL'}") + if result.file_inventory.missing_files: + print(f" Missing: {result.file_inventory.missing_files}") + if result.file_inventory.unexpected_files: + print(f" Unexpected: {result.file_inventory.unexpected_files}") + + print(f"\nRecord Count Checks ({len(result.record_count_checks)}):") + for c in result.record_count_checks: + status = "PASS" if c.passed else "FAIL" + print(f" [{status}] {c.message}") + + print(f"\nColumn Header Checks ({len(result.column_header_checks)}):") + for c in result.column_header_checks: + status = "PASS" if c.passed else "FAIL" + print(f" [{status}] {c.table_name}") + if c.mismatches: + for m in c.mismatches: + print(f" - {m}") + + print(f"\nManifest Consistency: {'PASS' if result.manifest_consistency.passed else 'FAIL'}") + print(f" total_records sum: {result.manifest_consistency.total_records_matches_sum}") + print(f" total_tables == 17: {result.manifest_consistency.total_tables_correct}") + print(f" sbase == 100: {result.manifest_consistency.sbase_correct}") + print(f" all files exist: {result.manifest_consistency.all_files_exist}") + + if result.errors: + print(f"\nErrors: {result.errors}") + + print(f"\nOVERALL: {'PASS' if result.all_passed else 'FAIL'}") + + sys.exit(0 if result.all_passed else 1) + + +if __name__ == "__main__": + main() diff --git a/data/fnm/tests/__init__.py b/data/fnm/tests/__init__.py new file mode 100644 index 00000000..9d48db4f --- /dev/null +++ b/data/fnm/tests/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/data/fnm/tests/conftest.py b/data/fnm/tests/conftest.py new file mode 100644 index 00000000..e17038e1 --- /dev/null +++ b/data/fnm/tests/conftest.py @@ -0,0 +1,43 @@ +"""Conftest providing FNM gating fixtures for this test directory.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from fnm.scripts.fnm_gating_fixtures import ( # noqa: F401 + require_fnm, + require_fnm_csvs, + require_fnm_raw, +) + + +def pytest_configure(config: pytest.Config) -> None: + """Register custom markers.""" + config.addinivalue_line("markers", "fnm: tests requiring FNM data (FNM_PATH env var)") + config.addinivalue_line("markers", "octave: tests requiring Octave and MATPOWER installation") + config.addinivalue_line( + "markers", "gridcal: tests requiring VeraGridEngine (GridCal) installed" + ) + config.addinivalue_line("markers", "docs: tests validating documentation artifacts") + + +@pytest.fixture +def require_gridcal() -> dict: + """Skip the test if VeraGridEngine (GridCal) is not installed. + + Returns a dict with useful paths for GridCal integration tests. + """ + try: + import VeraGridEngine # noqa: F401 + except ImportError: + pytest.skip("VeraGridEngine (GridCal) not installed") + + # Locate case39.m for integration tests + repo_root = Path(__file__).resolve().parent.parent.parent.parent + case39_path = repo_root / "data" / "networks" / "case39.m" + + return { + "case39_path": case39_path, + } diff --git a/data/fnm/tests/test_acpf_reference.py b/data/fnm/tests/test_acpf_reference.py new file mode 100644 index 00000000..069a5821 --- /dev/null +++ b/data/fnm/tests/test_acpf_reference.py @@ -0,0 +1,890 @@ +"""Tests for ACPF Reference Solution Extraction (PRD 03/02). + +Tests T01-T14 are synthetic (no FNM data required). +Tests T15-T16 require FNM_PATH and D6/D8 outputs. +""" + +from __future__ import annotations + +import csv +import json +import random +from pathlib import Path + +import pytest + +from fnm.scripts.acpf_reference import ( + ConvergenceInfo, + SolutionSource, + SolverSettings, + SystemSummary, + build_acpf_reference, + compute_system_summary, + determine_solution_source, + extract_bus_results, + write_branches_csv, + write_buses_csv, + write_generators_csv, + write_summary_json, +) + +# --------------------------------------------------------------------------- +# Helpers for synthetic CSV generation +# --------------------------------------------------------------------------- + + +def _write_bus_csv( + path: Path, + *, + n_buses: int = 50, + n_isolated: int = 3, + n_deenergized: int = 1, + with_header: bool = True, + vm_base: float = 1.0, + vm_spread: float = 0.05, + va_spread: float = 15.0, + seed: int = 42, +) -> list[dict]: + """Create a synthetic bus CSV and return the expected non-excluded rows.""" + rng = random.Random(seed) + rows = [] + expected = [] + + for i in range(1, n_buses + 1): + bus_num = i * 10 + if i <= n_isolated: + bus_type = 4 # isolated + elif i == n_isolated + 1: + bus_type = 3 # slack + else: + bus_type = 1 # PQ + + if i == n_buses and n_deenergized > 0: + vm = 0.0 + va = 0.0 + else: + vm = vm_base + rng.uniform(-vm_spread, vm_spread) + va = rng.uniform(-va_spread, va_spread) + + pd = rng.uniform(0, 50) + qd = rng.uniform(-10, 20) + + if with_header: + rows.append( + { + "bus_i": str(bus_num), + "type": str(bus_type), + "Pd": f"{pd:.4f}", + "Qd": f"{qd:.4f}", + "Gs": "0", + "Bs": "0", + "area": "1", + "Vm": f"{vm:.8f}", + "Va": f"{va:.6f}", + "baseKV": "138", + "zone": "1", + "Vmax": "1.1", + "Vmin": "0.9", + } + ) + else: + rows.append( + [ + str(bus_num), + str(bus_type), + f"{pd:.4f}", + f"{qd:.4f}", + "0", + "0", + "1", + f"{vm:.8f}", + f"{va:.6f}", + "138", + "1", + "1.1", + "0.9", + ] + ) + + if bus_type != 4 and vm != 0.0: + expected.append({"bus": bus_num, "VM": vm, "VA": va}) + + with open(path, "w", newline="", encoding="utf-8") as f: + if with_header: + writer = csv.DictWriter(f, fieldnames=list(rows[0].keys())) + writer.writeheader() + writer.writerows(rows) + else: + writer = csv.writer(f) + writer.writerows(rows) + + expected.sort(key=lambda r: r["bus"]) + return expected + + +def _write_branch_csv( + path: Path, + *, + n_branches: int = 30, + with_header: bool = True, + seed: int = 42, +) -> list[dict]: + """Create a synthetic branch CSV with P/Q flow values and return expected rows.""" + rng = random.Random(seed) + rows = [] + expected = [] + + for i in range(1, n_branches + 1): + from_bus = i * 10 + to_bus = (i + 1) * 10 + ckt = "1" + status = 1 + # Realistic flow: P_from positive, P_to negative, losses = P_from + P_to > 0 + p_from = rng.uniform(10, 200) + losses = rng.uniform(0.1, 5.0) + p_to = -(p_from - losses) + q_from = rng.uniform(-50, 50) + q_to = rng.uniform(-50, 50) + + if with_header: + rows.append( + { + "fbus": str(from_bus), + "tbus": str(to_bus), + "ckt": ckt, + "r": "0.01", + "x": "0.1", + "b": "0.02", + "rateA": "100", + "rateB": "100", + "rateC": "100", + "ratio": "0", + "status": str(status), + "angmin": "-360", + "angmax": "360", + "Pf": f"{p_from:.4f}", + "Qf": f"{q_from:.4f}", + "Pt": f"{p_to:.4f}", + "Qt": f"{q_to:.4f}", + } + ) + else: + rows.append( + [ + str(from_bus), + str(to_bus), + "0.01", + "0.1", + "0.02", + "100", + "100", + "100", + "0", + "0", + str(status), + "-360", + "360", + f"{p_from:.4f}", + f"{q_from:.4f}", + f"{p_to:.4f}", + f"{q_to:.4f}", + ] + ) + + expected.append( + { + "from_bus": from_bus, + "to_bus": to_bus, + "ckt": ckt if with_header else "1", + "P_from": p_from, + "Q_from": q_from, + "P_to": p_to, + "Q_to": q_to, + } + ) + + with open(path, "w", newline="", encoding="utf-8") as f: + if with_header: + writer = csv.DictWriter(f, fieldnames=list(rows[0].keys())) + writer.writeheader() + writer.writerows(rows) + else: + writer = csv.writer(f) + writer.writerows(rows) + + expected.sort(key=lambda r: (r["from_bus"], r["to_bus"], r["ckt"])) + return expected + + +def _write_gen_csv( + path: Path, + *, + n_generators: int = 15, + with_header: bool = True, + seed: int = 42, +) -> list[dict]: + """Create a synthetic generator CSV and return expected rows.""" + rng = random.Random(seed) + rows = [] + expected = [] + + for i in range(1, n_generators + 1): + bus_num = (i + 3) * 10 # start after isolated buses + machine_id = str(i % 3 + 1) + pg = rng.uniform(10, 500) + qg = rng.uniform(-100, 200) + status = 1 + + if with_header: + rows.append( + { + "bus": str(bus_num), + "machine_id": machine_id, + "Pg": f"{pg:.4f}", + "Qg": f"{qg:.4f}", + "Qmax": "999", + "Qmin": "-999", + "Vg": "1.0", + "status": str(status), + "Pmax": "999", + "Pmin": "0", + } + ) + else: + rows.append( + [ + str(bus_num), + f"{pg:.4f}", + f"{qg:.4f}", + "999", + "-999", + "1.0", + "100", + str(status), + "999", + "0", + ] + ) + + expected.append( + { + "bus": bus_num, + "machine_id": machine_id if with_header else str(1), + "P": pg, + "Q": qg, + } + ) + + with open(path, "w", newline="", encoding="utf-8") as f: + if with_header: + writer = csv.DictWriter(f, fieldnames=list(rows[0].keys())) + writer.writeheader() + writer.writerows(rows) + else: + writer = csv.writer(f) + writer.writerows(rows) + + expected.sort(key=lambda r: (r["bus"], r["machine_id"])) + return expected + + +def _write_snapshot_json(path: Path, classification: str = "solved") -> None: + """Write a minimal D8 snapshot confirmation JSON.""" + data = {"classification": classification} + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2), encoding="utf-8") + + +# =========================================================================== +# T01-T03: Path selection +# =========================================================================== + + +def test_determine_source_solved() -> None: + """T01: determine_solution_source('solved') returns EXTRACTED.""" + result = determine_solution_source("solved") + assert result == SolutionSource.EXTRACTED + + +def test_determine_source_flat_start() -> None: + """T02: determine_solution_source('flat_start') returns COMPUTED.""" + result = determine_solution_source("flat_start") + assert result == SolutionSource.COMPUTED + + +def test_determine_source_indeterminate_raises() -> None: + """T03: determine_solution_source('indeterminate') raises ValueError.""" + with pytest.raises(ValueError, match="indeterminate"): + determine_solution_source("indeterminate") + + +# =========================================================================== +# T04-T05: Bus extraction +# =========================================================================== + + +def test_extract_bus_results_excludes_isolated(tmp_path: Path) -> None: + """T04: extract_bus_results excludes type=4 and VM=0 buses.""" + bus_csv = tmp_path / "bus.csv" + expected = _write_bus_csv(bus_csv, n_buses=50, n_isolated=3, n_deenergized=1, with_header=True) + + result = extract_bus_results(bus_csv) + + # Should have 50 - 3 (isolated) - 1 (VM=0) = 46 buses + assert len(result) == 46 + assert len(result) == len(expected) + + # No isolated or deenergized buses in output + result_bus_nums = {r["bus"] for r in result} + # buses 10, 20, 30 are type=4 (isolated), bus 500 has VM=0 + assert 10 not in result_bus_nums + assert 20 not in result_bus_nums + assert 30 not in result_bus_nums + assert 500 not in result_bus_nums + + # Sorted ascending + bus_nums = [r["bus"] for r in result] + assert bus_nums == sorted(bus_nums) + + +def test_extract_bus_results_precision(tmp_path: Path) -> None: + """T05: VM values preserve at least 6 decimal places.""" + bus_csv = tmp_path / "bus.csv" + # Create a CSV with precise VM values + with open(bus_csv, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow( + [ + "bus_i", + "type", + "Pd", + "Qd", + "Gs", + "Bs", + "area", + "Vm", + "Va", + "baseKV", + "zone", + "Vmax", + "Vmin", + ] + ) + writer.writerow( + [ + "1", + "1", + "10", + "5", + "0", + "0", + "1", + "1.01234567", + "-3.456789", + "138", + "1", + "1.1", + "0.9", + ] + ) + writer.writerow( + [ + "2", + "3", + "20", + "10", + "0", + "0", + "1", + "0.98765432", + "2.123456", + "138", + "1", + "1.1", + "0.9", + ] + ) + + result = extract_bus_results(bus_csv) + assert len(result) == 2 + + # Check that VM precision is maintained to at least 6 decimal places + for r in result: + if r["bus"] == 1: + assert abs(r["VM"] - 1.01234567) < 1e-6 + elif r["bus"] == 2: + assert abs(r["VM"] - 0.98765432) < 1e-6 + + +# =========================================================================== +# T06-T07: Data structures +# =========================================================================== + + +def test_solver_settings_defaults() -> None: + """T06: SolverSettings fields set correctly, enforce_area_interchange defaults to None.""" + settings = SolverSettings( + name="runpf", + tolerance=1e-8, + max_iterations=100, + q_limits_enforced=True, + q_limit_strategy="two_stage_relaxed_then_enforced", + ) + assert settings.name == "runpf" + assert settings.tolerance == 1e-8 + assert settings.max_iterations == 100 + assert settings.q_limits_enforced is True + assert settings.q_limit_strategy == "two_stage_relaxed_then_enforced" + assert settings.enforce_area_interchange is None + + +def test_convergence_info_null_for_extracted() -> None: + """T07: ConvergenceInfo() with no args has all None fields.""" + info = ConvergenceInfo() + assert info.converged is None + assert info.iterations is None + assert info.final_mismatch_mw is None + assert info.final_mismatch_mvar is None + + +# =========================================================================== +# T08-T10: Output CSV format +# =========================================================================== + + +def test_write_buses_csv_schema(tmp_path: Path) -> None: + """T08: buses_acpf.csv has correct schema and precision.""" + rng = random.Random(42) + bus_results = [ + {"bus": i, "VM": 0.95 + rng.uniform(0, 0.1), "VA": rng.uniform(-15, 10)} + for i in range(1, 21) + ] + + output = tmp_path / "buses_acpf.csv" + write_buses_csv(bus_results, output) + + # Read back and verify + with open(output, encoding="utf-8") as f: + raw_text = f.read() + + lines = raw_text.strip().split("\n") + assert lines[0] == "bus,VM,VA" + assert len(lines) == 21 # header + 20 data rows + + # Verify bus column is integer + with open(output, encoding="utf-8") as f: + reader = csv.DictReader(f) + rows = list(reader) + assert len(rows) == 20 + for row in rows: + int(row["bus"]) # should not raise + + # Check VM has at least 6 decimal places in raw text + for line in lines[1:]: + vm_str = line.split(",")[1] + decimal_part = vm_str.split(".")[1] + assert len(decimal_part) >= 6, f"VM '{vm_str}' has fewer than 6 decimal places" + + +def test_write_branches_csv_schema(tmp_path: Path) -> None: + """T09: branches_acpf.csv has correct schema and positive losses.""" + rng = random.Random(42) + branch_results = [] + for i in range(1, 31): + p_from = rng.uniform(10, 200) + losses = rng.uniform(0.1, 5.0) + p_to = -(p_from - losses) + branch_results.append( + { + "from_bus": i * 10, + "to_bus": (i + 1) * 10, + "ckt": "1", + "P_from": p_from, + "Q_from": rng.uniform(-50, 50), + "P_to": p_to, + "Q_to": rng.uniform(-50, 50), + } + ) + + output = tmp_path / "branches_acpf.csv" + write_branches_csv(branch_results, output) + + with open(output, encoding="utf-8") as f: + reader = csv.DictReader(f) + rows = list(reader) + + assert len(rows) == 30 + + # Verify header + with open(output, encoding="utf-8") as f: + header = f.readline().strip() + assert header == "from_bus,to_bus,ckt,P_from,Q_from,P_to,Q_to" + + # Verify positive losses (P_from + P_to > 0) for all branches + for row in rows: + p_from = float(row["P_from"]) + p_to = float(row["P_to"]) + assert p_from + p_to > 0, f"Negative losses: P_from={p_from}, P_to={p_to}" + + +def test_write_generators_csv_schema(tmp_path: Path) -> None: + """T10: generators_acpf.csv has correct schema.""" + rng = random.Random(42) + gen_results = [ + { + "bus": (i + 3) * 10, + "machine_id": str(i % 3 + 1), + "P": rng.uniform(10, 500), + "Q": rng.uniform(-100, 200), + } + for i in range(1, 16) + ] + + output = tmp_path / "generators_acpf.csv" + write_generators_csv(gen_results, output) + + with open(output, encoding="utf-8") as f: + header = f.readline().strip() + assert header == "bus,machine_id,P,Q" + + with open(output, encoding="utf-8") as f: + reader = csv.DictReader(f) + rows = list(reader) + assert len(rows) == 15 + + +# =========================================================================== +# T11-T12: Summary JSON +# =========================================================================== + + +def test_write_summary_json_extracted_path(tmp_path: Path) -> None: + """T11: summary_acpf.json for extracted path has correct structure.""" + summary = SystemSummary( + total_gen_mw=50000.0, + total_gen_mvar=12000.0, + total_load_mw=48000.0, + total_load_mvar=11000.0, + total_loss_mw=2000.0, + total_loss_mvar=1000.0, + slack_bus=100, + power_balance_residual_mw=0.0, + ) + counts = { + "buses_total": 30000, + "buses_excluded_isolated": 50, + "buses_excluded_deenergized": 10, + "buses_in_output": 29940, + "branches_in_output": 35000, + "generators_in_output": 2000, + } + + output = tmp_path / "summary_acpf.json" + write_summary_json( + source=SolutionSource.EXTRACTED, + classification="solved", + canonical_parser="matpower", + settings=SolverSettings(), + convergence=None, + summary=summary, + counts=counts, + warnings=[], + output_path=output, + ) + + data = json.loads(output.read_text(encoding="utf-8")) + + assert data["solution_source"] == "extracted" + assert data["solver"]["name"] is None + assert data["solver"]["convergence"]["converged"] is None + assert isinstance(data["system_summary"]["total_gen_mw"], float) + assert isinstance(data["system_summary"]["total_gen_mvar"], float) + assert isinstance(data["system_summary"]["total_load_mw"], float) + assert isinstance(data["system_summary"]["total_load_mvar"], float) + assert isinstance(data["system_summary"]["total_loss_mw"], float) + assert isinstance(data["system_summary"]["total_loss_mvar"], float) + assert isinstance(data["system_summary"]["slack_bus"], int) + assert isinstance(data["system_summary"]["power_balance_residual_mw"], float) + + # Verify timestamp is a valid ISO 8601 string + from datetime import datetime + + datetime.fromisoformat(data["timestamp"]) + + +def test_write_summary_json_computed_path(tmp_path: Path) -> None: + """T12: summary_acpf.json for computed path has solver fields populated.""" + summary = SystemSummary( + total_gen_mw=50000.0, + total_gen_mvar=12000.0, + total_load_mw=48000.0, + total_load_mvar=11000.0, + total_loss_mw=2000.0, + total_loss_mvar=1000.0, + slack_bus=100, + power_balance_residual_mw=0.0, + ) + counts = { + "buses_total": 30000, + "buses_excluded_isolated": 50, + "buses_excluded_deenergized": 10, + "buses_in_output": 29940, + "branches_in_output": 35000, + "generators_in_output": 2000, + } + settings = SolverSettings( + name="runpf", + version="8.0", + tolerance=1e-8, + max_iterations=100, + q_limits_enforced=True, + q_limit_strategy="two_stage_relaxed_then_enforced", + enforce_area_interchange=False, + ) + convergence = ConvergenceInfo( + converged=True, + iterations=12, + final_mismatch_mw=0.0001, + final_mismatch_mvar=0.0002, + ) + + output = tmp_path / "summary_acpf.json" + write_summary_json( + source=SolutionSource.COMPUTED, + classification="flat_start", + canonical_parser="matpower", + settings=settings, + convergence=convergence, + summary=summary, + counts=counts, + warnings=[], + output_path=output, + ) + + data = json.loads(output.read_text(encoding="utf-8")) + + assert data["solution_source"] == "computed" + assert data["solver"]["name"] == "runpf" + assert data["solver"]["settings"]["tolerance"] == 1e-8 + assert data["solver"]["convergence"]["converged"] is True + assert data["solver"]["convergence"]["iterations"] == 12 + assert isinstance(data["solver"]["convergence"]["iterations"], int) + assert data["solver"]["convergence"]["iterations"] > 0 + + +# =========================================================================== +# T13-T14: System summary +# =========================================================================== + + +def _create_balanced_test_data( + tmp_path: Path, + total_gen_mw: float = 1000.0, + total_load_mw: float = 950.0, + total_loss_mw: float = 50.0, + n_branches: int = 10, + n_generators: int = 5, +) -> tuple[list[dict], list[dict], list[dict], Path]: + """Create test data with specific power balance.""" + # Bus CSV with load and a slack bus + bus_csv = tmp_path / "bus.csv" + with open(bus_csv, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow( + [ + "bus_i", + "type", + "Pd", + "Qd", + "Gs", + "Bs", + "area", + "Vm", + "Va", + "baseKV", + "zone", + "Vmax", + "Vmin", + ] + ) + load_per_bus = total_load_mw / 10 + for i in range(1, 11): + bus_type = 3 if i == 1 else 1 + writer.writerow( + [ + str(i), + str(bus_type), + f"{load_per_bus:.4f}", + "10.0", + "0", + "0", + "1", + "1.0", + "0.0", + "138", + "1", + "1.1", + "0.9", + ] + ) + + # Bus results + bus_results = [{"bus": i, "VM": 1.0, "VA": 0.0} for i in range(1, 11)] + + # Generator results + gen_per = total_gen_mw / n_generators + gen_results = [ + {"bus": 1, "machine_id": str(i), "P": gen_per, "Q": 50.0} + for i in range(1, n_generators + 1) + ] + + # Branch results with specified total losses + loss_per_branch = total_loss_mw / n_branches + branch_results = [] + for i in range(1, n_branches + 1): + p_from = 100.0 + p_to = -(100.0 - loss_per_branch) + branch_results.append( + { + "from_bus": i, + "to_bus": i + 1 if i < 10 else 1, + "ckt": "1", + "P_from": p_from, + "Q_from": 10.0, + "P_to": p_to, + "Q_to": -8.0, + } + ) + + return bus_results, branch_results, gen_results, bus_csv + + +def test_compute_system_summary_balanced(tmp_path: Path) -> None: + """T13: System summary with balanced power.""" + bus_results, branch_results, gen_results, bus_csv = _create_balanced_test_data( + tmp_path, + total_gen_mw=1000.0, + total_load_mw=950.0, + total_loss_mw=50.0, + ) + + summary = compute_system_summary(bus_results, branch_results, gen_results, bus_csv) + + assert abs(summary.total_loss_mw - 50.0) < 0.1 + assert abs(summary.power_balance_residual_mw) < 0.1 + + +def test_compute_system_summary_warns_on_imbalance(tmp_path: Path) -> None: + """T14: System summary correctly computes residual for imbalanced data.""" + bus_results, branch_results, gen_results, bus_csv = _create_balanced_test_data( + tmp_path, + total_gen_mw=1000.0, + total_load_mw=945.0, + total_loss_mw=50.0, + ) + + summary = compute_system_summary(bus_results, branch_results, gen_results, bus_csv) + + # residual = gen - load - losses = 1000 - 945 - 50 = 5 + assert abs(summary.power_balance_residual_mw - 5.0) < 0.1 + + +# =========================================================================== +# T15-T16: Integration tests (require FNM_PATH and D6/D8 outputs) +# =========================================================================== + + +@pytest.mark.fnm +def test_fnm_acpf_reference_produces_all_files(tmp_path: Path, require_fnm: object) -> None: + """T15: build_acpf_reference produces all four output files with real FNM data.""" + intermediate_dir = Path("data/fnm/intermediate/canonical") + snapshot_json = Path("data/fnm/intermediate/snapshot/snapshot_confirmation.json") + + if not intermediate_dir.is_dir() or not snapshot_json.exists(): + pytest.skip("D6/D8 intermediate outputs not available") + + output_dir = tmp_path / "acpf" + result_dir = build_acpf_reference( + intermediate_dir=intermediate_dir, + snapshot_json_path=snapshot_json, + canonical_parser="matpower", + output_dir=output_dir, + ) + + # All four output files must exist + assert (result_dir / "buses_acpf.csv").exists() + assert (result_dir / "branches_acpf.csv").exists() + assert (result_dir / "generators_acpf.csv").exists() + assert (result_dir / "summary_acpf.json").exists() + + # Bus CSV should have >20,000 rows for a 30K network + with open(result_dir / "buses_acpf.csv", encoding="utf-8") as f: + bus_rows = sum(1 for _ in f) - 1 # subtract header + assert bus_rows > 20000, f"Expected >20K bus rows, got {bus_rows}" + + # Branch CSV should have >30,000 rows + with open(result_dir / "branches_acpf.csv", encoding="utf-8") as f: + branch_rows = sum(1 for _ in f) - 1 + assert branch_rows > 30000, f"Expected >30K branch rows, got {branch_rows}" + + # Generator CSV should have >1,000 rows + with open(result_dir / "generators_acpf.csv", encoding="utf-8") as f: + gen_rows = sum(1 for _ in f) - 1 + assert gen_rows > 1000, f"Expected >1K generator rows, got {gen_rows}" + + # Summary JSON solution_source should match classification + summary = json.loads((result_dir / "summary_acpf.json").read_text(encoding="utf-8")) + classification = summary["snapshot_classification"] + if classification == "solved": + assert summary["solution_source"] == "extracted" + else: + assert summary["solution_source"] == "computed" + + # Log key stats for manual review + print("\n--- ACPF Reference Summary ---") + print(f" Buses: {bus_rows}") + print(f" Branches: {branch_rows}") + print(f" Generators: {gen_rows}") + print(f" Solution source: {summary['solution_source']}") + print(f" Total gen MW: {summary['system_summary']['total_gen_mw']:.1f}") + print(f" Total load MW: {summary['system_summary']['total_load_mw']:.1f}") + print(f" Total loss MW: {summary['system_summary']['total_loss_mw']:.1f}") + print(f" Balance residual MW: {summary['system_summary']['power_balance_residual_mw']:.4f}") + + +@pytest.mark.fnm +def test_fnm_acpf_reference_power_balance(tmp_path: Path, require_fnm: object) -> None: + """T16: ACPF reference power balance is within tolerance.""" + intermediate_dir = Path("data/fnm/intermediate/canonical") + snapshot_json = Path("data/fnm/intermediate/snapshot/snapshot_confirmation.json") + + if not intermediate_dir.is_dir() or not snapshot_json.exists(): + pytest.skip("D6/D8 intermediate outputs not available") + + output_dir = tmp_path / "acpf" + build_acpf_reference( + intermediate_dir=intermediate_dir, + snapshot_json_path=snapshot_json, + canonical_parser="matpower", + output_dir=output_dir, + ) + + summary = json.loads((output_dir / "summary_acpf.json").read_text(encoding="utf-8")) + ss = summary["system_summary"] + + # Power balance residual < 1 MW + assert abs(ss["power_balance_residual_mw"]) < 1.0, ( + f"Power balance residual {ss['power_balance_residual_mw']:.4f} MW exceeds 1.0 MW" + ) + + # Generation must exceed load (to cover losses) + assert ss["total_gen_mw"] > ss["total_load_mw"], ( + f"Generation {ss['total_gen_mw']:.1f} MW <= Load {ss['total_load_mw']:.1f} MW" + ) + + # Losses must be positive + assert ss["total_loss_mw"] > 0, f"Losses {ss['total_loss_mw']:.1f} MW <= 0" + + # Losses < 10% of generation + loss_pct = ss["total_loss_mw"] / ss["total_gen_mw"] + assert loss_pct < 0.10, f"Losses {loss_pct:.2%} of generation exceeds 10% threshold" diff --git a/data/fnm/tests/test_bus_exclusion_registry.py b/data/fnm/tests/test_bus_exclusion_registry.py new file mode 100644 index 00000000..193b0eea --- /dev/null +++ b/data/fnm/tests/test_bus_exclusion_registry.py @@ -0,0 +1,507 @@ +"""Tests for bus exclusion registry (PRD 03/01). + +Tests T01-T08 are synthetic unit tests requiring no FNM data. +Tests T09-T10 are synthetic integration tests (end-to-end with temp CSV files). +""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path + +from fnm.scripts.bus_exclusion_registry import ( + BusExclusionRegistry, + ExcludedBusRecord, + ExclusionReason, + ExclusionSummary, + IslandSummary, + RegistryMetadata, + build_connectivity_graph, + build_excluded_bus_records, + build_registry, + find_connected_components, + find_ide4_buses, + find_vm_zero_buses, + identify_main_island, + registry_to_csv, + registry_to_json, +) + +# --------------------------------------------------------------------------- +# Helpers to build synthetic bus/branch/transformer rows +# --------------------------------------------------------------------------- + + +def _bus_row( + bus_i: int, + ide: int = 1, + vm: float = 1.0, + va: float = 0.0, + name: str = "", + area: int = 1, + zone: int = 1, + baskv: float = 115.0, +) -> dict: + return { + "I": bus_i, + "IDE": ide, + "VM": vm, + "VA": va, + "NAME": name, + "AREA": area, + "ZONE": zone, + "BASKV": baskv, + } + + +def _branch_row(i: int, j: int, st: int = 1) -> dict: + return {"I": i, "J": j, "ST": st, "CKT": "1"} + + +def _xfmr_row(i: int, j: int, k: int = 0, stat: int = 1) -> dict: + return {"I": i, "J": j, "K": k, "STAT": stat, "CKT": "1"} + + +# --------------------------------------------------------------------------- +# T01: test_find_ide4_buses +# --------------------------------------------------------------------------- + + +def test_find_ide4_buses() -> None: + """T01: Create 10 buses, 2 with IDE=4, 1 with IDE=3, rest IDE=1. + Verify exactly the 2 IDE=4 bus numbers are returned.""" + buses = [ + _bus_row(1, ide=3), # slack + _bus_row(2, ide=1), + _bus_row(3, ide=1), + _bus_row(4, ide=4), # isolated + _bus_row(5, ide=1), + _bus_row(6, ide=1), + _bus_row(7, ide=4), # isolated + _bus_row(8, ide=1), + _bus_row(9, ide=1), + _bus_row(10, ide=1), + ] + result = find_ide4_buses(buses) + assert result == {4, 7} + + +# --------------------------------------------------------------------------- +# T02: test_find_vm_zero_buses +# --------------------------------------------------------------------------- + + +def test_find_vm_zero_buses() -> None: + """T02: Create 10 buses, 3 with VM=0.0, rest with VM in [0.95, 1.05]. + Verify exactly the 3 VM=0 bus numbers are returned. Verify VM=0.001 + is NOT included.""" + buses = [ + _bus_row(1, vm=1.0), + _bus_row(2, vm=0.0), # VM=0 + _bus_row(3, vm=0.95), + _bus_row(4, vm=0.0), # VM=0 + _bus_row(5, vm=1.05), + _bus_row(6, vm=0.0), # VM=0 + _bus_row(7, vm=0.001), # NOT zero + _bus_row(8, vm=0.98), + _bus_row(9, vm=1.02), + _bus_row(10, vm=0.99), + ] + result = find_vm_zero_buses(buses) + assert result == {2, 4, 6} + assert 7 not in result # 0.001 is not zero + + +# --------------------------------------------------------------------------- +# T03: test_build_connectivity_graph_excludes_ide4 +# --------------------------------------------------------------------------- + + +def test_build_connectivity_graph_excludes_ide4() -> None: + """T03: 5 buses in a chain (1-2-3-4-5), bus 3 is IDE=4. + After excluding bus 3, nodes {1,2} and {4,5} should be disconnected.""" + buses = [ + _bus_row(1, ide=1), + _bus_row(2, ide=1), + _bus_row(3, ide=4), # excluded + _bus_row(4, ide=1), + _bus_row(5, ide=1), + ] + branches = [ + _branch_row(1, 2), + _branch_row(2, 3), + _branch_row(3, 4), + _branch_row(4, 5), + ] + adj = build_connectivity_graph(buses, branches, [], excluded_bus_numbers={3}) + + # Node 3 not in graph + assert 3 not in adj + + # Nodes 1,2 connected to each other + assert 2 in adj[1] + assert 1 in adj[2] + + # Nodes 4,5 connected to each other + assert 5 in adj[4] + assert 4 in adj[5] + + # No path between {1,2} and {4,5} + assert adj[1] == {2} + assert adj[2] == {1} + assert adj[4] == {5} + assert adj[5] == {4} + + +# --------------------------------------------------------------------------- +# T04: test_find_connected_components_simple +# --------------------------------------------------------------------------- + + +def test_find_connected_components_simple() -> None: + """T04: Two disconnected clusters {1,2,3} and {4,5}.""" + adj: dict[int, set[int]] = { + 1: {2, 3}, + 2: {1, 3}, + 3: {1, 2}, + 4: {5}, + 5: {4}, + } + components = find_connected_components(adj) + assert len(components) == 2 + sizes = sorted([len(c) for c in components], reverse=True) + assert sizes == [3, 2] + + +# --------------------------------------------------------------------------- +# T05: test_identify_main_island +# --------------------------------------------------------------------------- + + +def test_identify_main_island() -> None: + """T05: Bus 1 is IDE=3 (slack). Two components: {1,2,3} and {4,5}. + Main island should contain bus 1.""" + buses = [ + _bus_row(1, ide=3), + _bus_row(2, ide=1), + _bus_row(3, ide=1), + _bus_row(4, ide=1), + _bus_row(5, ide=1), + ] + components = [{1, 2, 3}, {4, 5}] + idx, slack_bus = identify_main_island(components, buses) + assert slack_bus == 1 + assert 1 in components[idx] + + +# --------------------------------------------------------------------------- +# T06: test_three_winding_transformer_connectivity +# --------------------------------------------------------------------------- + + +def test_three_winding_transformer_connectivity() -> None: + """T06: 4 buses, one 3-winding transformer (I=1, J=2, K=3). No branches. + Buses 1, 2, 3 should be mutually connected. Bus 4 is isolated.""" + buses = [ + _bus_row(1, ide=1), + _bus_row(2, ide=1), + _bus_row(3, ide=1), + _bus_row(4, ide=1), + ] + xfmrs = [_xfmr_row(1, 2, k=3, stat=1)] + adj = build_connectivity_graph(buses, [], xfmrs, excluded_bus_numbers=set()) + + # 1, 2, 3 are all mutually connected + assert 2 in adj[1] and 3 in adj[1] + assert 1 in adj[2] and 3 in adj[2] + assert 1 in adj[3] and 2 in adj[3] + + # Bus 4 is isolated + assert adj[4] == set() + + +# --------------------------------------------------------------------------- +# T07: test_exclusion_priority_ordering +# --------------------------------------------------------------------------- + + +def test_exclusion_priority_ordering() -> None: + """T07: A bus that is IDE=4, VM=0, and disconnected. Primary reason + should be IDE_4_ISOLATED, all_reasons should contain all three.""" + buses = [_bus_row(99, ide=4, vm=0.0)] + ide4 = {99} + vm_zero = {99} + disconnected = {99} + + records = build_excluded_bus_records(buses, ide4, vm_zero, disconnected) + assert len(records) == 1 + rec = records[0] + assert rec.primary_reason == ExclusionReason.IDE_4_ISOLATED + assert ExclusionReason.IDE_4_ISOLATED in rec.all_reasons + assert ExclusionReason.VM_ZERO_DEENERGIZED in rec.all_reasons + assert ExclusionReason.DISCONNECTED_ISLAND in rec.all_reasons + assert len(rec.all_reasons) == 3 + + +# --------------------------------------------------------------------------- +# T08: test_out_of_service_branches_excluded_from_graph +# --------------------------------------------------------------------------- + + +def test_out_of_service_branches_excluded_from_graph() -> None: + """T08: 3 buses, branch 1-2 in-service (ST=1), branch 2-3 out-of-service + (ST=0). Edge 1-2 exists, edge 2-3 does not. Bus 3 is isolated node.""" + buses = [ + _bus_row(1, ide=1), + _bus_row(2, ide=1), + _bus_row(3, ide=1), + ] + branches = [ + _branch_row(1, 2, st=1), + _branch_row(2, 3, st=0), + ] + adj = build_connectivity_graph(buses, branches, [], excluded_bus_numbers=set()) + + assert 2 in adj[1] + assert 1 in adj[2] + assert 3 not in adj[2] + assert adj[3] == set() + + +# --------------------------------------------------------------------------- +# Helper: write CSV from rows +# --------------------------------------------------------------------------- + + +def _write_csv(path: Path, headers: list[str], rows: list[list]) -> None: + """Write a simple CSV file.""" + with open(path, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + writer.writerow(headers) + for row in rows: + writer.writerow(row) + + +# --------------------------------------------------------------------------- +# T09: test_build_registry_end_to_end_synthetic +# --------------------------------------------------------------------------- + + +def test_build_registry_end_to_end_synthetic(tmp_path: Path) -> None: + """T09: End-to-end with synthetic CSV files. + + 20 buses: + - Bus 1: IDE=3 (slack), VM=1.0 + - Buses 2-14: IDE=1, VM in [0.95..1.05] -- main island (14 buses incl slack) + - Bus 15: IDE=4, VM=0.0 (excluded: IDE=4) + - Bus 16: IDE=4, VM=1.0 (excluded: IDE=4) + - Bus 17: IDE=1, VM=0.0 (excluded: VM=0, in main island) + - Buses 18-20: IDE=1, VM=1.0 (excluded: disconnected island of 3) + + Branches form: + - Main network: chain 1-2-3-...-14 plus bus 17 connected to bus 14 + - Disconnected island: chain 18-19-20 + - No branches to IDE=4 buses (15, 16) + + Transformers: 3 in-service 2-winding connecting pairs in main island. + """ + bus_csv = tmp_path / "bus.csv" + branch_csv = tmp_path / "branch.csv" + xfmr_csv = tmp_path / "transformer.csv" + + # Bus data + bus_headers = ["I", "NAME", "BASKV", "IDE", "AREA", "ZONE", "VM", "VA"] + bus_data: list[list] = [] + # Slack bus + bus_data.append([1, "SLACK", 230.0, 3, 1, 1, 1.0, 0.0]) + # Main island buses 2-14 + for i in range(2, 15): + vm = 0.95 + (i % 10) * 0.01 + bus_data.append([i, f"BUS-{i}", 115.0, 1, 1, 1, vm, -1.0 * (i % 5)]) + # IDE=4 buses + bus_data.append([15, "ISOL-1", 230.0, 4, 2, 2, 0.0, 0.0]) + bus_data.append([16, "ISOL-2", 115.0, 4, 2, 2, 1.0, 0.0]) + # VM=0 bus (still connected in main island) + bus_data.append([17, "DEENRG", 69.0, 1, 1, 1, 0.0, 0.0]) + # Disconnected island + bus_data.append([18, "ISLAND-1", 69.0, 1, 3, 3, 1.01, -5.0]) + bus_data.append([19, "ISLAND-2", 69.0, 1, 3, 3, 0.99, -5.5]) + bus_data.append([20, "ISLAND-3", 69.0, 1, 3, 3, 1.0, -4.8]) + + _write_csv(bus_csv, bus_headers, bus_data) + + # Branch data: chain 1-2-3-...-14, then 14-17, then 18-19-20 + branch_headers = ["I", "J", "CKT", "ST"] + branch_data: list[list] = [] + for i in range(1, 14): + branch_data.append([i, i + 1, "1", 1]) + branch_data.append([14, 17, "1", 1]) # VM=0 bus connected + branch_data.append([18, 19, "1", 1]) # disconnected island + branch_data.append([19, 20, "1", 1]) + + _write_csv(branch_csv, branch_headers, branch_data) + + # Transformer data: 3 in-service 2-winding in main island + xfmr_headers = ["I", "J", "K", "CKT", "STAT"] + xfmr_data: list[list] = [ + [1, 2, 0, "1", 1], + [5, 6, 0, "1", 1], + [10, 11, 0, "1", 1], + ] + _write_csv(xfmr_csv, xfmr_headers, xfmr_data) + + # Build registry + registry = build_registry(bus_csv, branch_csv, xfmr_csv) + + # Verify summary + assert registry.summary.total_buses == 20 + assert registry.summary.ide4_count == 2 + # vm_zero_count counts ALL VM=0 buses (may overlap with IDE=4): bus 15 + bus 17 + assert registry.summary.vm_zero_count == 2 + + # Disconnected island: 3 buses (18, 19, 20) + # Total excluded = 2 (IDE=4) + 1 (VM=0) + 3 (disconnected) = 6 + assert registry.summary.excluded_total == 6 + assert registry.summary.remaining_for_verification == 14 + + # Connected components: main (15 buses: 1-14 + 17) + island (3 buses: 18-20) = 2 + assert registry.summary.connected_components == 2 + + # Verify excluded_buses sorted by bus number + bus_nums = [r.bus_number for r in registry.excluded_buses] + assert bus_nums == sorted(bus_nums) + assert len(registry.excluded_buses) == 6 + + # Verify specific records + excluded_map = {r.bus_number: r for r in registry.excluded_buses} + assert excluded_map[15].primary_reason == ExclusionReason.IDE_4_ISOLATED + assert excluded_map[16].primary_reason == ExclusionReason.IDE_4_ISOLATED + assert excluded_map[17].primary_reason == ExclusionReason.VM_ZERO_DEENERGIZED + assert excluded_map[18].primary_reason == ExclusionReason.DISCONNECTED_ISLAND + assert excluded_map[19].primary_reason == ExclusionReason.DISCONNECTED_ISLAND + assert excluded_map[20].primary_reason == ExclusionReason.DISCONNECTED_ISLAND + + +# --------------------------------------------------------------------------- +# T10: test_registry_csv_json_roundtrip +# --------------------------------------------------------------------------- + + +def test_registry_csv_json_roundtrip(tmp_path: Path) -> None: + """T10: Build a synthetic registry, write CSV+JSON, read back and verify.""" + # Build a small synthetic registry directly + rec1 = ExcludedBusRecord( + bus_number=100, + bus_name="BUS-A", + area=1, + zone=10, + base_kv=230.0, + primary_reason=ExclusionReason.IDE_4_ISOLATED, + all_reasons=[ExclusionReason.IDE_4_ISOLATED, ExclusionReason.VM_ZERO_DEENERGIZED], + island_id=None, + vm=0.0, + va=0.0, + ide=4, + ) + rec2 = ExcludedBusRecord( + bus_number=200, + bus_name="BUS-B", + area=2, + zone=20, + base_kv=115.0, + primary_reason=ExclusionReason.DISCONNECTED_ISLAND, + all_reasons=[ExclusionReason.DISCONNECTED_ISLAND], + island_id=1, + vm=1.01, + va=-3.5, + ide=1, + ) + + island_main = IslandSummary( + island_id=0, + bus_count=50, + is_main=True, + slack_bus=1, + sample_buses=[1, 2, 3, 4, 5], + voltage_levels=[230.0, 115.0], + ) + island_disc = IslandSummary( + island_id=1, + bus_count=5, + is_main=False, + slack_bus=None, + sample_buses=[200, 201, 202], + voltage_levels=[115.0], + ) + + summary = ExclusionSummary( + total_buses=57, + excluded_total=2, + excluded_by_reason={ + "ide_4_isolated": 1, + "disconnected_island": 1, + }, + remaining_for_verification=55, + connected_components=2, + main_island_size=50, + disconnected_island_count=1, + disconnected_island_sizes=[5], + islands=[island_main, island_disc], + ide4_count=1, + vm_zero_count=1, + disconnected_count=1, + ) + + metadata = RegistryMetadata( + bus_csv_path="bus.csv", + branch_csv_path="branch.csv", + transformer_csv_path="transformer.csv", + generated_timestamp="2026-03-06T12:00:00+00:00", + slack_bus_number=1, + vm_zero_threshold=0.0, + graph_node_count=56, + graph_edge_count=70, + ) + + registry = BusExclusionRegistry( + excluded_buses=[rec1, rec2], + summary=summary, + metadata=metadata, + ) + + # Write CSV and JSON + csv_path = tmp_path / "excluded_buses.csv" + json_path = tmp_path / "excluded_buses.json" + + registry_to_csv(registry, csv_path) + registry_to_json(registry, json_path) + + # Read CSV back + with open(csv_path, encoding="utf-8") as f: + reader = csv.DictReader(f) + csv_rows = list(reader) + + assert len(csv_rows) == 2 + assert csv_rows[0]["bus_number"] == "100" + assert csv_rows[0]["primary_reason"] == "ide_4_isolated" + assert "vm_zero_deenergized" in csv_rows[0]["all_reasons"] + assert csv_rows[0]["island_id"] == "" # None serialized as empty + assert csv_rows[1]["bus_number"] == "200" + assert csv_rows[1]["primary_reason"] == "disconnected_island" + assert csv_rows[1]["island_id"] == "1" + + # Read JSON back + with open(json_path, encoding="utf-8") as f: + json_data = json.load(f) + + assert "excluded_buses" in json_data + assert "summary" in json_data + assert "metadata" in json_data + + assert len(json_data["excluded_buses"]) == 2 + assert json_data["summary"]["excluded_total"] == len(json_data["excluded_buses"]) + assert ( + json_data["summary"]["remaining_for_verification"] + == json_data["summary"]["total_buses"] - json_data["summary"]["excluded_total"] + ) diff --git a/data/fnm/tests/test_csv_join_keys.py b/data/fnm/tests/test_csv_join_keys.py new file mode 100644 index 00000000..5220f98b --- /dev/null +++ b/data/fnm/tests/test_csv_join_keys.py @@ -0,0 +1,472 @@ +"""Tests for supplemental CSV join-key mapping (PRD 09). + +Tests T01-T07 are synthetic (no FNM data required). +Tests T08-T09 are integration tests with synthetic fixtures. +Test T10 requires actual FNM data (FNM_PATH env var + D7 outputs). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from fnm.scripts.csv_join_keys import ( + CandidateKey, + JoinCardinality, + KeyType, + analyze_csv, + build_join_key_report, + discover_candidate_keys, + get_default_key_patterns, + report_to_dict, + report_to_markdown, + validate_join, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _write_csv(path: Path, header: list[str], rows: list[list[str]]) -> None: + """Write a minimal CSV file.""" + lines = [",".join(header)] + for row in rows: + lines.append(",".join(row)) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +# --------------------------------------------------------------------------- +# T01: discover_candidate_keys — bus number +# --------------------------------------------------------------------------- + + +def test_discover_candidate_keys_bus_number() -> None: + """T01: CSV with 'bus_num' column discovers a BUS_NUMBER candidate.""" + columns = ["bus_num", "load_mw", "load_mvar"] + sample = [ + {"bus_num": "1", "load_mw": "100.0", "load_mvar": "50.0"}, + {"bus_num": "2", "load_mw": "200.0", "load_mvar": "75.0"}, + {"bus_num": "3", "load_mw": "150.0", "load_mvar": "60.0"}, + ] + patterns = get_default_key_patterns() + + candidates = discover_candidate_keys("test.csv", columns, sample, patterns) + + bus_candidates = [c for c in candidates if c.key_type == KeyType.BUS_NUMBER] + assert len(bus_candidates) == 1 + assert bus_candidates[0].csv_columns == ["bus_num"] + assert bus_candidates[0].confidence > 0.5 + + +# --------------------------------------------------------------------------- +# T02: discover_candidate_keys — branch composite +# --------------------------------------------------------------------------- + + +def test_discover_candidate_keys_branch_composite() -> None: + """T02: CSV with from_bus/to_bus/ckt discovers a BRANCH_COMPOSITE candidate.""" + columns = ["from_bus", "to_bus", "ckt", "rating_a"] + sample = [ + {"from_bus": "1", "to_bus": "2", "ckt": "1", "rating_a": "100.0"}, + {"from_bus": "3", "to_bus": "4", "ckt": "1", "rating_a": "200.0"}, + ] + patterns = get_default_key_patterns() + + candidates = discover_candidate_keys("test.csv", columns, sample, patterns) + + branch_candidates = [c for c in candidates if c.key_type == KeyType.BRANCH_COMPOSITE] + assert len(branch_candidates) >= 1 + assert branch_candidates[0].csv_columns == ["from_bus", "to_bus", "ckt"] + + +# --------------------------------------------------------------------------- +# T03: discover_candidate_keys — no match +# --------------------------------------------------------------------------- + + +def test_discover_candidate_keys_no_match() -> None: + """T03: CSV with unrelated columns returns no candidates (or only UNKNOWN).""" + columns = ["timestamp", "value", "category"] + sample = [ + {"timestamp": "2024-01-01", "value": "42.0", "category": "A"}, + ] + patterns = get_default_key_patterns() + + candidates = discover_candidate_keys("test.csv", columns, sample, patterns) + + real_candidates = [c for c in candidates if c.key_type != KeyType.UNKNOWN] + assert len(real_candidates) == 0 + + +# --------------------------------------------------------------------------- +# T04: validate_join — perfect match +# --------------------------------------------------------------------------- + + +def test_validate_join_perfect_match(tmp_path: Path) -> None: + """T04: All CSV bus numbers exist in intermediate table => 100% match.""" + # Supplemental CSV + csv_path = tmp_path / "test.csv" + _write_csv(csv_path, ["bus_num", "load_mw"], [["1", "100"], ["2", "200"], ["3", "150"]]) + + # Intermediate bus table + inter_dir = tmp_path / "intermediate" + inter_dir.mkdir() + _write_csv( + inter_dir / "bus.csv", + ["I", "NAME", "BASKV"], + [ + ["1", "BUS1", "138"], + ["2", "BUS2", "138"], + ["3", "BUS3", "230"], + ["4", "BUS4", "345"], + ["5", "BUS5", "500"], + ], + ) + + candidate = CandidateKey( + csv_file="test.csv", + csv_columns=["bus_num"], + key_type=KeyType.BUS_NUMBER, + confidence=0.9, + ) + + result = validate_join(csv_path, candidate, inter_dir, "bus", ["I"]) + + assert result.match_rate == 1.0 + assert result.matched_row_count == 3 + assert result.unmatched_row_count == 0 + assert result.is_valid is True + + +# --------------------------------------------------------------------------- +# T05: validate_join — partial match +# --------------------------------------------------------------------------- + + +def test_validate_join_partial_match(tmp_path: Path) -> None: + """T05: 3 of 5 CSV bus numbers match => 60% match, below threshold.""" + csv_path = tmp_path / "test.csv" + _write_csv( + csv_path, + ["bus_num", "load_mw"], + [["1", "100"], ["2", "200"], ["3", "150"], ["99", "50"], ["100", "75"]], + ) + + inter_dir = tmp_path / "intermediate" + inter_dir.mkdir() + _write_csv( + inter_dir / "bus.csv", + ["I", "NAME"], + [["1", "BUS1"], ["2", "BUS2"], ["3", "BUS3"]], + ) + + candidate = CandidateKey( + csv_file="test.csv", + csv_columns=["bus_num"], + key_type=KeyType.BUS_NUMBER, + confidence=0.9, + ) + + result = validate_join(csv_path, candidate, inter_dir, "bus", ["I"]) + + assert result.match_rate == pytest.approx(0.6) + assert result.unmatched_row_count == 2 + assert result.is_valid is False + # Verify unmatched samples contain bus 99 and 100 + unmatched_values = {s["bus_num"] for s in result.unmatched_sample} + assert "99" in unmatched_values + assert "100" in unmatched_values + + +# --------------------------------------------------------------------------- +# T06: validate_join — cardinality N:1 +# --------------------------------------------------------------------------- + + +def test_validate_join_cardinality_n_to_1(tmp_path: Path) -> None: + """T06: Multiple CSV rows reference same bus => MANY_TO_ONE cardinality.""" + csv_path = tmp_path / "test.csv" + _write_csv( + csv_path, + ["bus_num", "load_mw"], + [["1", "100"], ["1", "110"], ["1", "120"], ["2", "200"], ["2", "210"]], + ) + + inter_dir = tmp_path / "intermediate" + inter_dir.mkdir() + _write_csv( + inter_dir / "bus.csv", + ["I", "NAME"], + [["1", "BUS1"], ["2", "BUS2"]], + ) + + candidate = CandidateKey( + csv_file="test.csv", + csv_columns=["bus_num"], + key_type=KeyType.BUS_NUMBER, + confidence=0.9, + ) + + result = validate_join(csv_path, candidate, inter_dir, "bus", ["I"]) + + assert result.cardinality == JoinCardinality.MANY_TO_ONE + + +# --------------------------------------------------------------------------- +# T07: analyze_csv — selects primary join +# --------------------------------------------------------------------------- + + +def test_analyze_csv_selects_primary_join(tmp_path: Path) -> None: + """T07: CSV with bus_num (100% match) and area (85% match) => bus is primary.""" + csv_path = tmp_path / "test.csv" + _write_csv( + csv_path, + ["bus_num", "area", "load_mw"], + [ + ["1", "10", "100"], + ["2", "20", "200"], + ["3", "30", "150"], + ["4", "40", "175"], + ["5", "50", "125"], + ["6", "60", "110"], + ["7", "70", "130"], + ["8", "80", "140"], + ["9", "90", "160"], + ["10", "100", "180"], + ], + ) + + inter_dir = tmp_path / "intermediate" + inter_dir.mkdir() + + # Bus table: all 10 buses present => 100% match + _write_csv( + inter_dir / "bus.csv", + ["I", "NAME"], + [[str(i), f"BUS{i}"] for i in range(1, 11)], + ) + + # Area table: only 8 of 10 areas present => some won't match, but we need + # the area column pattern to match. We'll use areas 10-80 (8 of 10 match = 80%). + # Actually the threshold is 0.80, so 85% means 8.5 of 10. Let's use + # 9 out of 10 to get a cleaner 90% that is still below 100%. + # Wait - we want area to achieve 85%. With 10 rows, we need ~8-9 matches. + # Let's have areas 10,20,30,40,50,60,70,80 in table (8 match) but not 90,100 + # => 80% match. That's at the threshold. Let's add one more to get 90%. + # Use 9 matches for 90%, still below bus's 100%. + _write_csv( + inter_dir / "area.csv", + ["I", "ARNAME"], + [[str(i * 10), f"AREA{i}"] for i in range(1, 10)], # 10..90 (9 values) + ) + + mapping = analyze_csv(csv_path, inter_dir) + + assert mapping.primary_join is not None + assert mapping.primary_join.candidate.key_type == KeyType.BUS_NUMBER + assert mapping.primary_join.match_rate == 1.0 + + # Area join should be in secondary_joins + assert len(mapping.secondary_joins) >= 1 + area_joins = [ + sj for sj in mapping.secondary_joins if sj.candidate.key_type == KeyType.AREA_NUMBER + ] + assert len(area_joins) == 1 + assert area_joins[0].match_rate == pytest.approx(0.9) + + +# --------------------------------------------------------------------------- +# T08: build_report — end to end +# --------------------------------------------------------------------------- + + +def test_build_report_end_to_end(tmp_path: Path) -> None: + """T08: Build report with 3 synthetic CSVs and 3 intermediate tables.""" + fnm_dir = tmp_path / "fnm" + fnm_dir.mkdir() + inter_dir = tmp_path / "intermediate" + inter_dir.mkdir() + + # Intermediate tables + _write_csv( + inter_dir / "bus.csv", + ["I", "NAME", "BASKV"], + [[str(i), f"BUS{i}", "138"] for i in range(1, 11)], + ) + _write_csv( + inter_dir / "branch.csv", + ["I", "J", "CKT", "R"], + [["1", "2", "1", "0.01"], ["3", "4", "1", "0.02"], ["5", "6", "1", "0.03"]], + ) + _write_csv( + inter_dir / "generator.csv", + ["I", "ID", "NAME", "PG"], + [["1", "1", "GEN1", "100"], ["2", "1", "GEN2", "200"]], + ) + + # LINE_AND_TRANSFORMER.csv — uses branch composite keys + _write_csv( + fnm_dir / "LINE_AND_TRANSFORMER.csv", + ["from_bus", "to_bus", "ckt", "rating_a", "rating_b"], + [["1", "2", "1", "100", "120"], ["3", "4", "1", "200", "240"]], + ) + + # TRADING_HUB.csv — uses bus number + _write_csv( + fnm_dir / "TRADING_HUB.csv", + ["hub_name", "bus_num", "factor"], + [["HUB_A", "1", "0.5"], ["HUB_A", "2", "0.3"], ["HUB_B", "3", "0.8"]], + ) + + # CONTINGENCY.csv — uses branch composite keys + _write_csv( + fnm_dir / "CONTINGENCY.csv", + ["ctg_name", "from_bus", "to_bus", "ckt"], + [["CTG1", "1", "2", "1"], ["CTG2", "5", "6", "1"]], + ) + + manifest_names = [ + "LINE_AND_TRANSFORMER.csv", + "TRADING_HUB.csv", + "GEN_DISTRIBUTION_FACTOR.csv", + "CONTINGENCY.csv", + "INTERFACE.csv", + "INTERFACE_ELEMENT.csv", + "OUTAGE.csv", + ] + + report = build_join_key_report( + fnm_path=fnm_dir, + intermediate_dir=inter_dir, + manifest_csv_names=manifest_names, + ) + + assert len(report.csv_mappings) == 3 + assert sorted(report.csvs_found) == sorted( + [ + "LINE_AND_TRANSFORMER.csv", + "TRADING_HUB.csv", + "CONTINGENCY.csv", + ] + ) + assert sorted(report.csvs_missing) == sorted( + [ + "GEN_DISTRIBUTION_FACTOR.csv", + "INTERFACE.csv", + "INTERFACE_ELEMENT.csv", + "OUTAGE.csv", + ] + ) + assert report.overall_summary.total_csvs_analyzed == 3 + + +# --------------------------------------------------------------------------- +# T09: report_to_dict and JSON roundtrip +# --------------------------------------------------------------------------- + + +def test_report_to_dict_and_json_roundtrip(tmp_path: Path) -> None: + """T09: Convert report to dict, serialize/deserialize JSON, verify structure.""" + fnm_dir = tmp_path / "fnm" + fnm_dir.mkdir() + inter_dir = tmp_path / "intermediate" + inter_dir.mkdir() + + # Minimal intermediate table + _write_csv(inter_dir / "bus.csv", ["I", "NAME"], [["1", "BUS1"], ["2", "BUS2"]]) + + # Minimal CSV + _write_csv(fnm_dir / "TRADING_HUB.csv", ["hub", "bus_num"], [["H1", "1"], ["H1", "2"]]) + + report = build_join_key_report( + fnm_path=fnm_dir, + intermediate_dir=inter_dir, + manifest_csv_names=["TRADING_HUB.csv"], + ) + + d = report_to_dict(report) + json_str = json.dumps(d, indent=2) + loaded = json.loads(json_str) + + # Verify all top-level keys + assert "csv_mappings" in loaded + assert "csvs_found" in loaded + assert "csvs_missing" in loaded + assert "intermediate_tables_used" in loaded + assert "overall_summary" in loaded + assert "metadata" in loaded + + # Verify enum values are serialized as strings (not objects) + for mapping in loaded["csv_mappings"]: + for candidate in mapping["candidate_keys"]: + assert isinstance(candidate["key_type"], str) + for vj in mapping["validated_joins"]: + assert isinstance(vj["cardinality"], str) + + +# --------------------------------------------------------------------------- +# T10: FNM integration test (requires FNM_PATH and D7 outputs) +# --------------------------------------------------------------------------- + + +@pytest.mark.fnm +def test_fnm_join_key_report_all_csvs_analyzed(require_fnm: dict, tmp_path: Path) -> None: + """T10: Run with actual FNM data and verify results. + + Requires FNM_PATH env var and D7 intermediate tables. + """ + fnm_path = Path(require_fnm["fnm_path"]) + + # Locate intermediate directory relative to repo root + repo_root = Path(__file__).resolve().parent.parent.parent.parent + intermediate_dir = repo_root / "data" / "fnm" / "intermediate" / "canonical" + + if not intermediate_dir.is_dir(): + pytest.skip(f"Intermediate directory not found: {intermediate_dir}") + + output_dir = tmp_path / "csv_join_keys" + output_dir.mkdir() + + report = build_join_key_report( + fnm_path=fnm_path, + intermediate_dir=intermediate_dir, + ) + + # (a) csvs_found is non-empty + assert len(report.csvs_found) > 0, "No supplemental CSVs found at FNM_PATH" + + # (b) Every found CSV has at least one CandidateKey + for mapping in report.csv_mappings: + assert len(mapping.candidate_keys) > 0, ( + f"{mapping.csv_file} has no candidate keys discovered" + ) + + # (c) At least 5 of 7 CSVs have a valid primary join + csvs_with_primary = sum(1 for m in report.csv_mappings if m.primary_join is not None) + assert csvs_with_primary >= 5, ( + f"Only {csvs_with_primary} CSVs have a valid primary join (expected >= 5)" + ) + + # (d) Write report files and verify they are non-empty + import json as json_mod + + json_path = output_dir / "join_key_report.json" + json_path.write_text(json_mod.dumps(report_to_dict(report), indent=2) + "\n", encoding="utf-8") + assert json_path.stat().st_size > 0 + + md_path = output_dir / "join_key_report.md" + md_path.write_text(report_to_markdown(report), encoding="utf-8") + assert md_path.stat().st_size > 0 + + # Log summary for manual review + print("\n--- Join Key Report Summary ---") + print(f"CSVs found: {report.csvs_found}") + print(f"CSVs missing: {report.csvs_missing}") + print(f"CSVs with valid primary join: {csvs_with_primary}") + print(f"Average match rate: {report.overall_summary.average_match_rate:.1%}") + print(f"CSVs needing review: {report.overall_summary.csvs_needing_review}") diff --git a/data/fnm/tests/test_dcpf_acpf_characterization.py b/data/fnm/tests/test_dcpf_acpf_characterization.py new file mode 100644 index 00000000..dae32483 --- /dev/null +++ b/data/fnm/tests/test_dcpf_acpf_characterization.py @@ -0,0 +1,795 @@ +"""Tests for DCPF-vs-ACPF Characterization (PRD 03/04). + +Tests T01-T12 are synthetic (no FNM data required). +Tests T13-T14 require FNM_PATH and D2/D3 outputs. +""" + +from __future__ import annotations + +import csv +import json +import math +from pathlib import Path + +import pytest + +from fnm.scripts.dcpf_acpf_characterization import ( + AggregateStats, + BranchDeviation, + BusDeviation, + CharacterizationResult, + ComplianceFractions, + DeviationCause, + annotate_branch_causes, + annotate_bus_causes, + build_characterization, + compute_aggregate_stats, + compute_branch_deviations, + compute_bus_deviations, + compute_compliance_fractions, + join_branches, + join_buses, + write_characterization_json, +) + +# --------------------------------------------------------------------------- +# Helpers for writing synthetic CSV/JSON files +# --------------------------------------------------------------------------- + + +def _write_acpf_bus_csv(path: Path, rows: list[dict]) -> None: + """Write a synthetic buses_acpf.csv.""" + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["bus", "VM", "VA"]) + for r in rows: + writer.writerow([r["bus"], r.get("VM", 1.0), r["VA"]]) + + +def _write_dcpf_bus_csv(path: Path, rows: list[dict]) -> None: + """Write a synthetic buses_dcpf.csv.""" + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["bus", "VA"]) + for r in rows: + writer.writerow([r["bus"], r["VA"]]) + + +def _write_acpf_branch_csv(path: Path, rows: list[dict]) -> None: + """Write a synthetic branches_acpf.csv.""" + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["from_bus", "to_bus", "ckt", "P_from", "Q_from", "P_to", "Q_to"]) + for r in rows: + writer.writerow( + [ + r["from_bus"], + r["to_bus"], + r["ckt"], + r.get("P_from", 0), + r.get("Q_from", 0), + r.get("P_to", 0), + r.get("Q_to", 0), + ] + ) + + +def _write_dcpf_branch_csv(path: Path, rows: list[dict]) -> None: + """Write a synthetic branches_dcpf.csv.""" + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["from_bus", "to_bus", "ckt", "P_flow_MW"]) + for r in rows: + writer.writerow([r["from_bus"], r["to_bus"], r["ckt"], r["P_flow_MW"]]) + + +def _write_summary_json(path: Path, data: dict) -> None: + """Write a synthetic summary JSON.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2), encoding="utf-8") + + +def _write_intermediate_bus_csv(path: Path, rows: list[dict]) -> None: + """Write a synthetic intermediate format bus CSV.""" + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["bus_i", "base_kv", "area", "type"]) + for r in rows: + writer.writerow( + [ + r.get("bus", r.get("bus_i", 0)), + r.get("base_kv", 0), + r.get("area", 0), + r.get("type", 1), + ] + ) + + +def _write_intermediate_branch_csv(path: Path, rows: list[dict]) -> None: + """Write a synthetic intermediate format branch CSV.""" + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["f_bus", "t_bus", "br_x", "tap", "shift", "br_status", "ckt", "rate_a"]) + for r in rows: + writer.writerow( + [ + r.get("from_bus", 0), + r.get("to_bus", 0), + r.get("x_pu", r.get("br_x", 0.01)), + r.get("tap_ratio", r.get("tap", 0)), + r.get("shift_deg", r.get("shift", 0)), + r.get("status", r.get("br_status", 1)), + r.get("ckt", "1"), + r.get("rate_a", ""), + ] + ) + + +def _make_acpf_summary(slack_bus: int = 1) -> dict: + """Create a minimal ACPF summary JSON dict.""" + return { + "system_summary": { + "total_gen_mw": 1000.0, + "total_load_mw": 950.0, + "total_loss_mw": 50.0, + "slack_bus": slack_bus, + }, + } + + +def _make_dcpf_summary(slack_bus: int = 1) -> dict: + """Create a minimal DCPF summary JSON dict.""" + return { + "settings": { + "slack_bus": slack_bus, + }, + "power_summary": { + "total_generation_mw": 950.0, + "total_load_mw": 950.0, + }, + } + + +def _setup_synthetic_characterization( + tmp_path: Path, + *, + acpf_buses: list[dict] | None = None, + dcpf_buses: list[dict] | None = None, + acpf_branches: list[dict] | None = None, + dcpf_branches: list[dict] | None = None, + intermediate_buses: list[dict] | None = None, + intermediate_branches: list[dict] | None = None, + slack_bus: int = 1, +) -> tuple[Path, Path, Path, Path]: + """Set up synthetic data for build_characterization. + + Returns: + (acpf_dir, dcpf_dir, intermediate_dir, output_dir) + """ + acpf_dir = tmp_path / "acpf" + dcpf_dir = tmp_path / "dcpf" + int_dir = tmp_path / "intermediate" + out_dir = tmp_path / "output" + + # Default buses: 25 matched buses with small deviations + if acpf_buses is None: + acpf_buses = [{"bus": i, "VM": 1.0, "VA": float(i)} for i in range(1, 26)] + if dcpf_buses is None: + dcpf_buses = [{"bus": i, "VA": float(i) + 0.5} for i in range(1, 26)] + + # Default branches: 35 matched branches + if acpf_branches is None: + acpf_branches = [ + { + "from_bus": i, + "to_bus": i + 1, + "ckt": "1", + "P_from": float(50 + i), + "Q_from": 10.0, + "P_to": float(-49 - i), + "Q_to": -9.0, + } + for i in range(1, 36) + ] + if dcpf_branches is None: + dcpf_branches = [ + {"from_bus": i, "to_bus": i + 1, "ckt": "1", "P_flow_MW": float(52 + i)} + for i in range(1, 36) + ] + + # Default intermediate buses + if intermediate_buses is None: + intermediate_buses = [ + {"bus_i": i, "base_kv": 230.0, "area": 1, "type": 3 if i == slack_bus else 1} + for i in range(1, 40) + ] + + # Default intermediate branches + if intermediate_branches is None: + intermediate_branches = [ + { + "from_bus": i, + "to_bus": i + 1, + "br_x": 0.01, + "tap": 0, + "shift": 0, + "br_status": 1, + "ckt": "1", + "rate_a": 500, + } + for i in range(1, 40) + ] + + _write_acpf_bus_csv(acpf_dir / "buses_acpf.csv", acpf_buses) + _write_dcpf_bus_csv(dcpf_dir / "buses_dcpf.csv", dcpf_buses) + _write_acpf_branch_csv(acpf_dir / "branches_acpf.csv", acpf_branches) + _write_dcpf_branch_csv(dcpf_dir / "branches_dcpf.csv", dcpf_branches) + _write_summary_json(acpf_dir / "summary_acpf.json", _make_acpf_summary(slack_bus)) + _write_summary_json(dcpf_dir / "summary_dcpf.json", _make_dcpf_summary(slack_bus)) + _write_intermediate_bus_csv(int_dir / "bus.csv", intermediate_buses) + _write_intermediate_branch_csv(int_dir / "branch.csv", intermediate_branches) + + return acpf_dir, dcpf_dir, int_dir, out_dir + + +# --------------------------------------------------------------------------- +# T01-T03: Join operation tests +# --------------------------------------------------------------------------- + + +class TestJoinBuses: + """T01: test_join_buses_inner_join.""" + + def test_join_buses_inner_join(self) -> None: + """ACPF buses [1..5], DCPF buses [2..6] -> 4 matched (2-5).""" + acpf = [{"bus": i, "VM": 1.0, "VA": float(i)} for i in range(1, 6)] + dcpf = [{"bus": i, "VA": float(i) + 0.1} for i in range(2, 7)] + + matched, summary = join_buses(acpf, dcpf) + + assert len(matched) == 4 + matched_bus_nums = {m["bus"] for m in matched} + assert matched_bus_nums == {2, 3, 4, 5} + + assert summary["buses_in_acpf"] == 5 + assert summary["buses_in_dcpf"] == 5 + assert summary["buses_matched"] == 4 + assert summary["buses_acpf_only"] == 1 + assert summary["buses_dcpf_only"] == 1 + + +class TestJoinBranches: + """T02: test_join_branches_composite_key.""" + + def test_join_branches_composite_key(self) -> None: + """Composite key (from_bus, to_bus, ckt) matching with parallel circuits.""" + acpf = [ + {"from_bus": 1, "to_bus": 2, "ckt": "1", "P_from": 100.0}, + {"from_bus": 1, "to_bus": 2, "ckt": "2", "P_from": 80.0}, + {"from_bus": 3, "to_bus": 4, "ckt": "1", "P_from": 50.0}, + ] + dcpf = [ + {"from_bus": 1, "to_bus": 2, "ckt": "1", "P_flow_MW": 105.0}, + {"from_bus": 1, "to_bus": 2, "ckt": "2", "P_flow_MW": 82.0}, + {"from_bus": 5, "to_bus": 6, "ckt": "1", "P_flow_MW": 30.0}, + ] + + matched, summary = join_branches(acpf, dcpf) + + assert len(matched) == 2 + assert summary["branches_acpf_only"] == 1 + assert summary["branches_dcpf_only"] == 1 + + +class TestJoinBusesEmptyIntersection: + """T03: test_join_buses_empty_intersection_raises.""" + + def test_join_buses_empty_intersection_raises(self, tmp_path: Path) -> None: + """ACPF buses [1,2,3], DCPF buses [4,5,6] -> ValueError on zero match.""" + acpf_buses = [{"bus": i, "VM": 1.0, "VA": float(i)} for i in [1, 2, 3]] + dcpf_buses = [{"bus": i, "VA": float(i)} for i in [4, 5, 6]] + + # Need branches that also won't match, but at least exist + acpf_branches = [ + { + "from_bus": 1, + "to_bus": 2, + "ckt": "1", + "P_from": 100.0, + "Q_from": 0, + "P_to": -100, + "Q_to": 0, + } + for _ in range(35) + ] + dcpf_branches = [ + {"from_bus": 4, "to_bus": 5, "ckt": "1", "P_flow_MW": 100.0} for _ in range(35) + ] + + acpf_dir, dcpf_dir, int_dir, out_dir = _setup_synthetic_characterization( + tmp_path, + acpf_buses=acpf_buses, + dcpf_buses=dcpf_buses, + acpf_branches=acpf_branches, + dcpf_branches=dcpf_branches, + ) + + with pytest.raises(ValueError, match="zero matched buses"): + build_characterization(acpf_dir, dcpf_dir, int_dir, out_dir) + + +# --------------------------------------------------------------------------- +# T04-T06: Deviation computation tests +# --------------------------------------------------------------------------- + + +class TestBusDeviationSigns: + """T04: test_bus_deviation_signs.""" + + def test_bus_deviation_signs(self) -> None: + """VA_acpf=10, VA_dcpf=12 -> delta=2; VA_acpf=-5, VA_dcpf=-7 -> delta=-2.""" + matched = [ + {"bus": 1, "VM_acpf": 1.0, "VA_acpf": 10.0, "VA_dcpf": 12.0}, + {"bus": 2, "VM_acpf": 1.0, "VA_acpf": -5.0, "VA_dcpf": -7.0}, + ] + intermediate_buses: list[dict] = [] + + devs = compute_bus_deviations(matched, intermediate_buses) + + assert len(devs) == 2 + # Bus 1: delta = 12 - 10 = 2.0 + assert devs[0].bus == 1 + assert math.isclose(devs[0].delta_VA_deg, 2.0) + assert math.isclose(devs[0].abs_delta_VA_deg, 2.0) + + # Bus 2: delta = -7 - (-5) = -2.0 + assert devs[1].bus == 2 + assert math.isclose(devs[1].delta_VA_deg, -2.0) + assert math.isclose(devs[1].abs_delta_VA_deg, 2.0) + + +class TestBranchDeviationNearZeroFlow: + """T05: test_branch_deviation_near_zero_flow_excluded.""" + + def test_branch_deviation_near_zero_flow_excluded(self) -> None: + """Branch A (100 MW) gets pct; branch B (0.5 MW) gets None for pct.""" + matched = [ + {"from_bus": 1, "to_bus": 2, "ckt": "1", "P_from_acpf": 100.0, "P_flow_dcpf": 105.0}, + {"from_bus": 3, "to_bus": 4, "ckt": "1", "P_from_acpf": 0.5, "P_flow_dcpf": 1.0}, + ] + intermediate_branches: list[dict] = [] + + devs = compute_branch_deviations(matched, intermediate_branches) + + assert len(devs) == 2 + + # Branch A: delta_P_pct = (105-100)/100 * 100 = 5% + assert math.isclose(devs[0].delta_P_pct, 5.0) # type: ignore[arg-type] + assert math.isclose(devs[0].abs_delta_P_pct, 5.0) # type: ignore[arg-type] + + # Branch B: near-zero flow -> None + assert devs[1].delta_P_pct is None + assert devs[1].abs_delta_P_pct is None + + +class TestBranchDeviationPercentageDirection: + """T06: test_branch_deviation_percentage_direction.""" + + def test_branch_deviation_percentage_direction(self) -> None: + """P_from_acpf=-200, P_flow_dcpf=-180 -> delta_P_MW=20, delta_P_pct=10%.""" + matched = [ + {"from_bus": 1, "to_bus": 2, "ckt": "1", "P_from_acpf": -200.0, "P_flow_dcpf": -180.0}, + ] + intermediate_branches: list[dict] = [] + + devs = compute_branch_deviations(matched, intermediate_branches) + + assert len(devs) == 1 + bd = devs[0] + + # delta_P_MW = -180 - (-200) = 20.0 + assert math.isclose(bd.delta_P_MW, 20.0) + assert math.isclose(bd.abs_delta_P_MW, 20.0) + + # delta_P_pct = 20 / 200 * 100 = 10.0 + assert bd.delta_P_pct is not None + assert math.isclose(bd.delta_P_pct, 10.0) + assert bd.abs_delta_P_pct is not None + assert math.isclose(bd.abs_delta_P_pct, 10.0) + + +# --------------------------------------------------------------------------- +# T07-T09: Aggregate statistics and compliance tests +# --------------------------------------------------------------------------- + + +class TestAggregateStatsKnownDistribution: + """T07: test_aggregate_stats_known_distribution.""" + + def test_aggregate_stats_known_distribution(self) -> None: + """90 values of 1.0 and 10 values of 5.0.""" + values = [1.0] * 90 + [5.0] * 10 + + stats = compute_aggregate_stats(values) + + # mean = (90*1 + 10*5) / 100 = 140/100 = 1.4 + assert math.isclose(stats.mean, 1.4, rel_tol=1e-6) + assert math.isclose(stats.median, 1.0) + assert stats.max == 5.0 + # p95: at index 94.05 in sorted array (all 1.0 up to index 89, then 5.0) + # So p95 should be 5.0 + assert math.isclose(stats.p95, 5.0) + + +class TestComplianceFractions: + """T08: test_compliance_fractions_known_distribution.""" + + def test_compliance_fractions_known_distribution(self) -> None: + """90 values of 1.0, 10 values of 5.0 with thresholds [0.5, 1.0, 2.0, 5.0].""" + values = [1.0] * 90 + [5.0] * 10 + + comp = compute_compliance_fractions(values, [0.5, 1.0, 2.0, 5.0]) + + assert len(comp.fractions) == 4 + # 0% below 0.5 (all values are >= 1.0) + assert math.isclose(comp.fractions[0], 0.0) + # 90% at or below 1.0 + assert math.isclose(comp.fractions[1], 0.90) + # 90% at or below 2.0 + assert math.isclose(comp.fractions[2], 0.90) + # 100% at or below 5.0 + assert math.isclose(comp.fractions[3], 1.0) + + +class TestExpectedRangeCheckWarning: + """T09: test_expected_range_check_warning.""" + + def test_expected_range_check_warning(self, tmp_path: Path) -> None: + """93% within 3 degrees -> expected_range_checks.angle fails, warning emitted.""" + # Create 25 buses: 23 with small deviations, 2 with large deviations + # 23/25 = 92% within 3 degrees + acpf_buses = [{"bus": i, "VM": 1.0, "VA": 0.0} for i in range(1, 26)] + dcpf_buses_data = [] + for i in range(1, 26): + if i <= 23: + dcpf_buses_data.append({"bus": i, "VA": 1.0}) # delta=1.0 deg + else: + dcpf_buses_data.append({"bus": i, "VA": 5.0}) # delta=5.0 deg + + # Branches: 35 with moderate deviations + acpf_branches = [ + { + "from_bus": i, + "to_bus": i + 1, + "ckt": "1", + "P_from": 100.0, + "Q_from": 10.0, + "P_to": -99.0, + "Q_to": -9.0, + } + for i in range(1, 36) + ] + dcpf_branches = [ + {"from_bus": i, "to_bus": i + 1, "ckt": "1", "P_flow_MW": 102.0} for i in range(1, 36) + ] + + acpf_dir, dcpf_dir, int_dir, out_dir = _setup_synthetic_characterization( + tmp_path, + acpf_buses=acpf_buses, + dcpf_buses=dcpf_buses_data, + acpf_branches=acpf_branches, + dcpf_branches=dcpf_branches, + ) + + result_dir = build_characterization(acpf_dir, dcpf_dir, int_dir, out_dir) + + # Read the JSON to verify + json_path = result_dir / "dcpf_vs_acpf_characterization.json" + assert json_path.exists() + data = json.loads(json_path.read_text(encoding="utf-8")) + + angle_check = data["expected_range_checks"]["angle_95pct_within_3deg"] + assert angle_check["met"] is False + + # Verify warnings mention the threshold + assert any("95%" in w and "3 degrees" in w for w in data["warnings"]) + + +# --------------------------------------------------------------------------- +# T10-T11: Cause annotation tests +# --------------------------------------------------------------------------- + + +class TestBranchCausePhaseShifter: + """T10: test_branch_cause_phase_shifter.""" + + def test_branch_cause_phase_shifter(self) -> None: + """Branch with shift_deg=15 -> PHASE_SHIFTER cause.""" + branch_devs = [ + BranchDeviation( + from_bus=1, + to_bus=2, + ckt="1", + P_from_acpf_MW=100.0, + P_flow_dcpf_MW=110.0, + delta_P_MW=10.0, + abs_delta_P_MW=10.0, + delta_P_pct=10.0, + abs_delta_P_pct=10.0, + x_pu=0.05, + tap_ratio=1.0, + shift_deg=15.0, + is_transformer=True, + ), + ] + acpf_buses = [{"bus": 1, "VM": 1.0}, {"bus": 2, "VM": 1.0}] + intermediate_branches: list[dict] = [] + + result = annotate_branch_causes(branch_devs, acpf_buses, intermediate_branches) + + assert len(result) == 1 + assert DeviationCause.PHASE_SHIFTER in result[0].causes + assert result[0].causes[0] == DeviationCause.PHASE_SHIFTER + + +class TestBusCauseLowVoltage: + """T11: test_bus_cause_low_voltage.""" + + def test_bus_cause_low_voltage(self) -> None: + """Bus with VM_acpf=0.92 -> LOW_VOLTAGE cause.""" + bus_devs = [ + BusDeviation( + bus=1, + VA_acpf_deg=10.0, + VA_dcpf_deg=12.0, + delta_VA_deg=2.0, + abs_delta_VA_deg=2.0, + VM_acpf_pu=0.92, + base_kv=115.0, + area=1, + ), + ] + + result = annotate_bus_causes(bus_devs, slack_bus=999, bus_adjacency={}) + + assert len(result) == 1 + assert DeviationCause.LOW_VOLTAGE in result[0].causes + + +# --------------------------------------------------------------------------- +# T12: Output format test +# --------------------------------------------------------------------------- + + +class TestWriteCharacterizationJsonRoundtrip: + """T12: test_write_characterization_json_roundtrip.""" + + def test_write_characterization_json_roundtrip(self, tmp_path: Path) -> None: + """Build synthetic CharacterizationResult, write JSON, read back, verify keys.""" + # Create 20 bus deviations + bus_devs = [ + BusDeviation( + bus=i, + VA_acpf_deg=float(i), + VA_dcpf_deg=float(i) + 0.5, + delta_VA_deg=0.5, + abs_delta_VA_deg=0.5, + VM_acpf_pu=1.0, + base_kv=230.0, + area=1, + causes=[DeviationCause.UNCATEGORIZED], + ) + for i in range(1, 21) + ] + + # Create 30 branch deviations + branch_devs = [ + BranchDeviation( + from_bus=i, + to_bus=i + 1, + ckt="1", + P_from_acpf_MW=100.0, + P_flow_dcpf_MW=105.0, + delta_P_MW=5.0, + abs_delta_P_MW=5.0, + delta_P_pct=5.0, + abs_delta_P_pct=5.0, + x_pu=0.01, + tap_ratio=1.0, + shift_deg=0.0, + is_transformer=False, + causes=[DeviationCause.UNCATEGORIZED], + ) + for i in range(1, 31) + ] + + angle_stats = AggregateStats( + count=20, + mean=0.5, + median=0.5, + std=0.0, + min=0.5, + max=0.5, + p05=0.5, + p95=0.5, + ) + flow_stats = AggregateStats( + count=30, + mean=5.0, + median=5.0, + std=0.0, + min=5.0, + max=5.0, + p05=5.0, + p95=5.0, + ) + angle_comp = ComplianceFractions( + thresholds=[0.5, 1.0, 2.0, 3.0, 5.0, 10.0], + fractions=[1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + ) + flow_comp = ComplianceFractions( + thresholds=[1.0, 2.0, 5.0, 10.0, 20.0, 50.0], + fractions=[0.0, 0.0, 1.0, 1.0, 1.0, 1.0], + ) + + result = CharacterizationResult( + bus_deviations=bus_devs, + branch_deviations=branch_devs, + angle_stats_signed=angle_stats, + angle_stats_absolute=angle_stats, + angle_compliance=angle_comp, + flow_mw_stats_signed=flow_stats, + flow_mw_stats_absolute=flow_stats, + flow_pct_stats_signed=flow_stats, + flow_pct_stats_absolute=flow_stats, + flow_pct_compliance=flow_comp, + join_summary={ + "buses_in_acpf": 20, + "buses_in_dcpf": 20, + "buses_matched": 20, + "buses_acpf_only": 0, + "buses_dcpf_only": 0, + "branches_in_acpf": 30, + "branches_in_dcpf": 30, + "branches_matched": 30, + "branches_acpf_only": 0, + "branches_dcpf_only": 0, + }, + system_level={ + "acpf_total_gen_mw": 1000.0, + "dcpf_total_gen_mw": 950.0, + "acpf_total_load_mw": 950.0, + "dcpf_total_load_mw": 950.0, + "acpf_total_loss_mw": 50.0, + "acpf_loss_pct_of_gen": 5.0, + "acpf_slack_bus": 1.0, + "dcpf_slack_bus": 1.0, + }, + expected_range_checks={ + "angle_95pct_within_3deg": { + "threshold_pct": 95.0, + "threshold_deg": 3.0, + "actual_pct": 100.0, + "met": True, + }, + "flow_90pct_within_10pct": { + "threshold_pct": 90.0, + "threshold_flow_pct": 10.0, + "actual_pct": 100.0, + "met": True, + }, + }, + worst_buses=bus_devs[:5], + worst_branches=branch_devs[:5], + warnings=[], + metadata={ + "acpf_summary_path": "test", + "dcpf_summary_path": "test", + "acpf_buses_path": "test", + "dcpf_buses_path": "test", + "acpf_branches_path": "test", + "dcpf_branches_path": "test", + "intermediate_dir": "test", + "timestamp": "2024-01-01T00:00:00Z", + }, + ) + + json_path = tmp_path / "characterization.json" + write_characterization_json(result, json_path) + + # Read back + data = json.loads(json_path.read_text(encoding="utf-8")) + + # Verify all top-level keys + expected_keys = { + "metadata", + "join_summary", + "system_level", + "angle_deviation", + "flow_deviation_mw", + "flow_deviation_pct", + "expected_range_checks", + "worst_buses", + "worst_branches", + "warnings", + } + assert set(data.keys()) == expected_keys + + # Verify worst_buses has 5 entries + assert len(data["worst_buses"]) == 5 + + # Verify angle compliance value + assert isinstance(data["angle_deviation"]["compliance"]["pct_within_3_0_deg"], float) + assert 0 <= data["angle_deviation"]["compliance"]["pct_within_3_0_deg"] <= 100 + + # Verify near_zero_flow_threshold_mw + assert data["flow_deviation_pct"]["near_zero_flow_threshold_mw"] == 1.0 + + +# --------------------------------------------------------------------------- +# T13-T14: Integration tests (require FNM_PATH and D2/D3 outputs) +# --------------------------------------------------------------------------- + + +@pytest.mark.fnm +class TestFnmCharacterizationProducesReports: + """T13: test_fnm_characterization_produces_reports.""" + + def test_fnm_characterization_produces_reports(self, require_fnm: dict, tmp_path: Path) -> None: + """Run build_characterization with actual ACPF/DCPF references.""" + repo_root = Path(__file__).resolve().parent.parent.parent.parent + acpf_dir = repo_root / "data" / "fnm" / "reference" / "acpf" + dcpf_dir = repo_root / "data" / "fnm" / "reference" / "dcpf" + int_dir = repo_root / "data" / "fnm" / "intermediate" / "canonical" + out_dir = tmp_path / "output" + + result_dir = build_characterization(acpf_dir, dcpf_dir, int_dir, out_dir) + + json_path = result_dir / "dcpf_vs_acpf_characterization.json" + md_path = result_dir / "dcpf_vs_acpf_characterization.md" + assert json_path.exists() + assert md_path.exists() + + data = json.loads(json_path.read_text(encoding="utf-8")) + assert data["join_summary"]["buses_matched"] > 20000 + assert data["join_summary"]["branches_matched"] > 30000 + assert data["angle_deviation"]["count"] > 20000 + assert data["flow_deviation_pct"]["count"] > 20000 + + +@pytest.mark.fnm +class TestFnmCharacterizationExpectedRanges: + """T14: test_fnm_characterization_expected_ranges.""" + + def test_fnm_characterization_expected_ranges(self, require_fnm: dict, tmp_path: Path) -> None: + """Verify characterization meets relaxed floor thresholds.""" + repo_root = Path(__file__).resolve().parent.parent.parent.parent + acpf_dir = repo_root / "data" / "fnm" / "reference" / "acpf" + dcpf_dir = repo_root / "data" / "fnm" / "reference" / "dcpf" + int_dir = repo_root / "data" / "fnm" / "intermediate" / "canonical" + out_dir = tmp_path / "output" + + result_dir = build_characterization(acpf_dir, dcpf_dir, int_dir, out_dir) + + json_path = result_dir / "dcpf_vs_acpf_characterization.json" + data = json.loads(json_path.read_text(encoding="utf-8")) + + # Relaxed floors + assert data["angle_deviation"]["compliance"]["pct_within_3_0_deg"] > 90.0 + assert data["flow_deviation_pct"]["compliance"]["pct_within_10_0_pct"] > 80.0 + + # Worst-case lists + assert len(data["worst_buses"]) == 50 + assert len(data["worst_branches"]) == 50 + + # Every worst-case entry has causes + for wb in data["worst_buses"]: + assert len(wb["all_causes"]) > 0 + for wb in data["worst_branches"]: + assert len(wb["all_causes"]) > 0 diff --git a/data/fnm/tests/test_dcpf_reference.py b/data/fnm/tests/test_dcpf_reference.py new file mode 100644 index 00000000..ff912197 --- /dev/null +++ b/data/fnm/tests/test_dcpf_reference.py @@ -0,0 +1,1001 @@ +"""Tests for DCPF Reference Solution Computation (PRD 03/03). + +All synthetic tests use programmatically created BusRecord, GeneratorRecord, +and BranchRecord instances -- no CSV fixture files needed for pure unit tests. +FNM integration tests are gated by the ``require_fnm`` fixture. +""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path + +import pytest + +from fnm.scripts.dcpf_reference import ( + FLOW_TOLERANCE_MW, + BranchFlow, + BranchRecord, + BusRecord, + DCPFSolution, + GeneratorRecord, + build_b_matrix, + compute_bus_injections, + compute_phase_shift_injections, + filter_active_buses, + identify_slack_bus, + solve_dcpf, + validate_dcpf_solution, + write_buses_csv, + write_summary_json, +) + +# --------------------------------------------------------------------------- +# Helper: build a simple 3-bus triangle system +# --------------------------------------------------------------------------- + + +def _make_3bus_triangle( + x_12: float = 0.1, + x_13: float = 0.3, + x_23: float = 0.2, +) -> tuple[list[BusRecord], list[BranchRecord]]: + """Build a 3-bus triangle: bus 1 = slack, buses 2 and 3 = PQ. + + Branch reactances are configurable. All branches are in service, + tap=1.0, shift=0.0. + """ + buses = [ + BusRecord(bus_number=1, bus_type=3, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=2, bus_type=1, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=3, bus_type=1, pd_mw=0.0, base_kv=345.0), + ] + branches = [ + BranchRecord( + from_bus=1, + to_bus=2, + circuit_id="1", + x_pu=x_12, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + BranchRecord( + from_bus=1, + to_bus=3, + circuit_id="1", + x_pu=x_13, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + BranchRecord( + from_bus=2, + to_bus=3, + circuit_id="1", + x_pu=x_23, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + ] + return buses, branches + + +def _solve_3bus_system( + buses: list[BusRecord], + generators: list[GeneratorRecord], + branches: list[BranchRecord], + base_mva: float = 100.0, +) -> DCPFSolution: + """Helper to run a complete DCPF solve on a small system.""" + excluded: set[int] = set() + active = filter_active_buses(buses, excluded) + slack = identify_slack_bus(active) + injections = compute_bus_injections(active, generators, excluded) + b_result = build_b_matrix(active, branches, excluded, slack, base_mva) + phase_inj = compute_phase_shift_injections(branches, excluded, base_mva) + solution = solve_dcpf(b_result, injections, phase_inj, branches, excluded) + + # Set correct total gen/load + total_gen = sum(g.pg_mw for g in generators if g.status == 1) + total_load = sum(b.pd_mw for b in active) + + return DCPFSolution( + bus_angles_deg=solution.bus_angles_deg, + branch_flows_mw=solution.branch_flows_mw, + total_generation_mw=total_gen, + total_load_mw=total_load, + slack_bus=solution.slack_bus, + slack_injection_mw=injections.get(slack, 0.0), + active_bus_count=solution.active_bus_count, + active_branch_count=solution.active_branch_count, + zero_impedance_branches=solution.zero_impedance_branches, + base_mva=solution.base_mva, + ) + + +# =========================================================================== +# T01: test_build_b_matrix_3bus +# =========================================================================== + + +class TestBuildBMatrix3Bus: + """T01: Construct a 3-bus, 3-branch triangle and verify B' matrix entries.""" + + def test_matrix_dimensions(self) -> None: + """B' should be (N-1) x (N-1) = 2x2 for a 3-bus system.""" + buses, branches = _make_3bus_triangle() + result = build_b_matrix(buses, branches, set(), slack_bus=1, base_mva=100.0) + assert len(result.b_prime) == 2 + assert len(result.b_prime[0]) == 2 + + def test_diagonal_entries(self) -> None: + """Diagonal entries should be the sum of connected susceptances.""" + buses, branches = _make_3bus_triangle(x_12=0.1, x_13=0.3, x_23=0.2) + result = build_b_matrix(buses, branches, set(), slack_bus=1, base_mva=100.0) + + # Bus 2 connects to bus 1 (x=0.1) and bus 3 (x=0.2) + # B'[0,0] for bus 2 = 1/0.1 + 1/0.2 = 10 + 5 = 15.0 + idx2 = result.bus_index_map[2] + assert abs(result.b_prime[idx2][idx2] - 15.0) < 1e-10 + + # Bus 3 connects to bus 1 (x=0.3) and bus 2 (x=0.2) + # B'[1,1] for bus 3 = 1/0.3 + 1/0.2 = 3.333 + 5.0 = 8.333 + idx3 = result.bus_index_map[3] + assert abs(result.b_prime[idx3][idx3] - (1 / 0.3 + 1 / 0.2)) < 1e-10 + + def test_off_diagonal_entries(self) -> None: + """Off-diagonal entries should be the negative susceptance of the + connecting branch.""" + buses, branches = _make_3bus_triangle(x_12=0.1, x_13=0.3, x_23=0.2) + result = build_b_matrix(buses, branches, set(), slack_bus=1, base_mva=100.0) + + idx2 = result.bus_index_map[2] + idx3 = result.bus_index_map[3] + + # Off-diagonal B'[2,3] = B'[3,2] = -1/0.2 = -5.0 + assert abs(result.b_prime[idx2][idx3] - (-1 / 0.2)) < 1e-10 + assert abs(result.b_prime[idx3][idx2] - (-1 / 0.2)) < 1e-10 + + +# =========================================================================== +# T02: test_solve_dcpf_3bus +# =========================================================================== + + +class TestSolveDCPF3Bus: + """T02: Solve a 3-bus triangle with known injections.""" + + def _build(self) -> DCPFSolution: + buses, branches = _make_3bus_triangle() + generators = [ + GeneratorRecord(bus_number=2, pg_mw=100.0, status=1, machine_id="1"), + ] + # Bus 3 has 100 MW load + buses_with_load = [ + BusRecord(bus_number=1, bus_type=3, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=2, bus_type=1, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=3, bus_type=1, pd_mw=100.0, base_kv=345.0), + ] + return _solve_3bus_system(buses_with_load, generators, branches) + + def test_slack_angle_zero(self) -> None: + solution = self._build() + assert solution.bus_angles_deg[1] == 0.0 + + def test_non_slack_angles_nonzero(self) -> None: + solution = self._build() + assert solution.bus_angles_deg[2] != 0.0 + assert solution.bus_angles_deg[3] != 0.0 + + def test_power_balance(self) -> None: + solution = self._build() + assert abs(solution.total_generation_mw - solution.total_load_mw) < FLOW_TOLERANCE_MW + + def test_branch_flow_injection_consistency(self) -> None: + """Sum of branch flows into each non-slack bus should equal its injection.""" + solution = self._build() + # Bus 2: injection = +100 MW (gen) - 0 (load) = +100 MW + # Bus 3: injection = 0 (gen) - 100 (load) = -100 MW + # Net flow into bus 2 = sum of flows where bus 2 is the to_bus minus + # sum of flows where bus 2 is from_bus + for bus_num, expected_inj in [(2, 100.0), (3, -100.0)]: + net_flow = 0.0 + for flow in solution.branch_flows_mw: + if flow.from_bus == bus_num: + net_flow -= flow.p_flow_mw + elif flow.to_bus == bus_num: + net_flow += flow.p_flow_mw + # Net flow out of bus = injection (generation - load) + # Net flow in = -injection for the bus + # Actually: sum of P_from for branches FROM this bus + + # sum of P_to (= -P_from) for branches TO this bus + # should equal the injection. + # P_flow_mw is positive from->to, so: + # For bus i: injection = sum(P_flow for branches FROM i) - sum(P_flow for branches TO i) + net_out = 0.0 + for flow in solution.branch_flows_mw: + if flow.from_bus == bus_num: + net_out += flow.p_flow_mw + elif flow.to_bus == bus_num: + net_out -= flow.p_flow_mw + assert abs(net_out - expected_inj) < FLOW_TOLERANCE_MW + + +# =========================================================================== +# T03: test_zero_impedance_branch_replacement +# =========================================================================== + + +class TestZeroImpedanceBranch: + """T03: Verify zero-impedance branches are handled correctly.""" + + def test_replacement_applied(self) -> None: + """A branch with X=0 should be replaced with ZERO_IMPEDANCE_REPLACEMENT.""" + buses = [ + BusRecord(bus_number=1, bus_type=3, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=2, bus_type=1, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=3, bus_type=1, pd_mw=0.0, base_kv=345.0), + ] + branches = [ + BranchRecord( + from_bus=1, + to_bus=2, + circuit_id="1", + x_pu=0.0, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + BranchRecord( + from_bus=2, + to_bus=3, + circuit_id="1", + x_pu=0.1, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + ] + result = build_b_matrix(buses, branches, set(), slack_bus=1, base_mva=100.0) + assert len(result.zero_impedance_branches) == 1 + assert result.zero_impedance_branches[0] == (1, 2, "1") + + def test_near_identical_angles(self) -> None: + """Buses connected by a zero-impedance branch should have nearly + identical angles (< 0.01 degrees difference).""" + buses = [ + BusRecord(bus_number=1, bus_type=3, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=2, bus_type=1, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=3, bus_type=1, pd_mw=50.0, base_kv=345.0), + ] + generators = [ + GeneratorRecord(bus_number=2, pg_mw=50.0, status=1, machine_id="1"), + ] + branches = [ + BranchRecord( + from_bus=1, + to_bus=2, + circuit_id="1", + x_pu=0.0, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + BranchRecord( + from_bus=2, + to_bus=3, + circuit_id="1", + x_pu=0.1, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + ] + solution = _solve_3bus_system(buses, generators, branches) + # Bus 1 (slack) angle = 0.0, bus 2 should be very close to 0.0 + angle_diff = abs(solution.bus_angles_deg[1] - solution.bus_angles_deg[2]) + assert angle_diff < 0.01 + + +# =========================================================================== +# T04: test_out_of_service_branch_excluded +# =========================================================================== + + +class TestOutOfServiceBranch: + """T04: Out-of-service branches are excluded from B-matrix and flows.""" + + def test_excluded_branch_count(self) -> None: + buses = [ + BusRecord(bus_number=1, bus_type=3, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=2, bus_type=1, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=3, bus_type=1, pd_mw=0.0, base_kv=345.0), + ] + branches = [ + BranchRecord( + from_bus=1, + to_bus=2, + circuit_id="1", + x_pu=0.1, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + BranchRecord( + from_bus=2, + to_bus=3, + circuit_id="1", + x_pu=0.2, + tap_ratio=1.0, + shift_deg=0.0, + status=0, + is_transformer=False, + ), + ] + result = build_b_matrix(buses, branches, set(), slack_bus=1, base_mva=100.0) + assert result.excluded_branch_count == 1 + + def test_b_matrix_reflects_single_branch(self) -> None: + """With one of two branches out of service, diagonal should reflect + only the in-service branch.""" + buses = [ + BusRecord(bus_number=1, bus_type=3, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=2, bus_type=1, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=3, bus_type=1, pd_mw=0.0, base_kv=345.0), + ] + branches = [ + BranchRecord( + from_bus=1, + to_bus=2, + circuit_id="1", + x_pu=0.1, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + BranchRecord( + from_bus=2, + to_bus=3, + circuit_id="1", + x_pu=0.2, + tap_ratio=1.0, + shift_deg=0.0, + status=0, + is_transformer=False, + ), + ] + result = build_b_matrix(buses, branches, set(), slack_bus=1, base_mva=100.0) + idx2 = result.bus_index_map[2] + # Bus 2 only has the in-service branch to bus 1 (x=0.1) + assert abs(result.b_prime[idx2][idx2] - 1 / 0.1) < 1e-10 + + def test_out_of_service_not_in_flows(self) -> None: + """A 3-bus triangle with one branch out of service. The out-of-service + branch should not appear in flows, but the network stays connected + via the other two branches.""" + buses = [ + BusRecord(bus_number=1, bus_type=3, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=2, bus_type=1, pd_mw=50.0, base_kv=345.0), + BusRecord(bus_number=3, bus_type=1, pd_mw=0.0, base_kv=345.0), + ] + generators = [ + GeneratorRecord(bus_number=1, pg_mw=50.0, status=1, machine_id="1"), + ] + branches = [ + BranchRecord( + from_bus=1, + to_bus=2, + circuit_id="1", + x_pu=0.1, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + BranchRecord( + from_bus=1, + to_bus=3, + circuit_id="1", + x_pu=0.2, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + BranchRecord( + from_bus=2, + to_bus=3, + circuit_id="1", + x_pu=0.2, + tap_ratio=1.0, + shift_deg=0.0, + status=0, + is_transformer=False, + ), + ] + solution = _solve_3bus_system(buses, generators, branches) + # Only 2 in-service branches should appear in flows + assert len(solution.branch_flows_mw) == 2 + # The out-of-service branch (2->3) should not appear + flow_pairs = {(f.from_bus, f.to_bus) for f in solution.branch_flows_mw} + assert (2, 3) not in flow_pairs + + +# =========================================================================== +# T05: test_transformer_tap_ratio_in_b_matrix +# =========================================================================== + + +class TestTransformerTapRatio: + """T05: Verify tap-adjusted susceptance in B-matrix.""" + + def test_tap_adjusted_diagonal(self) -> None: + """From-side diagonal should include 1/(X*t^2), to-side 1/X.""" + buses = [ + BusRecord(bus_number=1, bus_type=3, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=2, bus_type=1, pd_mw=0.0, base_kv=345.0), + ] + x = 0.05 + t = 1.05 + branches = [ + BranchRecord( + from_bus=1, + to_bus=2, + circuit_id="1", + x_pu=x, + tap_ratio=t, + shift_deg=0.0, + status=1, + is_transformer=True, + ), + ] + result = build_b_matrix(buses, branches, set(), slack_bus=1, base_mva=100.0) + idx2 = result.bus_index_map[2] + # Bus 2 is the to-side: diagonal = 1/X + assert abs(result.b_prime[idx2][idx2] - 1 / x) < 1e-10 + + def test_tap_differs_from_unity(self) -> None: + """B-matrix with tap=1.05 should differ from tap=1.0.""" + x = 0.05 + + # Use a 3-bus system where bus 2 is the from-side of the transformer + # to bus 3, so the tap-adjusted diagonal appears in the matrix. + buses_swap = [ + BusRecord(bus_number=1, bus_type=3, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=2, bus_type=1, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=3, bus_type=1, pd_mw=0.0, base_kv=345.0), + ] + branches_swap_tap1 = [ + BranchRecord( + from_bus=2, + to_bus=3, + circuit_id="1", + x_pu=x, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + BranchRecord( + from_bus=1, + to_bus=2, + circuit_id="1", + x_pu=0.1, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + ] + branches_swap_tap105 = [ + BranchRecord( + from_bus=2, + to_bus=3, + circuit_id="1", + x_pu=x, + tap_ratio=1.05, + shift_deg=0.0, + status=1, + is_transformer=True, + ), + BranchRecord( + from_bus=1, + to_bus=2, + circuit_id="1", + x_pu=0.1, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + ] + + r_tap1 = build_b_matrix(buses_swap, branches_swap_tap1, set(), slack_bus=1, base_mva=100.0) + r_tap105 = build_b_matrix( + buses_swap, branches_swap_tap105, set(), slack_bus=1, base_mva=100.0 + ) + + # Bus 2 is from-side of the transformer to bus 3 + idx2_t1 = r_tap1.bus_index_map[2] + idx2_t105 = r_tap105.bus_index_map[2] + + # With tap=1.0: from-side diagonal contribution = 1/X = 20 + # With tap=1.05: from-side diagonal contribution = 1/(X*t^2) = 1/(0.05*1.1025) ~18.14 + # Plus the branch to bus 1: 1/0.1 = 10 for both + diag_tap1 = r_tap1.b_prime[idx2_t1][idx2_t1] + diag_tap105 = r_tap105.b_prime[idx2_t105][idx2_t105] + assert diag_tap1 != diag_tap105 + + +# =========================================================================== +# T06: test_phase_shifter_injection +# =========================================================================== + + +class TestPhaseShifterInjection: + """T06: Phase-shifting transformer injection modifications.""" + + def test_opposite_sign_injections(self) -> None: + """Phase shift should produce opposite-sign injections at endpoints.""" + branches = [ + BranchRecord( + from_bus=1, + to_bus=2, + circuit_id="1", + x_pu=0.1, + tap_ratio=1.0, + shift_deg=10.0, + status=1, + is_transformer=True, + ), + ] + inj = compute_phase_shift_injections(branches, set(), base_mva=100.0) + assert 1 in inj + assert 2 in inj + # Opposite signs + assert inj[1] * inj[2] < 0 + # |values| should be equal + assert abs(abs(inj[1]) - abs(inj[2])) < 1e-10 + + def test_full_solve_with_phase_shifter(self) -> None: + """The DCPF solution should reflect the phase shift in branch flow.""" + buses = [ + BusRecord(bus_number=1, bus_type=3, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=2, bus_type=1, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=3, bus_type=1, pd_mw=0.0, base_kv=345.0), + ] + # No net injection -- all flow is driven by the phase shifter + generators: list[GeneratorRecord] = [] + branches = [ + BranchRecord( + from_bus=1, + to_bus=2, + circuit_id="1", + x_pu=0.1, + tap_ratio=1.0, + shift_deg=10.0, + status=1, + is_transformer=True, + ), + BranchRecord( + from_bus=2, + to_bus=3, + circuit_id="1", + x_pu=0.1, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + BranchRecord( + from_bus=1, + to_bus=3, + circuit_id="1", + x_pu=0.1, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + ] + solution = _solve_3bus_system(buses, generators, branches) + + # Find the flow on the phase-shifting branch (1->2) + ps_flow = None + for f in solution.branch_flows_mw: + if f.from_bus == 1 and f.to_bus == 2: + ps_flow = f + break + assert ps_flow is not None + # Flow should be non-zero (driven by phase shift) + assert abs(ps_flow.p_flow_mw) > 0.1 + + +# =========================================================================== +# T07: test_parallel_branches +# =========================================================================== + + +class TestParallelBranches: + """T07: Two parallel branches with different reactances.""" + + def test_both_branches_in_flows(self) -> None: + """Both parallel branches should appear in the flow results.""" + buses = [ + BusRecord(bus_number=1, bus_type=3, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=2, bus_type=1, pd_mw=100.0, base_kv=345.0), + ] + generators = [ + GeneratorRecord(bus_number=1, pg_mw=100.0, status=1, machine_id="1"), + ] + branches = [ + BranchRecord( + from_bus=1, + to_bus=2, + circuit_id="1", + x_pu=0.1, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + BranchRecord( + from_bus=1, + to_bus=2, + circuit_id="2", + x_pu=0.2, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + ] + solution = _solve_3bus_system(buses, generators, branches) + assert len(solution.branch_flows_mw) == 2 + + def test_flows_inversely_proportional_to_reactance(self) -> None: + """Flow should split inversely proportional to reactance.""" + buses = [ + BusRecord(bus_number=1, bus_type=3, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=2, bus_type=1, pd_mw=100.0, base_kv=345.0), + ] + generators = [ + GeneratorRecord(bus_number=1, pg_mw=100.0, status=1, machine_id="1"), + ] + x1, x2 = 0.1, 0.2 + branches = [ + BranchRecord( + from_bus=1, + to_bus=2, + circuit_id="1", + x_pu=x1, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + BranchRecord( + from_bus=1, + to_bus=2, + circuit_id="2", + x_pu=x2, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + ] + solution = _solve_3bus_system(buses, generators, branches) + + flows = sorted(solution.branch_flows_mw, key=lambda f: f.circuit_id) + f1 = flows[0].p_flow_mw # circuit "1", x=0.1 + f2 = flows[1].p_flow_mw # circuit "2", x=0.2 + + # f1/f2 should equal x2/x1 = 2.0 + assert abs(f1 / f2 - x2 / x1) < 1e-6 + + def test_sum_of_flows_equals_injection(self) -> None: + """Total flow across parallel branches should equal bus injection.""" + buses = [ + BusRecord(bus_number=1, bus_type=3, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=2, bus_type=1, pd_mw=100.0, base_kv=345.0), + ] + generators = [ + GeneratorRecord(bus_number=1, pg_mw=100.0, status=1, machine_id="1"), + ] + branches = [ + BranchRecord( + from_bus=1, + to_bus=2, + circuit_id="1", + x_pu=0.1, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + BranchRecord( + from_bus=1, + to_bus=2, + circuit_id="2", + x_pu=0.2, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + ] + solution = _solve_3bus_system(buses, generators, branches) + total_flow = sum(f.p_flow_mw for f in solution.branch_flows_mw) + # Total flow from bus 1 to bus 2 should equal 100 MW (load at bus 2) + assert abs(total_flow - 100.0) < FLOW_TOLERANCE_MW + + +# =========================================================================== +# T08: test_validate_dcpf_consistent_solution +# =========================================================================== + + +class TestValidateConsistentSolution: + """T08: Validate a correctly solved DCPF returns all_checks_passed.""" + + def test_all_checks_passed(self) -> None: + buses, branches = _make_3bus_triangle() + buses_with_load = [ + BusRecord(bus_number=1, bus_type=3, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=2, bus_type=1, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=3, bus_type=1, pd_mw=100.0, base_kv=345.0), + ] + generators = [ + GeneratorRecord(bus_number=2, pg_mw=100.0, status=1, machine_id="1"), + ] + solution = _solve_3bus_system(buses_with_load, generators, branches) + validation = validate_dcpf_solution(solution) + + assert validation.all_checks_passed is True + assert validation.power_balance_ok is True + assert validation.power_balance_residual_mw < FLOW_TOLERANCE_MW + assert validation.flow_angle_consistency_ok is True + assert validation.flow_angle_max_deviation_mw < FLOW_TOLERANCE_MW + assert validation.slack_angle_zero is True + + +# =========================================================================== +# T09: test_validate_dcpf_detects_inconsistency +# =========================================================================== + + +class TestValidateDetectsInconsistency: + """T09: Validate detects intentionally wrong branch flows.""" + + def test_wrong_flows_detected(self) -> None: + """Constructing a solution with all flows set to 0 while angles are + non-zero should fail flow-angle consistency.""" + buses, branches = _make_3bus_triangle() + buses_with_load = [ + BusRecord(bus_number=1, bus_type=3, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=2, bus_type=1, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=3, bus_type=1, pd_mw=100.0, base_kv=345.0), + ] + generators = [ + GeneratorRecord(bus_number=2, pg_mw=100.0, status=1, machine_id="1"), + ] + good_solution = _solve_3bus_system(buses_with_load, generators, branches) + + # Create bad branch flows (all zero) + bad_flows = [ + BranchFlow( + from_bus=f.from_bus, + to_bus=f.to_bus, + circuit_id=f.circuit_id, + p_flow_mw=0.0, # Intentionally wrong + angle_diff_deg=f.angle_diff_deg, + x_pu=f.x_pu, + is_zero_impedance_replaced=f.is_zero_impedance_replaced, + ) + for f in good_solution.branch_flows_mw + ] + + bad_solution = DCPFSolution( + bus_angles_deg=good_solution.bus_angles_deg, + branch_flows_mw=bad_flows, + total_generation_mw=good_solution.total_generation_mw, + total_load_mw=good_solution.total_load_mw, + slack_bus=good_solution.slack_bus, + slack_injection_mw=good_solution.slack_injection_mw, + active_bus_count=good_solution.active_bus_count, + active_branch_count=good_solution.active_branch_count, + zero_impedance_branches=good_solution.zero_impedance_branches, + base_mva=good_solution.base_mva, + ) + + validation = validate_dcpf_solution(bad_solution) + assert validation.flow_angle_consistency_ok is False + assert validation.all_checks_passed is False + assert validation.flow_angle_max_deviation_mw > FLOW_TOLERANCE_MW + + +# =========================================================================== +# T10: test_write_buses_csv_schema +# =========================================================================== + + +class TestWriteBusesCSV: + """T10: Verify buses_dcpf.csv output schema and content.""" + + def test_schema_and_content(self, tmp_path: Path) -> None: + # Build a 5-bus system + buses = [ + BusRecord(bus_number=1, bus_type=3, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=2, bus_type=1, pd_mw=20.0, base_kv=345.0), + BusRecord(bus_number=3, bus_type=1, pd_mw=30.0, base_kv=345.0), + BusRecord(bus_number=4, bus_type=1, pd_mw=25.0, base_kv=345.0), + BusRecord(bus_number=5, bus_type=1, pd_mw=25.0, base_kv=345.0), + ] + generators = [ + GeneratorRecord(bus_number=1, pg_mw=100.0, status=1, machine_id="1"), + ] + branches = [ + BranchRecord( + from_bus=1, + to_bus=2, + circuit_id="1", + x_pu=0.1, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + BranchRecord( + from_bus=2, + to_bus=3, + circuit_id="1", + x_pu=0.15, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + BranchRecord( + from_bus=3, + to_bus=4, + circuit_id="1", + x_pu=0.2, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + BranchRecord( + from_bus=4, + to_bus=5, + circuit_id="1", + x_pu=0.1, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + BranchRecord( + from_bus=1, + to_bus=5, + circuit_id="1", + x_pu=0.25, + tap_ratio=1.0, + shift_deg=0.0, + status=1, + is_transformer=False, + ), + ] + solution = _solve_3bus_system(buses, generators, branches) + + csv_path = tmp_path / "buses_dcpf.csv" + write_buses_csv(solution, csv_path) + + # Read back + with open(csv_path, encoding="utf-8") as f: + reader = csv.DictReader(f) + rows = list(reader) + + # Check columns + assert list(rows[0].keys()) == ["bus", "VA"] + + # Check row count + assert len(rows) == 5 + + # Check sorted by bus ascending + bus_nums = [int(r["bus"]) for r in rows] + assert bus_nums == sorted(bus_nums) + + # Check slack bus has VA == 0.0 + slack_row = [r for r in rows if int(r["bus"]) == 1][0] + assert float(slack_row["VA"]) == 0.0 + + +# =========================================================================== +# T11: test_write_summary_json_schema +# =========================================================================== + + +class TestWriteSummaryJSON: + """T11: Verify summary_dcpf.json output schema.""" + + def test_schema(self, tmp_path: Path) -> None: + buses, branches = _make_3bus_triangle() + buses_with_load = [ + BusRecord(bus_number=1, bus_type=3, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=2, bus_type=1, pd_mw=0.0, base_kv=345.0), + BusRecord(bus_number=3, bus_type=1, pd_mw=100.0, base_kv=345.0), + ] + generators = [ + GeneratorRecord(bus_number=2, pg_mw=100.0, status=1, machine_id="1"), + ] + solution = _solve_3bus_system(buses_with_load, generators, branches) + validation = validate_dcpf_solution(solution) + + json_path = tmp_path / "summary_dcpf.json" + write_summary_json(solution, validation, json_path) + + with open(json_path, encoding="utf-8") as f: + data = json.load(f) + + # Check all top-level keys + expected_keys = { + "solver", + "formulation", + "base_mva", + "settings", + "network_summary", + "power_summary", + "angle_summary", + "validation", + "zero_impedance_branches", + } + assert expected_keys.issubset(set(data.keys())) + + # Solver value + assert data["solver"] == "stdlib_gaussian_elimination" + + # Validation.all_checks_passed is a boolean + assert isinstance(data["validation"]["all_checks_passed"], bool) + + +# =========================================================================== +# T12-T14: FNM integration tests (gated by require_fnm) +# =========================================================================== + + +@pytest.mark.fnm +class TestFnmDCPFCompletes: + """T12: Run DCPF on actual FNM data and verify completion.""" + + def test_completes(self, require_fnm: object, tmp_path: Path) -> None: + pytest.skip("FNM integration tests require FNM_PATH and D1/D6 outputs") + + +@pytest.mark.fnm +class TestFnmDCPFValidation: + """T13: Run FNM DCPF and verify validation passes.""" + + def test_validation_passes(self, require_fnm: object, tmp_path: Path) -> None: + pytest.skip("FNM integration tests require FNM_PATH and D1/D6 outputs") + + +@pytest.mark.fnm +class TestFnmDCPFOutputRowCounts: + """T14: Verify FNM DCPF output row counts and no NaN/inf values.""" + + def test_output_row_counts(self, require_fnm: object, tmp_path: Path) -> None: + pytest.skip("FNM integration tests require FNM_PATH and D1/D6 outputs") diff --git a/data/fnm/tests/test_field_criticality_matrix.py b/data/fnm/tests/test_field_criticality_matrix.py new file mode 100644 index 00000000..3dcf0b86 --- /dev/null +++ b/data/fnm/tests/test_field_criticality_matrix.py @@ -0,0 +1,555 @@ +"""Structural validation tests for the field criticality matrix document. + +Tests verify that data/fnm/docs/field-criticality-matrix.md contains all required +sections, classifies every schema field, maintains count consistency, and respects +Phase 1 D7 JSON Schema annotations (x-psse-present-but-inactive, x-psse-preservation-critical) +as specified in PRD 02/05. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +# --------------------------------------------------------------------------- +# Paths and constants +# --------------------------------------------------------------------------- + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +DOC_PATH = REPO_ROOT / "data" / "fnm" / "docs" / "field-criticality-matrix.md" +SCHEMAS_DIR = REPO_ROOT / "data" / "fnm" / "intermediate" / "schemas" +MAPPING_GUIDE_PATH = REPO_ROOT / "data" / "fnm" / "docs" / "mapping-guide.md" + +VALID_TIERS = {"DCPF-critical", "ACPF-critical", "Informational", "Discardable"} + +# Schema filename to display name mapping (matches the document H2 headings) +SCHEMA_DISPLAY_NAMES: dict[str, str] = { + "bus": "Bus", + "load": "Load", + "fixed_shunt": "Fixed Shunt", + "generator": "Generator", + "branch": "Branch", + "transformer": "Transformer", + "area": "Area", + "two_terminal_dc": "Two-Terminal DC", + "vsc_dc": "VSC DC", + "impedance_correction": "Impedance Correction", + "multi_terminal_dc": "Multi-Terminal DC", + "multi_section_line": "Multi-Section Line", + "zone": "Zone", + "interarea_transfer": "Interarea Transfer", + "owner": "Owner", + "facts": "FACTS", + "switched_shunt": "Switched Shunt", +} + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _load_schema(name: str) -> dict: + """Load a JSON Schema file by record-type name (without .schema.json).""" + path = SCHEMAS_DIR / f"{name}.schema.json" + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def _non_manifest_schemas() -> list[str]: + """Return sorted list of non-manifest schema basenames (without extension).""" + schemas = [] + for p in sorted(SCHEMAS_DIR.glob("*.schema.json")): + name = p.stem.replace(".schema", "") + if name != "manifest": + schemas.append(name) + return schemas + + +def _split_h2_sections(text: str) -> dict[str, str]: + """Split markdown text into H2 sections keyed by heading.""" + sections: dict[str, str] = {} + current_heading = "" + current_lines: list[str] = [] + for line in text.splitlines(): + if line.startswith("## ") and not line.startswith("### "): + if current_heading: + sections[current_heading] = "\n".join(current_lines) + current_heading = line[3:].strip() + current_lines = [] + else: + current_lines.append(line) + if current_heading: + sections[current_heading] = "\n".join(current_lines) + return sections + + +def _parse_field_table(section_body: str) -> list[dict[str, str]]: + """Parse a pipe-delimited markdown table from a section body. + + Returns a list of dicts with keys: Field, Type, Tier, Rationale. + """ + rows: list[dict[str, str]] = [] + in_table = False + header_found = False + for line in section_body.splitlines(): + stripped = line.strip() + if not stripped.startswith("|"): + if in_table: + break + continue + parts = [p.strip() for p in stripped.split("|")] + parts = [p for p in parts if p] + if len(parts) < 4: + continue + if "Field" in parts[0] and "Type" in parts[1] and "Tier" in parts[2]: + header_found = True + continue + if header_found and stripped.startswith("|--"): + in_table = True + continue + if in_table: + # Strip backticks from field name + field = parts[0].strip("`").strip() + rows.append( + { + "Field": field, + "Type": parts[1], + "Tier": parts[2], + "Rationale": parts[3] if len(parts) > 3 else "", + } + ) + return rows + + +def _parse_summary_table(section_body: str) -> list[dict[str, str | int]]: + """Parse the summary table from the Summary section body. + + Returns a list of dicts with keys: Record Type, Total, DCPF-Critical, + ACPF-Critical, Informational, Discardable. + """ + rows: list[dict[str, str | int]] = [] + in_table = False + header_found = False + for line in section_body.splitlines(): + stripped = line.strip() + if not stripped.startswith("|"): + if in_table: + break + continue + parts = [p.strip() for p in stripped.split("|")] + parts = [p for p in parts if p] + if len(parts) < 6: + continue + if "Record Type" in parts[0] and "Total" in parts[1]: + header_found = True + continue + if header_found and stripped.startswith("|--"): + in_table = True + continue + if in_table: + # Strip markdown bold + rt = parts[0].replace("**", "").strip() + try: + total = int(parts[1].replace("**", "").strip()) + dcpf = int(parts[2].replace("**", "").strip()) + acpf = int(parts[3].replace("**", "").strip()) + info = int(parts[4].replace("**", "").strip()) + disc = int(parts[5].replace("**", "").strip()) + except ValueError: + continue + rows.append( + { + "Record Type": rt, + "Total": total, + "DCPF-Critical": dcpf, + "ACPF-Critical": acpf, + "Informational": info, + "Discardable": disc, + } + ) + return rows + + +def _get_tier3_record_types() -> set[str]: + """Parse the mapping guide to find Tier 3 record types. + + Returns a set of display names (e.g., 'Zone', 'Owner'). + """ + if not MAPPING_GUIDE_PATH.exists(): + return set() + text = MAPPING_GUIDE_PATH.read_text(encoding="utf-8") + tier3: set[str] = set() + for line in text.splitlines(): + stripped = line.strip() + if not stripped.startswith("|"): + continue + parts = [p.strip() for p in stripped.split("|")] + parts = [p for p in parts if p] + if len(parts) < 5: + continue + # Look for rows where the Tier column (index 3) is "3" + if parts[3].strip() == "3": + tier3.add(parts[1].strip()) + return tier3 + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def doc_text() -> str: + """Read the field criticality matrix document.""" + assert DOC_PATH.exists(), f"Document not found at {DOC_PATH}" + return DOC_PATH.read_text(encoding="utf-8") + + +@pytest.fixture(scope="module") +def doc_sections(doc_text: str) -> dict[str, str]: + """Split the document into H2 sections.""" + return _split_h2_sections(doc_text) + + +@pytest.fixture(scope="module") +def schema_names() -> list[str]: + """Return all non-manifest schema basenames.""" + return _non_manifest_schemas() + + +@pytest.fixture(scope="module") +def record_type_field_tables(doc_sections: dict[str, str]) -> dict[str, list[dict[str, str]]]: + """Parse field classification tables for all record-type sections.""" + tables: dict[str, list[dict[str, str]]] = {} + for display_name in SCHEMA_DISPLAY_NAMES.values(): + if display_name in doc_sections: + tables[display_name] = _parse_field_table(doc_sections[display_name]) + return tables + + +@pytest.fixture(scope="module") +def summary_rows(doc_sections: dict[str, str]) -> list[dict[str, str | int]]: + """Parse summary table rows.""" + assert "Summary" in doc_sections, "Summary section not found" + return _parse_summary_table(doc_sections["Summary"]) + + +# --------------------------------------------------------------------------- +# T01-T05: Structural validation tests +# --------------------------------------------------------------------------- + + +@pytest.mark.docs +def test_document_exists() -> None: + """T01: Verify the document exists and is non-empty.""" + assert DOC_PATH.exists(), f"Document not found at {DOC_PATH}" + content = DOC_PATH.read_text(encoding="utf-8") + assert len(content.strip()) > 0, "Document is empty" + + +@pytest.mark.docs +def test_tier_definitions_section_present(doc_sections: dict[str, str]) -> None: + """T02: Verify Tier Definitions section with exactly 4 tier rows.""" + assert "Tier Definitions" in doc_sections, "Missing '## Tier Definitions' section" + body = doc_sections["Tier Definitions"] + # Check that all four tier labels appear + for tier_label in ("DCPF-critical", "ACPF-critical", "Informational", "Discardable"): + assert tier_label in body, f"Tier label '{tier_label}' not found in Tier Definitions" + # Count table data rows (exclude header and separator) + tier_rows = 0 + in_table = False + header_found = False + for line in body.splitlines(): + stripped = line.strip() + if not stripped.startswith("|"): + continue + if "Tier" in stripped and "Label" in stripped: + header_found = True + continue + if header_found and stripped.startswith("|--"): + in_table = True + continue + if in_table: + tier_rows += 1 + assert tier_rows == 4, f"Expected 4 tier definition rows, found {tier_rows}" + + +@pytest.mark.docs +def test_summary_table_present_and_complete( + summary_rows: list[dict[str, str | int]], + schema_names: list[str], +) -> None: + """T03: Verify summary table has one row per non-empty record type plus grand total.""" + summary_rt_names = {str(r["Record Type"]) for r in summary_rows} + # Should have a 'Total' row + assert "Total" in summary_rt_names, "Missing grand total row in summary table" + # All non-manifest schemas should have a summary row + for schema_name in schema_names: + display = SCHEMA_DISPLAY_NAMES.get(schema_name, schema_name) + assert display in summary_rt_names, ( + f"Missing summary row for record type '{display}' (schema: {schema_name})" + ) + # Verify required columns exist (implicitly verified by parsing) + for row in summary_rows: + for col in ( + "Record Type", + "Total", + "DCPF-Critical", + "ACPF-Critical", + "Informational", + "Discardable", + ): + assert col in row, f"Missing column '{col}' in summary row {row}" + + +@pytest.mark.docs +def test_every_non_empty_record_type_has_section( + doc_sections: dict[str, str], + schema_names: list[str], +) -> None: + """T04: Verify every non-manifest schema has an H2 section.""" + for schema_name in schema_names: + display = SCHEMA_DISPLAY_NAMES.get(schema_name, schema_name) + assert display in doc_sections, ( + f"Missing H2 section for record type '{display}' (schema: {schema_name})" + ) + + +@pytest.mark.docs +def test_field_tables_have_required_columns( + record_type_field_tables: dict[str, list[dict[str, str]]], +) -> None: + """T05: Verify each field table has Field, Type, Tier, Rationale columns.""" + for rt_name, rows in record_type_field_tables.items(): + assert len(rows) > 0, f"No field rows found in table for '{rt_name}'" + for row in rows: + for col in ("Field", "Type", "Tier", "Rationale"): + assert col in row, f"Missing column '{col}' in field table for '{rt_name}'" + + +# --------------------------------------------------------------------------- +# T06-T08: Field coverage tests +# --------------------------------------------------------------------------- + + +@pytest.mark.docs +def test_all_schema_fields_classified( + record_type_field_tables: dict[str, list[dict[str, str]]], + schema_names: list[str], +) -> None: + """T06: Verify every JSON Schema property appears in the classification table.""" + for schema_name in schema_names: + schema = _load_schema(schema_name) + schema_fields = set(schema.get("properties", {}).keys()) + display = SCHEMA_DISPLAY_NAMES.get(schema_name, schema_name) + assert display in record_type_field_tables, ( + f"No field table found for '{display}' (schema: {schema_name})" + ) + doc_fields = {row["Field"] for row in record_type_field_tables[display]} + missing = schema_fields - doc_fields + assert not missing, f"Fields missing from '{display}' table: {sorted(missing)}" + + +@pytest.mark.docs +def test_no_duplicate_fields_per_record_type( + record_type_field_tables: dict[str, list[dict[str, str]]], +) -> None: + """T07: Verify no field name appears more than once in a single record type.""" + for rt_name, rows in record_type_field_tables.items(): + fields = [row["Field"] for row in rows] + seen: set[str] = set() + duplicates: set[str] = set() + for f in fields: + if f in seen: + duplicates.add(f) + seen.add(f) + assert not duplicates, f"Duplicate fields in '{rt_name}': {sorted(duplicates)}" + + +@pytest.mark.docs +def test_all_tier_values_valid( + record_type_field_tables: dict[str, list[dict[str, str]]], +) -> None: + """T08: Verify every Tier cell contains a valid tier label.""" + for rt_name, rows in record_type_field_tables.items(): + for row in rows: + tier = row["Tier"] + assert tier in VALID_TIERS, ( + f"Invalid tier '{tier}' for field '{row['Field']}' in '{rt_name}'. " + f"Must be one of: {sorted(VALID_TIERS)}" + ) + + +# --------------------------------------------------------------------------- +# T09-T12: Consistency tests +# --------------------------------------------------------------------------- + + +@pytest.mark.docs +def test_summary_counts_match_detail_tables( + summary_rows: list[dict[str, str | int]], + record_type_field_tables: dict[str, list[dict[str, str]]], +) -> None: + """T09: Verify per-tier counts in summary match the detail tables.""" + summary_by_rt = {str(r["Record Type"]): r for r in summary_rows} + for rt_name, rows in record_type_field_tables.items(): + assert rt_name in summary_by_rt, ( + f"Record type '{rt_name}' in detail tables but not in summary" + ) + summary = summary_by_rt[rt_name] + # Count tiers from detail table + tier_counts: dict[str, int] = {t: 0 for t in VALID_TIERS} + for row in rows: + tier_counts[row["Tier"]] += 1 + total = len(rows) + # Verify total + assert total == summary["Total"], ( + f"'{rt_name}': detail table has {total} rows but summary Total is {summary['Total']}" + ) + # Verify per-tier counts + assert tier_counts["DCPF-critical"] == summary["DCPF-Critical"], ( + f"'{rt_name}': DCPF-Critical mismatch: " + f"detail={tier_counts['DCPF-critical']}, summary={summary['DCPF-Critical']}" + ) + assert tier_counts["ACPF-critical"] == summary["ACPF-Critical"], ( + f"'{rt_name}': ACPF-Critical mismatch: " + f"detail={tier_counts['ACPF-critical']}, summary={summary['ACPF-Critical']}" + ) + assert tier_counts["Informational"] == summary["Informational"], ( + f"'{rt_name}': Informational mismatch: " + f"detail={tier_counts['Informational']}, summary={summary['Informational']}" + ) + assert tier_counts["Discardable"] == summary["Discardable"], ( + f"'{rt_name}': Discardable mismatch: " + f"detail={tier_counts['Discardable']}, summary={summary['Discardable']}" + ) + # Verify Total = sum of tiers + tier_sum = sum(tier_counts.values()) + assert tier_sum == summary["Total"], ( + f"'{rt_name}': Tier sum ({tier_sum}) != Total ({summary['Total']})" + ) + + +@pytest.mark.docs +def test_grand_total_row_correct( + summary_rows: list[dict[str, str | int]], +) -> None: + """T10: Verify the grand total row equals column-wise sum of all per-record-type rows.""" + grand_total = None + rt_rows = [] + for row in summary_rows: + if str(row["Record Type"]) == "Total": + grand_total = row + else: + rt_rows.append(row) + assert grand_total is not None, "Grand total row not found in summary" + for col in ("Total", "DCPF-Critical", "ACPF-Critical", "Informational", "Discardable"): + expected = sum(int(r[col]) for r in rt_rows) + actual = int(grand_total[col]) + assert actual == expected, f"Grand total '{col}': expected {expected}, got {actual}" + + +@pytest.mark.docs +def test_present_but_inactive_fields_are_discardable( + record_type_field_tables: dict[str, list[dict[str, str]]], + schema_names: list[str], +) -> None: + """T11: Every field with x-psse-present-but-inactive must be Discardable.""" + for schema_name in schema_names: + schema = _load_schema(schema_name) + display = SCHEMA_DISPLAY_NAMES.get(schema_name, schema_name) + if display not in record_type_field_tables: + continue + doc_fields = {row["Field"]: row for row in record_type_field_tables[display]} + for field_name, field_spec in schema.get("properties", {}).items(): + if field_spec.get("x-psse-present-but-inactive", False): + assert field_name in doc_fields, ( + f"Present-but-inactive field '{field_name}' in '{display}' " + f"not found in field table" + ) + assert doc_fields[field_name]["Tier"] == "Discardable", ( + f"Present-but-inactive field '{field_name}' in '{display}' " + f"should be Discardable, got '{doc_fields[field_name]['Tier']}'" + ) + + +@pytest.mark.docs +def test_preservation_critical_fields_are_dcpf_or_acpf( + record_type_field_tables: dict[str, list[dict[str, str]]], + schema_names: list[str], +) -> None: + """T12: Every field with x-psse-preservation-critical must be DCPF or ACPF-critical.""" + allowed = {"DCPF-critical", "ACPF-critical"} + for schema_name in schema_names: + schema = _load_schema(schema_name) + display = SCHEMA_DISPLAY_NAMES.get(schema_name, schema_name) + if display not in record_type_field_tables: + continue + doc_fields = {row["Field"]: row for row in record_type_field_tables[display]} + for field_name, field_spec in schema.get("properties", {}).items(): + if field_spec.get("x-psse-preservation-critical", False): + assert field_name in doc_fields, ( + f"Preservation-critical field '{field_name}' in '{display}' " + f"not found in field table" + ) + tier = doc_fields[field_name]["Tier"] + assert tier in allowed, ( + f"Preservation-critical field '{field_name}' in '{display}' " + f"must be DCPF-critical or ACPF-critical, got '{tier}'" + ) + + +# --------------------------------------------------------------------------- +# T13-T14: Cross-reference and constraint tests +# --------------------------------------------------------------------------- + + +@pytest.mark.docs +def test_tier3_record_types_have_no_dcpf_or_acpf_fields( + record_type_field_tables: dict[str, list[dict[str, str]]], +) -> None: + """T13: No field in a Tier 3 record type may be DCPF-critical or ACPF-critical.""" + tier3_types = _get_tier3_record_types() + forbidden = {"DCPF-critical", "ACPF-critical"} + for display_name in tier3_types: + if display_name not in record_type_field_tables: + continue + for row in record_type_field_tables[display_name]: + assert row["Tier"] not in forbidden, ( + f"Tier 3 record type '{display_name}' has field '{row['Field']}' " + f"classified as '{row['Tier']}' -- must be Informational or Discardable" + ) + + +@pytest.mark.docs +def test_rationale_column_non_empty_and_non_generic( + record_type_field_tables: dict[str, list[dict[str, str]]], +) -> None: + """T14: Every rationale must be non-empty, non-generic, and at least 8 words.""" + generic_rationales = { + "dcpf-critical", + "acpf-critical", + "informational", + "discardable", + "needed for power flow", + "power flow", + "not needed", + } + for rt_name, rows in record_type_field_tables.items(): + for row in rows: + rationale = row["Rationale"].strip() + assert len(rationale) > 0, f"Empty rationale for field '{row['Field']}' in '{rt_name}'" + # Check word count + words = rationale.split() + assert len(words) >= 8, ( + f"Rationale for '{row['Field']}' in '{rt_name}' has only {len(words)} " + f"words (minimum 8): '{rationale}'" + ) + # Check not solely a generic phrase + rationale_lower = rationale.lower().strip().rstrip(".") + assert rationale_lower not in generic_rationales, ( + f"Generic rationale for '{row['Field']}' in '{rt_name}': '{rationale}'" + ) diff --git a/data/fnm/tests/test_gridcal_parser.py b/data/fnm/tests/test_gridcal_parser.py new file mode 100644 index 00000000..bb0c17eb --- /dev/null +++ b/data/fnm/tests/test_gridcal_parser.py @@ -0,0 +1,262 @@ +"""Tests for GridCal v31 parser module. + +T01-T07: Synthetic tests that validate data structures and constants without GridCal. +T08-T09: Integration tests requiring GridCal (skip if not installed). +T10-T12: FNM integration tests requiring both GridCal and FNM_PATH. +""" + +from __future__ import annotations + +import json + +import pytest + +from fnm.scripts.gridcal_parser import ( + GRIDCAL_ELEMENT_COLLECTIONS, + PSSE_TO_GRIDCAL_MAPPING, + GridCalParserSummary, + MultiCircuitCounts, + ParserLog, + ParserLogEntry, + PsseIntermediateCounts, + build_record_type_mapping, + parser_log_to_dict, + summary_to_dict, +) +from fnm.scripts.raw_record_counter import PSSE_V31_SECTION_NAMES + +# --------------------------------------------------------------------------- +# Synthetic tests (T01-T07) — no GridCal needed +# --------------------------------------------------------------------------- + + +class TestPsseToGridcalMapping: + """T01: Verify PSSE_TO_GRIDCAL_MAPPING covers all 17 PSS/E sections.""" + + def test_psse_to_gridcal_mapping_covers_all_17_sections(self) -> None: + assert len(PSSE_TO_GRIDCAL_MAPPING) == 17 + for section_name in PSSE_V31_SECTION_NAMES: + assert section_name in PSSE_TO_GRIDCAL_MAPPING, ( + f"Missing mapping for PSS/E section: {section_name}" + ) + + +class TestBuildRecordTypeMapping: + """T02-T04: Verify build_record_type_mapping output.""" + + def test_build_record_type_mapping_all_sections(self) -> None: + """T02: 17 entries, each with valid status and non-empty notes.""" + mappings = build_record_type_mapping() + assert len(mappings) == 17 + + valid_statuses = {"mapped", "dropped", "merged"} + for m in mappings: + assert m.status in valid_statuses, ( + f"Invalid status '{m.status}' for section '{m.psse_section}'" + ) + assert m.notes, f"Empty notes for section '{m.psse_section}'" + assert m.psse_section in PSSE_V31_SECTION_NAMES + + def test_build_record_type_mapping_dropped_sections(self) -> None: + """T03: Multi-Terminal DC, Multi-Section Line, Interarea Transfer, Owner are dropped.""" + mappings = build_record_type_mapping() + mapping_by_section = {m.psse_section: m for m in mappings} + + dropped_sections = [ + "Multi-Terminal DC", + "Multi-Section Line", + "Interarea Transfer", + "Owner", + ] + for section_name in dropped_sections: + m = mapping_by_section[section_name] + assert m.status == "dropped", ( + f"Expected 'dropped' for '{section_name}', got '{m.status}'" + ) + assert m.gridcal_collection is None + + def test_build_record_type_mapping_merged_sections(self) -> None: + """T04: Impedance Correction has status='merged'.""" + mappings = build_record_type_mapping() + mapping_by_section = {m.psse_section: m for m in mappings} + + m = mapping_by_section["Impedance Correction"] + assert m.status == "merged" + assert m.gridcal_collection is None + + +class TestSummaryRoundtrip: + """T05: Build GridCalParserSummary with synthetic data, json.dumps succeeds.""" + + def test_summary_to_dict_roundtrip(self) -> None: + summary = GridCalParserSummary( + raw_path="/fake/path/test.raw", + psse_intermediate_counts=PsseIntermediateCounts(bus=100, load=50, generator=10), + multicircuit_counts=MultiCircuitCounts(buses=100, loads=50, generators=10), + parser_log=ParserLog( + entries=[ + ParserLogEntry( + time="2025-01-01T00:00:00Z", + severity="INFO", + message="Test message", + ) + ], + info_count=1, + warning_count=0, + error_count=0, + ), + record_type_mapping=build_record_type_mapping(), + csv_files=["/fake/output/gridcal_buses.csv"], + log_file="/fake/output/parser_log.json", + timestamp="2025-01-01T00:00:00Z", + ) + + result = summary_to_dict(summary) + + # Must be JSON-serializable + json_str = json.dumps(result) + assert json_str + + # Roundtrip: parse back and verify key fields + parsed = json.loads(json_str) + assert parsed["raw_path"] == "/fake/path/test.raw" + assert parsed["psse_intermediate_counts"]["bus"] == 100 + assert parsed["multicircuit_counts"]["buses"] == 100 + assert len(parsed["record_type_mapping"]) == 17 + assert parsed["csv_files"] == ["/fake/output/gridcal_buses.csv"] + + +class TestParserLogToDict: + """T06: Verify parser_log_to_dict structure with 3 synthetic entries.""" + + def test_parser_log_to_dict_structure(self) -> None: + log = ParserLog( + entries=[ + ParserLogEntry( + time="2025-01-01T00:00:00Z", + severity="INFO", + message="Info message", + ), + ParserLogEntry( + time="2025-01-01T00:00:01Z", + severity="WARNING", + message="Warning message", + device="BUS-1", + ), + ParserLogEntry( + time="2025-01-01T00:00:02Z", + severity="ERROR", + message="Error message", + device="GEN-1", + device_class="Generator", + ), + ], + info_count=1, + warning_count=1, + error_count=1, + ) + + result = parser_log_to_dict(log) + + assert len(result["entries"]) == 3 + assert result["info_count"] == 1 + assert result["warning_count"] == 1 + assert result["error_count"] == 1 + + # Verify entry structure + entry0 = result["entries"][0] + assert "time" in entry0 + assert "severity" in entry0 + assert "message" in entry0 + assert "device" in entry0 + assert "device_class" in entry0 + + # Must be JSON-serializable + json_str = json.dumps(result) + assert json_str + + +class TestGridCalElementCollections: + """T07: Verify GRIDCAL_ELEMENT_COLLECTIONS is non-empty and valid.""" + + def test_gridcal_element_collections_tuple_not_empty(self) -> None: + assert isinstance(GRIDCAL_ELEMENT_COLLECTIONS, tuple) + assert len(GRIDCAL_ELEMENT_COLLECTIONS) > 0 + + for name in GRIDCAL_ELEMENT_COLLECTIONS: + assert isinstance(name, str) + assert name.isidentifier(), f"'{name}' is not a valid Python identifier" + + +# --------------------------------------------------------------------------- +# GridCal integration tests (T08-T09) — require GridCal, no FNM +# --------------------------------------------------------------------------- + + +@pytest.mark.gridcal +class TestGridCalCase39: + """T08-T09: Integration tests loading case39.m via GridCal.""" + + def test_load_case39_produces_multicircuit(self, require_gridcal, tmp_path) -> None: + """T08: Load case39.m, verify it produces a MultiCircuit with buses.""" + from fnm.scripts.gridcal_parser import count_multicircuit, load_raw_with_logging + + case39_path = require_gridcal["case39_path"] + grid, _psse_circuit, _logger = load_raw_with_logging(case39_path) + + counts = count_multicircuit(grid) + # IEEE 39-bus system should have 39 buses + assert counts.buses == 39, f"Expected 39 buses, got {counts.buses}" + + def test_export_case39_csv_tables(self, require_gridcal, tmp_path) -> None: + """T09: Export collections to tmp_path, verify CSV files exist.""" + from fnm.scripts.gridcal_parser import export_all_collections, load_raw_with_logging + + case39_path = require_gridcal["case39_path"] + grid, _psse_circuit, _logger = load_raw_with_logging(case39_path) + + csv_files = export_all_collections(grid, tmp_path) + assert len(csv_files) > 0, "Expected at least one CSV file exported" + + for csv_path in csv_files: + assert csv_path.exists(), f"CSV file not found: {csv_path}" + assert csv_path.stat().st_size > 0, f"CSV file is empty: {csv_path}" + + +# --------------------------------------------------------------------------- +# FNM integration tests (T10-T12) — require FNM_PATH + GridCal +# --------------------------------------------------------------------------- + + +@pytest.mark.fnm +@pytest.mark.gridcal +class TestGridCalFnm: + """T10-T12: FNM integration tests requiring both GridCal and FNM_PATH.""" + + def test_load_fnm_raw_produces_multicircuit(self, require_gridcal, require_fnm_raw) -> None: + """T10: Load the FNM RAW file, verify non-zero bus count.""" + from fnm.scripts.gridcal_parser import count_multicircuit, load_raw_with_logging + + grid, _psse_circuit, _logger = load_raw_with_logging(require_fnm_raw) + counts = count_multicircuit(grid) + assert counts.buses > 0, "Expected non-zero bus count from FNM RAW" + + def test_export_fnm_csv_tables(self, require_gridcal, require_fnm_raw, tmp_path) -> None: + """T11: Export FNM collections to CSV, verify multiple files.""" + from fnm.scripts.gridcal_parser import export_all_collections, load_raw_with_logging + + grid, _psse_circuit, _logger = load_raw_with_logging(require_fnm_raw) + csv_files = export_all_collections(grid, tmp_path) + assert len(csv_files) >= 5, f"Expected at least 5 CSV files from FNM, got {len(csv_files)}" + + def test_fnm_parser_log_captured(self, require_gridcal, require_fnm_raw) -> None: + """T12: Verify parser log is captured during FNM parsing.""" + from fnm.scripts.gridcal_parser import extract_logger_entries, load_raw_with_logging + + _grid, _psse_circuit, logger = load_raw_with_logging(require_fnm_raw) + parser_log = extract_logger_entries(logger) + + # The log object should be valid even if empty + assert isinstance(parser_log.entries, list) + total = parser_log.info_count + parser_log.warning_count + parser_log.error_count + assert total == len(parser_log.entries) diff --git a/data/fnm/tests/test_intermediate_schema.py b/data/fnm/tests/test_intermediate_schema.py new file mode 100644 index 00000000..4fb465d2 --- /dev/null +++ b/data/fnm/tests/test_intermediate_schema.py @@ -0,0 +1,688 @@ +"""Tests for PRD 07 -- Intermediate Format Schema Specification. + +T01-T10: Synthetic tests (no FNM data required). +T11-T14: Integration/end-to-end tests (require FNM_PATH, skip if unset). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from fnm.scripts.intermediate_schema import ( + PSSE_V31_RECORD_TYPES, + PerUnitBase, + TableSchema, + detect_inactive_fields, + get_table_schemas, + manifest_to_json_schema, + table_schema_to_json_schema, + validate_tables, + write_schemas, +) + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def all_table_schemas() -> list[TableSchema]: + """Return all 17 table schemas.""" + return get_table_schemas() + + +@pytest.fixture +def bus_schema() -> TableSchema: + """Return the Bus table schema.""" + schemas = get_table_schemas() + return next(ts for ts in schemas if ts.record_type == "Bus") + + +@pytest.fixture +def bus_json_schema(bus_schema: TableSchema) -> dict: + """Return the Bus JSON Schema dict.""" + return table_schema_to_json_schema(bus_schema) + + +# --------------------------------------------------------------------------- +# T01: get_table_schemas covers all 17 types +# --------------------------------------------------------------------------- + + +def test_get_table_schemas_covers_all_17_types( + all_table_schemas: list[TableSchema], +) -> None: + """Call get_table_schemas(). Verify it returns exactly 17 TableSchema + objects, one per PSS/E v31 record type, in section order.""" + assert len(all_table_schemas) == 17 + + returned_types = [ts.record_type for ts in all_table_schemas] + assert tuple(returned_types) == PSSE_V31_RECORD_TYPES + + for ts in all_table_schemas: + assert len(ts.fields) > 0, f"{ts.record_type} has no fields" + assert len(ts.primary_key) > 0, f"{ts.record_type} has no primary_key" + assert ts.table_name, f"{ts.record_type} has no table_name" + assert ts.description, f"{ts.record_type} has no description" + + +# --------------------------------------------------------------------------- +# T02: table_schema_to_json_schema produces valid Draft 2020-12 +# --------------------------------------------------------------------------- + + +def test_table_schema_to_json_schema_valid_draft_2020_12( + bus_json_schema: dict, +) -> None: + """Convert the Bus TableSchema to JSON Schema. Verify Draft 2020-12 + structure and validate against the meta-schema.""" + schema = bus_json_schema + + assert schema["$schema"] == "https://json-schema.org/draft/2020-12/schema" + assert schema["type"] == "object" + assert isinstance(schema["properties"], dict) + assert isinstance(schema["required"], list) + assert len(schema["properties"]) > 0 + assert "I" in schema["properties"] + assert "additionalProperties" in schema + + # Verify custom extension keywords are present + bus_i = schema["properties"]["I"] + assert "x-psse-unit" in bus_i + assert "x-psse-per-unit-base" in bus_i + assert "x-psse-preservation-critical" in bus_i + assert "x-psse-present-but-inactive" in bus_i + assert "x-psse-valid-range" in bus_i + + # Validate against meta-schema using jsonschema + try: + from jsonschema.validators import Draft202012Validator + + Draft202012Validator.check_schema(schema) + except ImportError: + pytest.skip("jsonschema not installed") + + +# --------------------------------------------------------------------------- +# T03: manifest schema valid Draft 2020-12 +# --------------------------------------------------------------------------- + + +def test_manifest_schema_valid_draft_2020_12() -> None: + """Call manifest_to_json_schema(). Validate against the meta-schema. + Verify required manifest properties.""" + schema = manifest_to_json_schema() + + assert schema["$schema"] == "https://json-schema.org/draft/2020-12/schema" + assert schema["type"] == "object" + + expected_required = { + "sbase", + "basfrq", + "rev", + "case_id", + "canonical_parser", + "tables", + "total_records", + "total_tables", + "non_empty_record_types", + "schema_version", + "generated_timestamp", + } + assert set(schema["required"]) == expected_required + + # Each expected required property should be in properties + for prop in expected_required: + assert prop in schema["properties"], f"Missing property: {prop}" + + try: + from jsonschema.validators import Draft202012Validator + + Draft202012Validator.check_schema(schema) + except ImportError: + pytest.skip("jsonschema not installed") + + +# --------------------------------------------------------------------------- +# T04: preservation-critical fields present +# --------------------------------------------------------------------------- + + +def test_preservation_critical_fields_present( + all_table_schemas: list[TableSchema], +) -> None: + """Verify that preservation-critical fields are marked correctly.""" + schemas_by_type = {ts.record_type: ts for ts in all_table_schemas} + + # Transformer: K, CW, CZ, CM, WINDV1-3, NOMV1-3, ANG1, RATA1-3 + xfmr = schemas_by_type["Transformer"] + xfmr_fields = {f.name: f for f in xfmr.fields} + for fname in [ + "K", + "CW", + "CZ", + "CM", + "WINDV1", + "WINDV2", + "WINDV3", + "NOMV1", + "NOMV2", + "NOMV3", + "ANG1", + "RATA1", + "RATA2", + "RATA3", + ]: + assert fname in xfmr_fields, f"Missing transformer field: {fname}" + assert xfmr_fields[fname].preservation_critical, ( + f"Transformer.{fname} should be preservation_critical" + ) + + # Switched Shunt: BINIT, N1-N8, B1-B8, MODSW, SWREM + ss = schemas_by_type["Switched Shunt"] + ss_fields = {f.name: f for f in ss.fields} + ss_critical = ["MODSW", "SWREM", "BINIT"] + for i in range(1, 9): + ss_critical.extend([f"N{i}", f"B{i}"]) + for fname in ss_critical: + assert fname in ss_fields, f"Missing switched shunt field: {fname}" + assert ss_fields[fname].preservation_critical, ( + f"SwitchedShunt.{fname} should be preservation_critical" + ) + + # Generator: IREG + gen = schemas_by_type["Generator"] + gen_fields = {f.name: f for f in gen.fields} + assert gen_fields["IREG"].preservation_critical + + # Area: ISW, PDES, PTOL + area = schemas_by_type["Area"] + area_fields = {f.name: f for f in area.fields} + for fname in ["ISW", "PDES", "PTOL"]: + assert area_fields[fname].preservation_critical, ( + f"Area.{fname} should be preservation_critical" + ) + + # Multi-Section Line: I, J, ID, DUM1-DUM9 + msl = schemas_by_type["Multi-Section Line"] + msl_fields = {f.name: f for f in msl.fields} + for fname in ["I", "J", "ID"] + [f"DUM{i}" for i in range(1, 10)]: + assert msl_fields[fname].preservation_critical, ( + f"MultiSectionLine.{fname} should be preservation_critical" + ) + + +# --------------------------------------------------------------------------- +# T05: per-unit base annotations complete +# --------------------------------------------------------------------------- + + +def test_per_unit_base_annotations_complete( + all_table_schemas: list[TableSchema], +) -> None: + """Verify every field has a per-unit base. Check specific expectations.""" + for ts in all_table_schemas: + for f in ts.fields: + assert f.per_unit_base is not None, f"{ts.record_type}.{f.name} has None per_unit_base" + assert isinstance(f.per_unit_base, PerUnitBase), ( + f"{ts.record_type}.{f.name} per_unit_base is not PerUnitBase" + ) + + # Specific checks + schemas_by_type = {ts.record_type: ts for ts in all_table_schemas} + + # Transformer impedance fields should be MIXED + xfmr = schemas_by_type["Transformer"] + xfmr_fields = {f.name: f for f in xfmr.fields} + for fname in ["R1_2", "X1_2", "R2_3", "X2_3", "R3_1", "X3_1"]: + assert xfmr_fields[fname].per_unit_base == PerUnitBase.MIXED, ( + f"Transformer.{fname} should be MIXED" + ) + + # Bus VM should be BUS_KV + bus = schemas_by_type["Bus"] + bus_fields = {f.name: f for f in bus.fields} + assert bus_fields["VM"].per_unit_base == PerUnitBase.BUS_KV + + # Branch R should be SYSTEM_MVA + branch = schemas_by_type["Branch"] + branch_fields = {f.name: f for f in branch.fields} + assert branch_fields["R"].per_unit_base == PerUnitBase.SYSTEM_MVA + + +# --------------------------------------------------------------------------- +# T06: write_schemas creates files +# --------------------------------------------------------------------------- + + +def test_write_schemas_creates_files(tmp_path: Path) -> None: + """Call write_schemas() with three types. Verify files are created + and valid.""" + non_empty = ["Bus", "Generator", "Area"] + paths = write_schemas(tmp_path, non_empty) + + expected_files = [ + tmp_path / "schemas" / "bus.schema.json", + tmp_path / "schemas" / "generator.schema.json", + tmp_path / "schemas" / "area.schema.json", + tmp_path / "schemas" / "manifest.schema.json", + ] + + for ef in expected_files: + assert ef.exists(), f"Expected file not created: {ef}" + + assert len(paths) == len(expected_files) + + # Verify each file is valid JSON and passes meta-schema validation + try: + from jsonschema.validators import Draft202012Validator + except ImportError: + pytest.skip("jsonschema not installed") + + for ef in expected_files: + data = json.loads(ef.read_text(encoding="utf-8")) + assert "$schema" in data + Draft202012Validator.check_schema(data) + + +# --------------------------------------------------------------------------- +# T07: write_schemas respects non-empty filter +# --------------------------------------------------------------------------- + + +def test_write_schemas_respects_non_empty_filter(tmp_path: Path) -> None: + """Call write_schemas() with only Bus. Verify only bus + manifest + schemas created.""" + write_schemas(tmp_path, ["Bus"]) + + schema_dir = tmp_path / "schemas" + schema_files = list(schema_dir.glob("*.schema.json")) + schema_names = {f.name for f in schema_files} + + assert "bus.schema.json" in schema_names + assert "manifest.schema.json" in schema_names + assert len(schema_files) == 2, f"Expected 2 files but got {len(schema_files)}: {schema_names}" + + +# --------------------------------------------------------------------------- +# T08: detect_inactive_fields +# --------------------------------------------------------------------------- + + +def test_detect_inactive_fields_identifies_uniform_defaults( + tmp_path: Path, +) -> None: + """Create synthetic CSV, verify inactive detection.""" + # Write a synthetic bus CSV with NVHI uniform at default (1.1) + # and VM varying + csv_dir = tmp_path / "csvs" + csv_dir.mkdir() + + bus_csv = csv_dir / "bus.csv" + bus_csv.write_text( + "I,NAME,BASKV,IDE,AREA,ZONE,OWNER,VM,VA,NVHI,NVLO,EVHI,EVLO\n" + "1,BUS1,138.0,1,1,1,1,0.98,0.0,1.1,0.9,1.1,0.9\n" + "2,BUS2,138.0,1,1,1,1,1.01,0.0,1.1,0.9,1.1,0.9\n" + "3,BUS3,138.0,1,1,1,1,1.05,0.0,1.1,0.9,1.1,0.9\n", + encoding="utf-8", + ) + + result = detect_inactive_fields( + csv_dir=csv_dir, + non_empty_types=["Bus"], + record_type_to_table_name={"Bus": "bus"}, + ) + + assert "Bus" in result + inactive = result["Bus"] + assert "NVHI" in inactive + assert "VM" not in inactive + + +# --------------------------------------------------------------------------- +# T09: validate_tables conformant +# --------------------------------------------------------------------------- + + +def test_validate_tables_conformant(tmp_path: Path) -> None: + """Create conformant CSVs and schemas, verify is_conformant == True.""" + # Write schemas + non_empty = ["Bus", "Generator"] + write_schemas(tmp_path, non_empty) + + # Create CSV directory + csv_dir = tmp_path / "tables" + csv_dir.mkdir() + + # Bus CSV + bus_csv = csv_dir / "bus.csv" + bus_csv.write_text( + "I,NAME,BASKV,IDE,AREA,ZONE,OWNER,VM,VA,NVHI,NVLO,EVHI,EVLO\n" + "1,BUS1,138.0,1,1,1,1,1.0,0.0,1.1,0.9,1.1,0.9\n" + "2,BUS2,345.0,2,1,1,1,1.02,5.0,1.1,0.9,1.1,0.9\n", + encoding="utf-8", + ) + + # Generator CSV -- include all required fields + gen_schema = get_table_schemas() + gen_ts = next(ts for ts in gen_schema if ts.record_type == "Generator") + gen_required = [f.name for f in gen_ts.fields if f.required] + gen_header = ",".join(gen_required) + gen_row1_vals = [] + gen_row2_vals = [] + for f in gen_ts.fields: + if not f.required: + continue + if f.data_type == "integer": + gen_row1_vals.append("1") + gen_row2_vals.append("2") + elif f.data_type == "number": + gen_row1_vals.append("100.0") + gen_row2_vals.append("200.0") + else: + gen_row1_vals.append("G1") + gen_row2_vals.append("G2") + + gen_csv = csv_dir / "generator.csv" + gen_csv.write_text( + gen_header + "\n" + ",".join(gen_row1_vals) + "\n" + ",".join(gen_row2_vals) + "\n", + encoding="utf-8", + ) + + # Create manifest + manifest = { + "sbase": 100.0, + "basfrq": 60.0, + "rev": 31.0, + "case_id": "Test Case", + "canonical_parser": "gridcal", + "tables": [ + { + "table_name": "bus", + "record_type": "Bus", + "file_name": "bus.csv", + "record_count": 2, + "column_count": 13, + "schema_file": "schemas/bus.schema.json", + }, + { + "table_name": "generator", + "record_type": "Generator", + "file_name": "generator.csv", + "record_count": 2, + "column_count": len(gen_required), + "schema_file": "schemas/generator.schema.json", + }, + ], + "total_records": 4, + "total_tables": 2, + "non_empty_record_types": ["Bus", "Generator"], + "schema_version": "1.0.0", + "generated_timestamp": "2026-01-01T00:00:00Z", + } + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + report = validate_tables( + csv_dir=csv_dir, + schema_dir=tmp_path / "schemas", + manifest_path=manifest_path, + ) + + assert report.is_conformant, ( + f"Expected conformant but got errors: {[f.message for f in report.errors]}" + ) + assert len(report.errors) == 0 + + +# --------------------------------------------------------------------------- +# T10: validate_tables missing required field +# --------------------------------------------------------------------------- + + +def test_validate_tables_missing_required_field(tmp_path: Path) -> None: + """Create bus.csv missing required I column. Verify error.""" + # Write schemas + write_schemas(tmp_path, ["Bus"]) + + csv_dir = tmp_path / "tables" + csv_dir.mkdir() + + # Bus CSV missing 'I' column + bus_csv = csv_dir / "bus.csv" + bus_csv.write_text( + "NAME,BASKV,IDE,AREA,ZONE,OWNER,VM,VA\nBUS1,138.0,1,1,1,1,1.0,0.0\n", + encoding="utf-8", + ) + + manifest = { + "sbase": 100.0, + "basfrq": 60.0, + "rev": 31.0, + "case_id": "Test", + "canonical_parser": "gridcal", + "tables": [ + { + "table_name": "bus", + "record_type": "Bus", + "file_name": "bus.csv", + "record_count": 1, + "column_count": 8, + "schema_file": "schemas/bus.schema.json", + }, + ], + "total_records": 1, + "total_tables": 1, + "non_empty_record_types": ["Bus"], + "schema_version": "1.0.0", + "generated_timestamp": "2026-01-01T00:00:00Z", + } + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + report = validate_tables( + csv_dir=csv_dir, + schema_dir=tmp_path / "schemas", + manifest_path=manifest_path, + ) + + assert not report.is_conformant + error_ids = [f.check_id for f in report.errors] + assert "missing_required_field" in error_ids + + # Verify it specifically notes field 'I' in table 'bus' + matching = [ + f + for f in report.errors + if f.check_id == "missing_required_field" and f.field_name == "I" and f.table_name == "bus" + ] + assert len(matching) > 0 + + +# --------------------------------------------------------------------------- +# T11-T14: Integration tests (require FNM_PATH) - marked with @pytest.mark.fnm +# --------------------------------------------------------------------------- + + +@pytest.mark.fnm +def test_write_schemas_for_fnm_non_empty_types(require_fnm: dict, tmp_path: Path) -> None: + """T11: Load D3 raw record count summary, write schemas for all + non-empty types.""" + repo_root = Path(require_fnm.get("repo_root", ".")) + raw_counts_path = repo_root / "data" / "fnm" / "intermediate" / "raw_counts.json" + if not raw_counts_path.exists(): + pytest.skip("D3 raw_counts.json not found") + + raw_data = json.loads(raw_counts_path.read_text(encoding="utf-8")) + non_empty = raw_data.get("non_empty_sections", []) + assert len(non_empty) > 0 + + paths = write_schemas(tmp_path, non_empty) + + # Verify a schema file for every non-empty type + for rt in non_empty: + table_name = rt.lower().replace(" ", "_").replace("-", "_") + schema_path = tmp_path / "schemas" / f"{table_name}.schema.json" + assert schema_path.exists(), f"Missing schema for {rt}" + + try: + from jsonschema.validators import Draft202012Validator + + for p in paths: + data = json.loads(p.read_text(encoding="utf-8")) + Draft202012Validator.check_schema(data) + except ImportError: + pass # jsonschema not required for this assertion + + +@pytest.mark.fnm +def test_inactive_fields_detected_from_canonical_parser(require_fnm: dict, tmp_path: Path) -> None: + """T12: Detect inactive fields from canonical parser CSV exports.""" + repo_root = Path(require_fnm.get("repo_root", ".")) + raw_counts_path = repo_root / "data" / "fnm" / "intermediate" / "raw_counts.json" + if not raw_counts_path.exists(): + pytest.skip("D3 raw_counts.json not found") + + raw_data = json.loads(raw_counts_path.read_text(encoding="utf-8")) + non_empty = raw_data.get("non_empty_sections", []) + + # Try to find canonical parser CSV directory + csv_dir = repo_root / "data" / "fnm" / "intermediate" / "csvs" + if not csv_dir.exists(): + pytest.skip("Canonical parser CSV directory not found") + + rt_to_tn = {rt: rt.lower().replace(" ", "_").replace("-", "_") for rt in non_empty} + + result = detect_inactive_fields(csv_dir, non_empty, rt_to_tn) + + # At least some fields should be inactive in a large model + assert isinstance(result, dict) + # Log for manual review + for rt, fields in sorted(result.items()): + print(f" {rt}: {fields}") + + +@pytest.mark.fnm +def test_generate_subcommand_produces_all_outputs(require_fnm: dict, tmp_path: Path) -> None: + """T13: Run generate subcommand, verify output files.""" + from fnm.scripts.intermediate_schema import main as schema_main + + repo_root = Path(require_fnm.get("repo_root", ".")) + raw_counts_path = repo_root / "data" / "fnm" / "intermediate" / "raw_counts.json" + if not raw_counts_path.exists(): + pytest.skip("D3 raw_counts.json not found") + + raw_data = json.loads(raw_counts_path.read_text(encoding="utf-8")) + non_empty = raw_data.get("non_empty_sections", []) + + schema_main( + [ + "generate", + "--raw-summary", + str(raw_counts_path), + "-o", + str(tmp_path), + ] + ) + + # Verify schemas directory + schema_dir = tmp_path / "schemas" + assert schema_dir.exists() + schema_files = list(schema_dir.glob("*.schema.json")) + assert len(schema_files) >= len(non_empty) + 1 # +1 for manifest + + # Verify markdown reference + md_path = tmp_path / "intermediate_format_reference.md" + assert md_path.exists() + md_text = md_path.read_text(encoding="utf-8") + for rt in non_empty: + assert rt in md_text, f"Markdown missing section for {rt}" + + +@pytest.mark.fnm +def test_validate_subcommand_on_canonical_output(require_fnm: dict, tmp_path: Path) -> None: + """T14: Validate canonical parser CSV exports against generated schemas.""" + from fnm.scripts.intermediate_schema import main as schema_main + + repo_root = Path(require_fnm.get("repo_root", ".")) + raw_counts_path = repo_root / "data" / "fnm" / "intermediate" / "raw_counts.json" + csv_dir = repo_root / "data" / "fnm" / "intermediate" / "csvs" + + if not raw_counts_path.exists(): + pytest.skip("D3 raw_counts.json not found") + if not csv_dir.exists(): + pytest.skip("Canonical parser CSV directory not found") + + # Generate schemas first + schema_main( + [ + "generate", + "--raw-summary", + str(raw_counts_path), + "-o", + str(tmp_path), + ] + ) + + # Create a synthetic manifest for the canonical CSVs + raw_data = json.loads(raw_counts_path.read_text(encoding="utf-8")) + non_empty = raw_data.get("non_empty_sections", []) + + tables_entries = [] + for rt in non_empty: + tn = rt.lower().replace(" ", "_").replace("-", "_") + csv_path = csv_dir / f"{tn}.csv" + if csv_path.exists(): + tables_entries.append( + { + "table_name": tn, + "record_type": rt, + "file_name": f"{tn}.csv", + "record_count": 0, # placeholder + "column_count": 1, # placeholder + "schema_file": f"schemas/{tn}.schema.json", + } + ) + + manifest = { + "sbase": 100.0, + "basfrq": 60.0, + "rev": 31.0, + "case_id": "FNM", + "canonical_parser": "gridcal", + "tables": tables_entries, + "total_records": 0, + "total_tables": len(tables_entries), + "non_empty_record_types": non_empty, + "schema_version": "1.0.0", + "generated_timestamp": "2026-01-01T00:00:00Z", + } + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + # Run validate + schema_main( + [ + "validate", + "--csv-dir", + str(csv_dir), + "--schema-dir", + str(tmp_path / "schemas"), + "--manifest", + str(manifest_path), + "-o", + str(tmp_path), + ] + ) + + report_path = tmp_path / "conformance_report.json" + assert report_path.exists() + report_data = json.loads(report_path.read_text(encoding="utf-8")) + print( + f" Conformance: errors={len(report_data['errors'])}, " + f"warnings={len(report_data['warnings'])}, " + f"info={len(report_data['info'])}" + ) diff --git a/data/fnm/tests/test_mapping_guide.py b/data/fnm/tests/test_mapping_guide.py new file mode 100644 index 00000000..8d9b6ece --- /dev/null +++ b/data/fnm/tests/test_mapping_guide.py @@ -0,0 +1,437 @@ +"""Tests for the Record-Type Mapping Guide (PRD 02/02). + +Validates the structural integrity and content consistency of the mapping guide +markdown document at ``data/fnm/docs/mapping-guide.md``. + +Tests T01-T11 are pure markdown parsing tests using only ``pathlib``, ``re``, +and ``pytest``. Test T12 requires ``FNM_PATH`` and is marked ``@pytest.mark.fnm``. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import pytest + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +MAPPING_GUIDE = REPO_ROOT / "data" / "fnm" / "docs" / "mapping-guide.md" + +ALL_17_RECORD_TYPES: list[str] = [ + "Bus", + "Load", + "Fixed Shunt", + "Generator", + "Branch", + "Transformer", + "Area", + "Two-Terminal DC", + "VSC DC", + "Impedance Correction", + "Multi-Terminal DC", + "Multi-Section Line", + "Zone", + "Interarea Transfer", + "Owner", + "FACTS", + "Switched Shunt", +] + +SIX_TOOLS: list[str] = [ + "PyPSA", + "pandapower", + "GridCal", + "PowerModels.jl", + "PowerSimulations.jl", + "MATPOWER", +] + +VALID_SUPPORT_VALUES = {"Y", "P", "N", "--"} + +TIER1_ESSENTIAL: set[str] = {"Bus", "Load", "Generator", "Branch", "Transformer"} + +TIER3_TYPES: set[str] = {"Zone", "Owner", "Interarea Transfer"} + +REQUIRED_ABSTRACTIONS: set[str] = { + "Bus", + "AC Line", + "2-Winding Transformer", + "3-Winding Transformer", + "Generator", + "Load", + "Fixed Shunt", + "Switched Shunt", + "Area", + "Zone", + "Owner", + "Two-Terminal HVDC Line", + "VSC HVDC Line", + "Multi-Terminal DC", + "Impedance Correction Table", + "Multi-Section Line", + "Interarea Transfer", + "FACTS Device", +} + +CROSS_REF_FILES: list[str] = [ + "intermediate-schema.md", + "per-unit-conventions.md", + "three-winding-transformers.md", + "field-criticality-matrix.md", +] + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _read_guide() -> str: + """Read the mapping guide content.""" + return MAPPING_GUIDE.read_text(encoding="utf-8") + + +def _parse_summary_matrix(text: str) -> list[dict[str, str]]: + """Parse the S3 summary matrix table into a list of row dicts. + + Returns a list of dicts with keys: '#', 'PSS/E Record Type', 'Abstraction', + 'Tier', 'FNM Status', and each of the six tool names. + """ + # Find the summary matrix section + s3_match = re.search( + r"## S3: Summary Matrix\s*\n(.*?)(?=\n## S[4-9]|\n## S\d{2}|\Z)", text, re.DOTALL + ) + assert s3_match, "S3: Summary Matrix section not found" + s3_text = s3_match.group(1) + + # Find all table lines (starting with |, not separator lines) + table_lines = [ + line.strip() + for line in s3_text.split("\n") + if line.strip().startswith("|") and not re.match(r"^\|[-\s|]+\|$", line.strip()) + ] + + assert len(table_lines) >= 2, "Summary matrix table must have header + data rows" + + # Parse header + header_cells = [c.strip() for c in table_lines[0].split("|")[1:-1]] + + # Parse data rows + rows: list[dict[str, str]] = [] + for line in table_lines[1:]: + cells = [c.strip() for c in line.split("|")[1:-1]] + if len(cells) == len(header_cells): + rows.append(dict(zip(header_cells, cells))) + + return rows + + +def _find_s5_subsections(text: str) -> list[str]: + """Return the list of record type names found in S5 subsection headers. + + Matches patterns like: ### S5.1: Bus (Section 1) + """ + pattern = r"### S5\.\d+:\s*(.+?)\s*\(Section \d+\)" + return re.findall(pattern, text) + + +def _parse_tool_support_table(subsection_text: str) -> dict[str, str]: + """Parse a per-record-type tool support table, returning {tool: Y/P/N}.""" + result: dict[str, str] = {} + # Match table rows with tool name, support value + # Format: | PyPSA | Y | ... | ... | + for line in subsection_text.split("\n"): + line = line.strip() + if not line.startswith("|"): + continue + cells = [c.strip() for c in line.split("|")[1:-1]] + if len(cells) >= 2 and cells[0] in SIX_TOOLS: + result[cells[0]] = cells[1] + return result + + +def _get_s5_subsection_text(text: str, record_type: str) -> str | None: + """Extract the full text of a single S5 subsection by record type name.""" + # Escape special regex chars in record type name + escaped = re.escape(record_type) + pattern = ( + rf"(### S5\.\d+:\s*{escaped}\s*\(Section \d+\).*?)" + r"(?=### S5\.\d+:|## S6:|\Z)" + ) + match = re.search(pattern, text, re.DOTALL) + return match.group(1) if match else None + + +# --------------------------------------------------------------------------- +# T01-T07: Structural validation tests +# --------------------------------------------------------------------------- + + +class TestStructuralValidation: + """Structural validation of the mapping guide markdown document.""" + + def test_document_exists(self) -> None: + """T01: Verify mapping-guide.md exists and is non-empty.""" + assert MAPPING_GUIDE.exists(), f"Mapping guide not found at {MAPPING_GUIDE}" + content = MAPPING_GUIDE.read_text(encoding="utf-8") + assert len(content.strip()) > 0, "Mapping guide is empty" + + def test_all_17_record_types_present(self) -> None: + """T02: All 17 PSS/E v31 record types have subsections in S5.""" + text = _read_guide() + found = _find_s5_subsections(text) + found_set = set(found) + + for rt in ALL_17_RECORD_TYPES: + assert rt in found_set, ( + f"Record type '{rt}' not found in S5 subsection headers. Found: {sorted(found_set)}" + ) + assert len(found) == 17, f"Expected 17 S5 subsections, found {len(found)}" + + def test_summary_matrix_has_all_rows(self) -> None: + """T03: Summary matrix has 17 rows, all 6 tool columns, valid cell values.""" + text = _read_guide() + rows = _parse_summary_matrix(text) + + # Exactly 17 data rows + assert len(rows) == 17, f"Expected 17 summary matrix rows, found {len(rows)}" + + # All six tool columns present + for tool in SIX_TOOLS: + assert all(tool in row for row in rows), ( + f"Tool column '{tool}' missing from summary matrix" + ) + + # Every tool cell is Y, P, N, or -- + for row in rows: + for tool in SIX_TOOLS: + val = row[tool] + assert val in VALID_SUPPORT_VALUES, ( + f"Invalid support value '{val}' for {tool} in row " + f"'{row.get('PSS/E Record Type', '?')}'. " + f"Must be one of {VALID_SUPPORT_VALUES}" + ) + + def test_summary_matrix_consistent_with_detail_sections(self) -> None: + """T04: Y/P/N values in S3 match per-record-type tables in S5.""" + text = _read_guide() + rows = _parse_summary_matrix(text) + + for row in rows: + rt = row["PSS/E Record Type"] + fnm_status = row["FNM Status"] + + # Skip empty record types (no detail tool support table) + if fnm_status.startswith("Empty"): + continue + + subsection = _get_s5_subsection_text(text, rt) + assert subsection is not None, f"S5 subsection not found for '{rt}'" + + detail_support = _parse_tool_support_table(subsection) + if not detail_support: + # Transformer has a special combined abstraction, check it exists + continue + + for tool in SIX_TOOLS: + matrix_val = row[tool] + detail_val = detail_support.get(tool) + if detail_val is not None: + assert matrix_val == detail_val, ( + f"Mismatch for '{rt}' / {tool}: " + f"S3 matrix says '{matrix_val}', S5 detail says '{detail_val}'" + ) + + def test_tier_classification_complete(self) -> None: + """T05: Every non-empty record type has Tier 1, 2, or 3.""" + text = _read_guide() + rows = _parse_summary_matrix(text) + + valid_tiers = {"1", "2", "3"} + for row in rows: + tier = row.get("Tier", "").strip() + assert tier in valid_tiers, ( + f"Record type '{row['PSS/E Record Type']}' has invalid tier '{tier}'. " + f"Must be one of {valid_tiers}" + ) + + def test_abstraction_vocabulary_covers_all_types(self) -> None: + """T06: S2 abstraction vocabulary covers all record types in S3.""" + text = _read_guide() + + # Parse S2 abstraction vocabulary table + s2_match = re.search( + r"## S2: Abstraction Vocabulary\s*\n(.*?)(?=\n## S3:|\Z)", + text, + re.DOTALL, + ) + assert s2_match, "S2: Abstraction Vocabulary section not found" + s2_text = s2_match.group(1) + + # Extract abstraction names from the table + abstractions_found: set[str] = set() + for line in s2_text.split("\n"): + line = line.strip() + if line.startswith("|") and not re.match(r"^\|[-\s|]+\|$", line): + cells = [c.strip() for c in line.split("|")[1:-1]] + if len(cells) >= 1 and cells[0] not in ("Abstraction", ""): + abstractions_found.add(cells[0]) + + # Every required abstraction must be present + for abstraction in REQUIRED_ABSTRACTIONS: + assert abstraction in abstractions_found, ( + f"Required abstraction '{abstraction}' not found in S2 vocabulary. " + f"Found: {sorted(abstractions_found)}" + ) + + def test_empty_record_types_marked(self) -> None: + """T07: Empty record types have 'not present in FNM' and no tool support table.""" + text = _read_guide() + rows = _parse_summary_matrix(text) + + for row in rows: + rt = row["PSS/E Record Type"] + fnm_status = row["FNM Status"] + + if not fnm_status.startswith("Empty"): + continue + + subsection = _get_s5_subsection_text(text, rt) + assert subsection is not None, f"S5 subsection not found for empty type '{rt}'" + + # Must contain "not present in FNM" (case-insensitive) + assert re.search(r"not present in (?:the )?FNM", subsection, re.IGNORECASE), ( + f"Empty record type '{rt}' subsection does not contain 'not present in FNM'" + ) + + # Must NOT contain a tool support table (no "| Tool |" header row) + has_tool_table = bool(re.search(r"\|\s*Tool\s*\|", subsection)) + assert not has_tool_table, ( + f"Empty record type '{rt}' should not have a tool support table" + ) + + +# --------------------------------------------------------------------------- +# T08-T10: Content consistency tests +# --------------------------------------------------------------------------- + + +class TestContentConsistency: + """Content consistency checks for tier classification and transformer mapping.""" + + def test_tier1_contains_essential_types(self) -> None: + """T08: Bus, Load, Generator, Branch, Transformer are all Tier 1.""" + text = _read_guide() + rows = _parse_summary_matrix(text) + + rt_to_tier = {row["PSS/E Record Type"]: row["Tier"] for row in rows} + + for rt in TIER1_ESSENTIAL: + assert rt_to_tier.get(rt) == "1", ( + f"Essential record type '{rt}' must be Tier 1, " + f"but found Tier {rt_to_tier.get(rt, 'MISSING')}" + ) + + def test_tier3_types_are_non_electrical(self) -> None: + """T09: Every Tier 3 type is Zone, Owner, or Interarea Transfer.""" + text = _read_guide() + rows = _parse_summary_matrix(text) + + for row in rows: + if row["Tier"] == "3": + rt = row["PSS/E Record Type"] + assert rt in TIER3_TYPES, ( + f"Record type '{rt}' is classified as Tier 3 but is not in the " + f"expected Tier 3 set {TIER3_TYPES}. Record types with direct " + f"electrical effect should not be Tier 3." + ) + + def test_transformer_section_covers_both_abstractions(self) -> None: + """T10: Transformer subsection mentions both 2-winding and 3-winding, plus K field.""" + text = _read_guide() + subsection = _get_s5_subsection_text(text, "Transformer") + assert subsection is not None, "Transformer subsection not found in S5" + + assert "2-Winding Transformer" in subsection, ( + "Transformer subsection must mention '2-Winding Transformer'" + ) + assert "3-Winding Transformer" in subsection, ( + "Transformer subsection must mention '3-Winding Transformer'" + ) + + # K field as distinguishing criterion + k_field_mentioned = bool( + re.search(r"\bK[ -]?field\b|\bK[= ]*0\b|\bK!=0\b|\bK is\b|\bK=0\b", subsection) + ) + assert k_field_mentioned, ( + "Transformer subsection must mention the K field as the criterion " + "distinguishing 2-winding from 3-winding transformers" + ) + + +# --------------------------------------------------------------------------- +# T11-T12: Cross-reference tests +# --------------------------------------------------------------------------- + + +class TestCrossReferences: + """Cross-reference validation tests.""" + + def test_cross_references_section_exists(self) -> None: + """T11: S6 exists and references required companion documents.""" + text = _read_guide() + + # S6 section must exist + assert re.search(r"## S6: Cross-References", text), "S6: Cross-References section not found" + + s6_match = re.search( + r"## S6: Cross-References\s*\n(.*?)(?=\n## |\Z)", + text, + re.DOTALL, + ) + assert s6_match, "Could not extract S6 content" + s6_text = s6_match.group(1) + + for ref_file in CROSS_REF_FILES: + assert ref_file in s6_text, f"Cross-reference to '{ref_file}' not found in S6 section" + + @pytest.mark.fnm + def test_non_empty_record_types_match_d3(self, require_fnm: Path) -> None: + """T12: Non-empty record types in S3 match D3 raw record counter output.""" + # Look for raw_counts.json in the intermediate directory + raw_counts_path = REPO_ROOT / "data" / "fnm" / "intermediate" / "raw_counts.json" + if not raw_counts_path.exists(): + pytest.skip(f"D3 raw counts file not found at {raw_counts_path}") + + with open(raw_counts_path, encoding="utf-8") as f: + raw_counts = json.load(f) + + # Extract non-empty sections from D3 output + d3_non_empty: set[str] = set() + if "sections" in raw_counts: + for section in raw_counts["sections"]: + name = section.get("name", "") + count = section.get("record_count", 0) + if count > 0: + d3_non_empty.add(name) + elif "non_empty_sections" in raw_counts: + d3_non_empty = set(raw_counts["non_empty_sections"]) + + # Extract non-empty types from summary matrix + text = _read_guide() + rows = _parse_summary_matrix(text) + matrix_non_empty: set[str] = set() + for row in rows: + if row["FNM Status"].startswith("Non-empty"): + matrix_non_empty.add(row["PSS/E Record Type"]) + + assert matrix_non_empty == d3_non_empty, ( + f"Non-empty record types mismatch.\n" + f" In S3 matrix but not in D3: {matrix_non_empty - d3_non_empty}\n" + f" In D3 but not in S3 matrix: {d3_non_empty - matrix_non_empty}" + ) diff --git a/data/fnm/tests/test_matpower_parser.py b/data/fnm/tests/test_matpower_parser.py new file mode 100644 index 00000000..62a63f6d --- /dev/null +++ b/data/fnm/tests/test_matpower_parser.py @@ -0,0 +1,415 @@ +"""Tests for the MATPOWER psse2mpc parser wrapper (PRD 01/04). + +T01-T08: Synthetic tests (no external dependencies). +T09: Octave integration test (requires Octave + MATPOWER, no FNM). +T10-T13: FNM integration tests (require FNM_PATH, skip if unset). +""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path + +import pytest + +from fnm.scripts.matpower_parser import ( + MPC_DROPPED_RECORD_TYPES, + MPC_LOSSY_RECORD_TYPES, + MatpowerParserLog, + MatpowerParserSummary, + ParserWarning, + build_known_limitations, + build_octave_command, + log_to_dict, + parse_octave_stdout, + parse_octave_warnings, + read_csv_field_counts, + run_psse2mpc, + summary_to_dict, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_VALID_HEADER_LINES = [ + " 0, 100.00, 31.0, 0, 0, 60.00 / PSS/E-31.0 test case", + "Test case identification line 1", + "Test case identification line 2", +] + + +def _make_raw_content( + header: list[str] | None = None, + sections: list[list[str]] | None = None, +) -> str: + """Build a minimal PSS/E v31 RAW file as a string.""" + hdr = header or _VALID_HEADER_LINES + secs = sections or [[] for _ in range(17)] + lines = list(hdr) + for body in secs: + lines.extend(body) + lines.append(" 0") + return "\n".join(lines) + "\n" + + +def _build_synthetic_raw() -> str: + """Build a synthetic 3-bus PSS/E v31 RAW file suitable for psse2mpc. + + Contains: + - 3 buses (slack + PV + PQ) + - 1 generator on bus 1 + - 2 branches (1-2, 2-3) + - 1 load on bus 3 + - Remaining sections empty + """ + bus_data = [ + " 1,'BUS1 ', 138.000,3, 1, 1, 1,1.05000, 0.0000", + " 2,'BUS2 ', 138.000,2, 1, 1, 1,1.03000, -2.1000", + " 3,'BUS3 ', 69.000,1, 1, 1, 1,1.01000, -5.0000", + ] + load_data = [ + " 3,'1 ',1,1,1, 50.000, 25.000, 0.000, 0.000, 0.000, 0.000,1", + ] + # Fixed shunt: empty + gen_data = [ + " 1,'1 ', 100.000, 0.000, 999.000, -999.000,1.05000,0, 200.000," + " 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000," + " 200.000, 0.000, 1,1.0000, 0,1.0000, 0,1,0, 0.000, 0.000", + ] + branch_data = [ + " 1, 2,'1 ', 0.01000, 0.10000, 0.02000, 200.00, 200.00, 200.00," + "0.00000,0.00000,0.00000,0.00000,1,1, 0.000, 1, 1,0.0000, 0, 0", + " 2, 3,'1 ', 0.02000, 0.20000, 0.04000, 100.00, 100.00, 100.00," + "0.00000,0.00000,0.00000,0.00000,1,1, 0.000, 1, 1,0.0000, 0, 0", + ] + # Sections: Bus, Load, Fixed Shunt, Generator, Branch, Transformer, + # Area, Two-Term DC, VSC DC, Imp Corr, MTDC, MSL, Zone, + # Interarea, Owner, FACTS, Switched Shunt + sections: list[list[str]] = [[] for _ in range(17)] + sections[0] = bus_data + sections[1] = load_data + # sections[2] = fixed shunt (empty) + sections[3] = gen_data + sections[4] = branch_data + return _make_raw_content(sections=sections) + + +# --------------------------------------------------------------------------- +# T01: test_build_octave_command_structure +# --------------------------------------------------------------------------- + + +def test_build_octave_command_structure() -> None: + """Verify command list structure from build_octave_command.""" + cmd = build_octave_command("/tmp/test.raw", "/tmp/output") + assert cmd[0] == "octave" + assert "--no-gui" in cmd + assert "--no-init-file" in cmd + # Script path should end with run_psse2mpc.m + assert any(arg.endswith("run_psse2mpc.m") for arg in cmd) + # raw_path and output_dir should be in the command + assert "/tmp/test.raw" in cmd + assert "/tmp/output" in cmd + + +# --------------------------------------------------------------------------- +# T02: test_parse_octave_stdout_valid +# --------------------------------------------------------------------------- + + +def test_parse_octave_stdout_valid() -> None: + """Parse synthetic stdout with MPC_BASEMVA, MPC_VERSION, MPC_FIELD_COUNT lines.""" + stdout = ( + "MPC_BASEMVA:100\n" + "MPC_VERSION:2\n" + "MPC_FIELD_COUNT:bus:3\n" + "MPC_FIELD_COUNT:gen:1\n" + "MPC_FIELD_COUNT:branch:2\n" + "CONVERSION_COMPLETE\n" + ) + result = parse_octave_stdout(stdout) + assert result["_baseMVA"] == 100.0 + assert result["_version"] == "2" + assert result["bus"] == 3 + assert result["gen"] == 1 + assert result["branch"] == 2 + + +# --------------------------------------------------------------------------- +# T03: test_parse_octave_stdout_empty +# --------------------------------------------------------------------------- + + +def test_parse_octave_stdout_empty() -> None: + """Empty string should return empty dict.""" + assert parse_octave_stdout("") == {} + assert parse_octave_stdout(" \n \n") == {} + + +# --------------------------------------------------------------------------- +# T04: test_parse_octave_warnings_classification +# --------------------------------------------------------------------------- + + +def test_parse_octave_warnings_classification() -> None: + """Classify warnings as skipped_record, phantom_bus, unsupported_field, conversion_warning.""" + stderr = ( + "PSSE2MPC_WARNING: Skipping Two-Terminal DC records\n" + "PSSE2MPC_WARNING: Phantom bus 99999 referenced but not in bus table\n" + "PSSE2MPC_WARNING: Unsupported field type in transformer record\n" + "PSSE2MPC_WARNING: Some generic conversion issue occurred\n" + "\n" # empty line should be skipped + ) + warnings = parse_octave_warnings(stderr) + assert len(warnings) == 4 + categories = [w.category for w in warnings] + assert categories[0] == "skipped_record" + assert categories[1] == "phantom_bus" + assert categories[2] == "unsupported_field" + assert categories[3] == "conversion_warning" + + +# --------------------------------------------------------------------------- +# T05: test_read_csv_field_counts +# --------------------------------------------------------------------------- + + +def test_read_csv_field_counts(tmp_path: Path) -> None: + """Create temp CSVs, verify row counts.""" + # Create synthetic CSV files (no headers, as csvwrite produces) + (tmp_path / "mpc_bus.csv").write_text("1,138.0,1.05,0\n2,138.0,1.03,-2.1\n3,69.0,1.01,-5.0\n") + (tmp_path / "mpc_gen.csv").write_text("1,100,0,999,-999\n") + (tmp_path / "mpc_branch.csv").write_text("1,2,0.01,0.1,0.02\n2,3,0.02,0.2,0.04\n") + # Non-matching file should be ignored + (tmp_path / "other.csv").write_text("should,be,ignored\n") + + counts = read_csv_field_counts(tmp_path) + assert counts["bus"] == 3 + assert counts["gen"] == 1 + assert counts["branch"] == 2 + assert "other" not in counts + + +# --------------------------------------------------------------------------- +# T06: test_build_known_limitations +# --------------------------------------------------------------------------- + + +def test_build_known_limitations() -> None: + """Verify all dropped + lossy record types are covered.""" + limitations = build_known_limitations() + record_types = {kl.record_type for kl in limitations} + + # All dropped types must be present + for rt in MPC_DROPPED_RECORD_TYPES: + assert rt in record_types, f"Missing dropped record type: {rt}" + + # All lossy types must be present + for rt in MPC_LOSSY_RECORD_TYPES: + assert rt in record_types, f"Missing lossy record type: {rt}" + + # Verify behaviors + dropped = {kl.record_type for kl in limitations if kl.behavior == "dropped"} + lossy = {kl.record_type for kl in limitations if kl.behavior == "lossy"} + assert dropped == set(MPC_DROPPED_RECORD_TYPES) + assert lossy == set(MPC_LOSSY_RECORD_TYPES) + + # All must have non-empty descriptions + for kl in limitations: + assert kl.description, f"Empty description for {kl.record_type}" + + +# --------------------------------------------------------------------------- +# T07: test_log_to_dict_json_serializable +# --------------------------------------------------------------------------- + + +def test_log_to_dict_json_serializable() -> None: + """Build MatpowerParserLog, convert, json.dumps succeeds.""" + log = MatpowerParserLog( + raw_path="/tmp/test.raw", + output_dir="/tmp/output", + return_code=0, + stdout="MPC_BASEMVA:100\nMPC_FIELD_COUNT:bus:3\n", + stderr="PSSE2MPC_WARNING: test warning\n", + baseMVA=100.0, + version="2", + field_counts_octave={"bus": 3}, + field_counts_csv={"bus": 3}, + warnings=[ParserWarning(line="test warning", category="conversion_warning")], + ) + d = log_to_dict(log) + json_str = json.dumps(d) + loaded = json.loads(json_str) + assert loaded["baseMVA"] == 100.0 + assert loaded["return_code"] == 0 + assert loaded["field_counts_octave"]["bus"] == 3 + assert len(loaded["warnings"]) == 1 + assert loaded["warnings"][0]["category"] == "conversion_warning" + + +# --------------------------------------------------------------------------- +# T08: test_summary_to_dict_json_serializable +# --------------------------------------------------------------------------- + + +def test_summary_to_dict_json_serializable() -> None: + """Build MatpowerParserSummary, convert, json.dumps succeeds.""" + log = MatpowerParserLog( + raw_path="/tmp/test.raw", + output_dir="/tmp/output", + return_code=0, + stdout="", + stderr="", + baseMVA=100.0, + version="2", + ) + limitations = build_known_limitations() + summary = MatpowerParserSummary( + log=log, + known_limitations=limitations, + success=True, + ) + d = summary_to_dict(summary) + json_str = json.dumps(d) + loaded = json.loads(json_str) + assert loaded["success"] is True + assert "log" in loaded + assert "known_limitations" in loaded + assert len(loaded["known_limitations"]) == len(MPC_DROPPED_RECORD_TYPES) + len( + MPC_LOSSY_RECORD_TYPES + ) + # Verify each limitation has required keys + for kl in loaded["known_limitations"]: + assert "record_type" in kl + assert "behavior" in kl + assert "description" in kl + + +# --------------------------------------------------------------------------- +# T09: test_run_psse2mpc_synthetic_case (Octave integration) +# --------------------------------------------------------------------------- + + +@pytest.mark.octave +def test_run_psse2mpc_synthetic_case(tmp_path: Path) -> None: + """Run psse2mpc on a synthetic PSS/E v31 RAW file. + + Verifies: + - Return code is 0 (success). + - Summary has bus/gen/branch counts. + - CSV files exist in the output directory. + """ + if shutil.which("octave") is None: + pytest.skip("Octave not available") + + # Write synthetic RAW file + raw_content = _build_synthetic_raw() + raw_path = tmp_path / "synthetic.raw" + raw_path.write_text(raw_content, encoding="utf-8") + + output_dir = tmp_path / "mpc_output" + + log = run_psse2mpc(raw_path, output_dir, timeout=120) + + # Conversion should succeed + assert log.return_code == 0, ( + f"psse2mpc failed with return code {log.return_code}.\n" + f"stdout: {log.stdout}\nstderr: {log.stderr}" + ) + + # Should have baseMVA + assert log.baseMVA is not None + assert log.baseMVA == 100.0 + + # Should have bus, gen, branch counts from Octave stdout + assert "bus" in log.field_counts_octave + assert log.field_counts_octave["bus"] == 3 + assert "gen" in log.field_counts_octave + assert log.field_counts_octave["gen"] == 1 + assert "branch" in log.field_counts_octave + assert log.field_counts_octave["branch"] == 2 + + # CSV files should exist + assert (output_dir / "mpc_bus.csv").exists() + assert (output_dir / "mpc_gen.csv").exists() + assert (output_dir / "mpc_branch.csv").exists() + + # CSV field counts should match Octave counts + assert log.field_counts_csv.get("bus") == 3 + assert log.field_counts_csv.get("gen") == 1 + assert log.field_counts_csv.get("branch") == 2 + + +# --------------------------------------------------------------------------- +# FNM integration tests (T10-T13) — require FNM_PATH +# --------------------------------------------------------------------------- + + +@pytest.mark.fnm +def test_fnm_psse2mpc_converts(require_fnm_raw: Path, tmp_path: Path) -> None: + """T10: psse2mpc converts the real FNM RAW file without error.""" + if shutil.which("octave") is None: + pytest.skip("Octave not available") + + output_dir = tmp_path / "fnm_mpc_output" + log = run_psse2mpc(require_fnm_raw, output_dir, timeout=300) + + assert log.return_code == 0, f"psse2mpc failed on FNM RAW file.\nstderr: {log.stderr[:500]}" + assert log.baseMVA is not None + + +@pytest.mark.fnm +def test_fnm_bus_csv_exists(require_fnm_raw: Path, tmp_path: Path) -> None: + """T11: mpc_bus.csv is produced and has rows in production-scale range.""" + if shutil.which("octave") is None: + pytest.skip("Octave not available") + + output_dir = tmp_path / "fnm_mpc_output" + log = run_psse2mpc(require_fnm_raw, output_dir, timeout=300) + + assert (output_dir / "mpc_bus.csv").exists() + bus_count = log.field_counts_csv.get("bus", 0) + assert 25000 <= bus_count <= 35000, ( + f"Bus count {bus_count} outside expected production-scale range" + ) + + +@pytest.mark.fnm +def test_fnm_branch_csv_exists(require_fnm_raw: Path, tmp_path: Path) -> None: + """T12: mpc_branch.csv is produced with a reasonable number of branches.""" + if shutil.which("octave") is None: + pytest.skip("Octave not available") + + output_dir = tmp_path / "fnm_mpc_output" + log = run_psse2mpc(require_fnm_raw, output_dir, timeout=300) + + assert (output_dir / "mpc_branch.csv").exists() + branch_count = log.field_counts_csv.get("branch", 0) + assert branch_count > 1000, ( + f"Branch count {branch_count} seems too low for a production-scale network" + ) + + +@pytest.mark.fnm +def test_fnm_known_limitations_documented(require_fnm_raw: Path, tmp_path: Path) -> None: + """T13: Known limitations are documented and summary is JSON-serializable.""" + if shutil.which("octave") is None: + pytest.skip("Octave not available") + + output_dir = tmp_path / "fnm_mpc_output" + log = run_psse2mpc(require_fnm_raw, output_dir, timeout=300) + limitations = build_known_limitations() + summary = MatpowerParserSummary( + log=log, + known_limitations=limitations, + success=log.return_code == 0, + ) + + d = summary_to_dict(summary) + json_str = json.dumps(d) + loaded = json.loads(json_str) + assert loaded["success"] is True + assert len(loaded["known_limitations"]) >= 9 # 6 dropped + 3 lossy diff --git a/data/fnm/tests/test_parser_comparison.py b/data/fnm/tests/test_parser_comparison.py new file mode 100644 index 00000000..a469e01c --- /dev/null +++ b/data/fnm/tests/test_parser_comparison.py @@ -0,0 +1,575 @@ +"""Tests for parser fidelity comparison and canonical parser selection (PRD 01/06). + +T01-T12: Synthetic tests (no FNM data required). +T13-T14: FNM integration tests (require FNM_PATH, skip if unset). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from fnm.scripts.parser_comparison import ( + CanonicalParserSelection, + ComparisonMetadata, + DiscrepancyType, + FidelityScore, + FieldCoverageEntry, + ParserComparisonReport, + ParserName, + RecordCountComparison, + SelectionRationale, + build_comparison_report, + build_data_loss_inventory, + compare_field_coverage, + compare_record_counts, + compute_fidelity_score, + report_to_dict, + select_canonical_parser, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_simple_mapping() -> dict[str, tuple[str | None, str | None]]: + """Return a minimal PSS/E -> parser table mapping for tests.""" + return { + "Bus": ("bus", "buses"), + "Load": (None, "loads"), + "Generator": ("gen", "generators"), + "Branch": ("branch", "lines"), + } + + +def _make_simple_psse_spec() -> dict[str, list[str]]: + """Return a minimal PSS/E field spec for tests.""" + return { + "Bus": ["I", "NAME", "BASKV", "IDE", "AREA", "ZONE", "VM", "VA"], + "Generator": ["I", "ID", "PG", "QG", "QT", "QB", "VS", "MBASE"], + "Branch": ["I", "J", "CKT", "R", "X", "B", "RATEA"], + } + + +def _make_simple_tier1() -> dict[str, list[str]]: + """Return a minimal tier-1 fields spec for tests.""" + return { + "Bus": ["I", "NAME", "BASKV", "VM", "VA"], + "Generator": ["I", "PG", "QG", "VS"], + } + + +# --------------------------------------------------------------------------- +# T01: test_compare_record_counts_all_match +# --------------------------------------------------------------------------- + + +def test_compare_record_counts_all_match() -> None: + """All counts match between raw and both parsers -> MATCH.""" + raw = {"Bus": 10, "Generator": 3, "Branch": 15} + matpower = {"bus": 10, "gen": 3, "branch": 15} + gridcal = {"buses": 10, "generators": 3, "lines": 15} + mapping = { + "Bus": ("bus", "buses"), + "Generator": ("gen", "generators"), + "Branch": ("branch", "lines"), + } + + result = compare_record_counts(raw, matpower, gridcal, mapping) + + assert len(result) == 3 + for c in result: + assert c.matpower_discrepancy == DiscrepancyType.MATCH + assert c.gridcal_discrepancy == DiscrepancyType.MATCH + + +# --------------------------------------------------------------------------- +# T02: test_compare_record_counts_data_loss +# --------------------------------------------------------------------------- + + +def test_compare_record_counts_data_loss() -> None: + """Parser count < raw count -> DATA_LOSS.""" + raw = {"Bus": 10} + matpower = {"bus": 8} + gridcal = {"buses": 10} + mapping = {"Bus": ("bus", "buses")} + + result = compare_record_counts(raw, matpower, gridcal, mapping) + + assert len(result) == 1 + assert result[0].matpower_discrepancy == DiscrepancyType.DATA_LOSS + assert result[0].gridcal_discrepancy == DiscrepancyType.MATCH + assert result[0].matpower_count == 8 + + +# --------------------------------------------------------------------------- +# T03: test_compare_record_counts_phantom_insertion +# --------------------------------------------------------------------------- + + +def test_compare_record_counts_phantom_insertion() -> None: + """Parser count > raw count -> PHANTOM_INSERTION.""" + raw = {"Bus": 10} + matpower = {"bus": 12} + gridcal = {"buses": 10} + mapping = {"Bus": ("bus", "buses")} + + result = compare_record_counts(raw, matpower, gridcal, mapping) + + assert len(result) == 1 + assert result[0].matpower_discrepancy == DiscrepancyType.PHANTOM_INSERTION + assert result[0].gridcal_discrepancy == DiscrepancyType.MATCH + + +# --------------------------------------------------------------------------- +# T04: test_compare_record_counts_record_type_missing +# --------------------------------------------------------------------------- + + +def test_compare_record_counts_record_type_missing() -> None: + """Parser has no table for a record type with raw data -> RECORD_TYPE_MISSING.""" + raw = {"Load": 5, "Bus": 10} + matpower = {"bus": 10} + gridcal = {"buses": 10, "loads": 5} + # Load maps to None for MATPOWER + mapping = {"Load": (None, "loads"), "Bus": ("bus", "buses")} + + result = compare_record_counts(raw, matpower, gridcal, mapping) + + load_cmp = [c for c in result if c.psse_section == "Load"][0] + assert load_cmp.matpower_discrepancy == DiscrepancyType.RECORD_TYPE_MISSING + assert load_cmp.gridcal_discrepancy == DiscrepancyType.MATCH + + +# --------------------------------------------------------------------------- +# T05: test_compare_field_coverage_common_and_unique +# --------------------------------------------------------------------------- + + +def test_compare_field_coverage_common_and_unique() -> None: + """Verify common, matpower-only, and gridcal-only field lists.""" + psse_spec = {"Bus": ["I", "NAME", "BASKV", "VM", "VA"]} + mp_columns = {"bus": ["i", "name", "baskv", "extra_mp"]} + gc_columns = {"buses": ["i", "name", "baskv", "extra_gc"]} + mapping = {"Bus": ("bus", "buses")} + + result = compare_field_coverage(psse_spec, mp_columns, gc_columns, mapping) + + assert len(result) == 1 + entry = result[0] + assert "i" in entry.common_fields + assert "name" in entry.common_fields + assert "baskv" in entry.common_fields + assert "extra_mp" in entry.matpower_only + assert "extra_gc" in entry.gridcal_only + # Coverage: 4 parser fields / 5 psse fields = 0.8 + assert abs(entry.matpower_coverage - 0.8) < 1e-6 + assert abs(entry.gridcal_coverage - 0.8) < 1e-6 + + +# --------------------------------------------------------------------------- +# T06: test_build_data_loss_inventory_from_discrepancies +# --------------------------------------------------------------------------- + + +def test_build_data_loss_inventory_from_discrepancies() -> None: + """Verify inventory entries are created for non-MATCH discrepancies.""" + counts = [ + RecordCountComparison( + psse_section="Bus", + raw_count=10, + matpower_count=8, + gridcal_count=10, + matpower_discrepancy=DiscrepancyType.DATA_LOSS, + gridcal_discrepancy=DiscrepancyType.MATCH, + ), + RecordCountComparison( + psse_section="Load", + raw_count=5, + matpower_count=None, + gridcal_count=5, + matpower_discrepancy=DiscrepancyType.RECORD_TYPE_MISSING, + gridcal_discrepancy=DiscrepancyType.MATCH, + ), + ] + fields: list[FieldCoverageEntry] = [] + + inventory = build_data_loss_inventory(counts, fields) + + assert len(inventory) == 2 + bus_loss = [e for e in inventory if e.psse_section == "Bus"][0] + assert bus_loss.parser == ParserName.MATPOWER + assert bus_loss.loss_type == DiscrepancyType.DATA_LOSS + assert bus_loss.delta == -2 + + load_loss = [e for e in inventory if e.psse_section == "Load"][0] + assert load_loss.parser == ParserName.MATPOWER + assert load_loss.loss_type == DiscrepancyType.RECORD_TYPE_MISSING + assert load_loss.delta is None + + +# --------------------------------------------------------------------------- +# T07: test_compute_fidelity_score_perfect +# --------------------------------------------------------------------------- + + +def test_compute_fidelity_score_perfect() -> None: + """All counts match, full field coverage -> score 1.0.""" + psse_spec = _make_simple_psse_spec() + tier1 = _make_simple_tier1() + + counts = [ + RecordCountComparison( + psse_section=s, + raw_count=10, + matpower_count=10, + gridcal_count=10, + matpower_discrepancy=DiscrepancyType.MATCH, + gridcal_discrepancy=DiscrepancyType.MATCH, + ) + for s in psse_spec + ] + + # Full field coverage: parser has all PSS/E fields + fields = [ + FieldCoverageEntry( + psse_section=s, + psse_fields=psse_spec[s], + matpower_fields=psse_spec[s], + gridcal_fields=psse_spec[s], + common_fields=[f.lower() for f in psse_spec[s]], + matpower_only=[], + gridcal_only=[], + matpower_coverage=1.0, + gridcal_coverage=1.0, + ) + for s in psse_spec + ] + + score = compute_fidelity_score(ParserName.MATPOWER, counts, fields, [], psse_spec, tier1) + + assert abs(score.overall - 1.0) < 1e-6 + assert abs(score.field_coverage - 1.0) < 1e-6 + assert abs(score.record_type_coverage - 1.0) < 1e-6 + assert abs(score.tier1_field_coverage - 1.0) < 1e-6 + assert abs(score.record_count_accuracy - 1.0) < 1e-6 + assert score.phantom_count == 0 + + +# --------------------------------------------------------------------------- +# T08: test_compute_fidelity_score_partial_coverage +# --------------------------------------------------------------------------- + + +def test_compute_fidelity_score_partial_coverage() -> None: + """Partial field coverage and some losses produce correct weighted score.""" + psse_spec = {"Bus": ["I", "NAME", "BASKV", "VM"]} + tier1 = {"Bus": ["I", "NAME"]} + + counts = [ + RecordCountComparison( + psse_section="Bus", + raw_count=10, + matpower_count=8, + gridcal_count=10, + matpower_discrepancy=DiscrepancyType.DATA_LOSS, + gridcal_discrepancy=DiscrepancyType.MATCH, + ), + ] + + # MATPOWER has 2/4 fields, GridCal has 4/4 + fields = [ + FieldCoverageEntry( + psse_section="Bus", + psse_fields=["I", "NAME", "BASKV", "VM"], + matpower_fields=["I", "NAME"], + gridcal_fields=["I", "NAME", "BASKV", "VM"], + common_fields=["i", "name"], + matpower_only=[], + gridcal_only=["baskv", "vm"], + matpower_coverage=0.5, + gridcal_coverage=1.0, + ), + ] + + mp_score = compute_fidelity_score(ParserName.MATPOWER, counts, fields, [], psse_spec, tier1) + gc_score = compute_fidelity_score(ParserName.GRIDCAL, counts, fields, [], psse_spec, tier1) + + # MATPOWER: field_cov=0.5, rt_cov=1.0, tier1=1.0 (I,NAME both present), + # rc_acc=0 (DATA_LOSS) + # overall = 0.35*0.5 + 0.30*1.0 + 0.20*1.0 + 0.15*0.0 = 0.675 + assert abs(mp_score.overall - 0.675) < 1e-4 + + # GridCal: field_cov=1.0, rt_cov=1.0, tier1=1.0, rc_acc=1.0 + # overall = 0.35*1.0 + 0.30*1.0 + 0.20*1.0 + 0.15*1.0 = 1.0 + assert abs(gc_score.overall - 1.0) < 1e-4 + + assert gc_score.overall > mp_score.overall + + +# --------------------------------------------------------------------------- +# T09: test_select_canonical_clear_winner +# --------------------------------------------------------------------------- + + +def test_select_canonical_clear_winner() -> None: + """Score diff > 0.05 -> CLEAR_WINNER.""" + mp = FidelityScore( + parser=ParserName.MATPOWER, + overall=0.60, + field_coverage=0.5, + record_type_coverage=0.7, + tier1_field_coverage=0.6, + record_count_accuracy=0.5, + phantom_count=0, + ) + gc = FidelityScore( + parser=ParserName.GRIDCAL, + overall=0.85, + field_coverage=0.9, + record_type_coverage=0.8, + tier1_field_coverage=0.9, + record_count_accuracy=0.8, + phantom_count=0, + ) + + selection = select_canonical_parser(mp, gc) + + assert selection.selected == ParserName.GRIDCAL + assert selection.rationale == SelectionRationale.CLEAR_WINNER + assert selection.score_diff > 0.05 + + +# --------------------------------------------------------------------------- +# T10: test_select_canonical_tier1_tiebreak +# --------------------------------------------------------------------------- + + +def test_select_canonical_tier1_tiebreak() -> None: + """Scores close but tier1 differs > 0.02 -> TIER1_TIEBREAK.""" + mp = FidelityScore( + parser=ParserName.MATPOWER, + overall=0.80, + field_coverage=0.8, + record_type_coverage=0.8, + tier1_field_coverage=0.90, + record_count_accuracy=0.8, + phantom_count=0, + ) + gc = FidelityScore( + parser=ParserName.GRIDCAL, + overall=0.82, + field_coverage=0.82, + record_type_coverage=0.82, + tier1_field_coverage=0.70, + record_count_accuracy=0.82, + phantom_count=0, + ) + + selection = select_canonical_parser(mp, gc) + + assert selection.rationale == SelectionRationale.TIER1_TIEBREAK + assert selection.selected == ParserName.MATPOWER + + +# --------------------------------------------------------------------------- +# T11: test_build_comparison_report_end_to_end +# --------------------------------------------------------------------------- + + +def test_build_comparison_report_end_to_end(tmp_path: Path) -> None: + """Synthetic D3/D4/D5 files produce a full report.""" + # D3 raw counts + d3 = { + "section_counts": { + "Bus": 10, + "Load": 5, + "Generator": 3, + "Branch": 12, + "Transformer": 4, + "Area": 2, + "Fixed Shunt": 1, + "Switched Shunt": 2, + "Two-Terminal DC": 0, + "VSC DC": 0, + "Impedance Correction": 0, + "Multi-Terminal DC": 0, + "Multi-Section Line": 0, + "Zone": 3, + "Interarea Transfer": 0, + "Owner": 1, + "FACTS": 0, + } + } + d3_path = tmp_path / "d3_counts.json" + d3_path.write_text(json.dumps(d3), encoding="utf-8") + + # D4 MATPOWER summary (uses log.field_counts_csv) + d4 = { + "success": True, + "log": { + "field_counts_csv": { + "bus": 10, + "gen": 3, + "branch": 16, # branches + transformers merged + "areas": 2, + } + }, + } + d4_path = tmp_path / "d4_summary.json" + d4_path.write_text(json.dumps(d4), encoding="utf-8") + + # D5 GridCal summary (uses multicircuit_counts) + d5 = { + "multicircuit_counts": { + "buses": 10, + "loads": 5, + "shunts": 1, + "generators": 3, + "lines": 12, + "transformers2w": 4, + "areas": 2, + "zones": 3, + "controllable_shunts": 2, + } + } + d5_path = tmp_path / "d5_summary.json" + d5_path.write_text(json.dumps(d5), encoding="utf-8") + + # CSV directories (empty, but must exist) + mp_csvs = tmp_path / "matpower_csvs" + mp_csvs.mkdir() + gc_csvs = tmp_path / "gridcal_csvs" + gc_csvs.mkdir() + + report = build_comparison_report(d3_path, d4_path, d5_path, mp_csvs, gc_csvs) + + assert isinstance(report, ParserComparisonReport) + assert len(report.record_counts) == 17 + assert isinstance(report.matpower_fidelity, FidelityScore) + assert isinstance(report.gridcal_fidelity, FidelityScore) + assert isinstance(report.selection, CanonicalParserSelection) + assert report.selection.selected in (ParserName.MATPOWER, ParserName.GRIDCAL) + + # GridCal should score higher since it preserves more record types + assert report.gridcal_fidelity.overall > report.matpower_fidelity.overall + + +# --------------------------------------------------------------------------- +# T12: test_report_to_dict_and_json_roundtrip +# --------------------------------------------------------------------------- + + +def test_report_to_dict_and_json_roundtrip() -> None: + """json.dumps succeeds and all expected top-level keys are present.""" + metadata = ComparisonMetadata( + timestamp="2026-01-01T00:00:00Z", + raw_counts_path="/d3.json", + matpower_summary_path="/d4.json", + gridcal_summary_path="/d5.json", + matpower_csv_dir="/mp", + gridcal_csv_dir="/gc", + ) + mp_fidelity = FidelityScore( + parser=ParserName.MATPOWER, + overall=0.75, + field_coverage=0.7, + record_type_coverage=0.8, + tier1_field_coverage=0.8, + record_count_accuracy=0.7, + phantom_count=1, + ) + gc_fidelity = FidelityScore( + parser=ParserName.GRIDCAL, + overall=0.85, + field_coverage=0.9, + record_type_coverage=0.8, + tier1_field_coverage=0.9, + record_count_accuracy=0.8, + phantom_count=0, + ) + selection = CanonicalParserSelection( + selected=ParserName.GRIDCAL, + rationale=SelectionRationale.CLEAR_WINNER, + matpower_score=0.75, + gridcal_score=0.85, + score_diff=0.10, + explanation="GridCal wins.", + ) + + report = ParserComparisonReport( + metadata=metadata, + record_counts=[ + RecordCountComparison( + psse_section="Bus", + raw_count=10, + matpower_count=10, + gridcal_count=10, + matpower_discrepancy=DiscrepancyType.MATCH, + gridcal_discrepancy=DiscrepancyType.MATCH, + ), + ], + field_coverage=[ + FieldCoverageEntry( + psse_section="Bus", + psse_fields=["I", "NAME"], + matpower_fields=["i", "name"], + gridcal_fields=["i", "name"], + common_fields=["i", "name"], + matpower_only=[], + gridcal_only=[], + matpower_coverage=1.0, + gridcal_coverage=1.0, + ), + ], + data_loss_inventory=[], + matpower_fidelity=mp_fidelity, + gridcal_fidelity=gc_fidelity, + selection=selection, + ) + + d = report_to_dict(report) + + # json.dumps must succeed + json_str = json.dumps(d, indent=2) + assert isinstance(json_str, str) + + # Roundtrip: parse back and check keys + parsed = json.loads(json_str) + expected_keys = { + "metadata", + "record_counts", + "field_coverage", + "data_loss_inventory", + "matpower_fidelity", + "gridcal_fidelity", + "selection", + } + assert set(parsed.keys()) == expected_keys + + # Check nested structure + assert parsed["selection"]["selected"] == "GRIDCAL" + assert parsed["selection"]["rationale"] == "CLEAR_WINNER" + assert parsed["matpower_fidelity"]["parser"] == "MATPOWER" + assert parsed["gridcal_fidelity"]["parser"] == "GRIDCAL" + + +# --------------------------------------------------------------------------- +# T13-T14: FNM integration tests (skip if FNM_PATH unset) +# --------------------------------------------------------------------------- + + +@pytest.mark.fnm +def test_fnm_full_comparison(require_fnm: dict) -> None: + """T13: Run full comparison on real FNM data (requires FNM_PATH).""" + pytest.skip("FNM integration test — requires FNM_PATH and D3/D4/D5 outputs") + + +@pytest.mark.fnm +def test_fnm_report_markdown_output(require_fnm: dict) -> None: + """T14: Generate markdown report from real FNM data (requires FNM_PATH).""" + pytest.skip("FNM integration test — requires FNM_PATH and D3/D4/D5 outputs") diff --git a/data/fnm/tests/test_pass_conditions.py b/data/fnm/tests/test_pass_conditions.py new file mode 100644 index 00000000..5f21629d --- /dev/null +++ b/data/fnm/tests/test_pass_conditions.py @@ -0,0 +1,608 @@ +"""Tests for Pass Condition Definitions (PRD 03/05). + +All tests use synthetic data with deterministic, known passing/failing counts. +No external dependencies beyond stdlib + pytest. +""" + +from __future__ import annotations + +import json +import math +from pathlib import Path + +from fnm.scripts.pass_conditions import ( + OutlierCause, + build_pass_condition_spec, + evaluate_acpf, + evaluate_dcpf, + load_spec, + write_json, + write_markdown, +) + +# --------------------------------------------------------------------------- +# Specification generation tests (T01-T04) +# --------------------------------------------------------------------------- + + +def test_build_spec_defaults() -> None: + """T01: Verify all default threshold values.""" + spec = build_pass_condition_spec() + + # ACPF aggregate + assert spec.acpf_aggregate.min_passing_fraction == 0.95 + assert spec.acpf_aggregate.vm_tolerance_pu == 0.005 + assert spec.acpf_aggregate.va_tolerance_deg == 0.5 + + # DCPF aggregate + assert spec.dcpf_aggregate.min_bus_passing_fraction == 0.95 + assert spec.dcpf_aggregate.va_tolerance_deg == 1.0 + assert spec.dcpf_aggregate.min_branch_passing_fraction == 0.90 + assert spec.dcpf_aggregate.p_tolerance_pct == 10.0 + assert spec.dcpf_aggregate.p_base_floor_mw == 1.0 + + # ACPF hard-fail + assert spec.acpf_hard_fail.max_failing_fraction == 0.20 + assert spec.acpf_hard_fail.vm_max_deviation_pu == 0.1 + assert spec.acpf_hard_fail.va_max_deviation_deg == 10.0 + + # DCPF hard-fail + assert spec.dcpf_hard_fail.max_bus_failing_fraction == 0.20 + assert spec.dcpf_hard_fail.max_branch_failing_fraction == 0.20 + assert spec.dcpf_hard_fail.p_max_deviation_pct == 50.0 + + # Version + assert spec.version == "1.0.0" + + +def test_spec_outlier_rules_count_and_order() -> None: + """T02: Verify outlier rule count, order, and content.""" + spec = build_pass_condition_spec() + rules = spec.outlier_classification.rules + + assert len(rules) == 5 + + expected_causes = [ + OutlierCause.SWITCHED_SHUNT, + OutlierCause.Q_LIMIT, + OutlierCause.SLACK_DISTRIBUTION, + OutlierCause.TAP_POSITION, + OutlierCause.ISLAND_BOUNDARY, + ] + for rule, expected_cause in zip(rules, expected_causes): + assert rule.cause == expected_cause + assert len(rule.description) > 0 + assert len(rule.required_data) > 0 + assert len(rule.match_condition) > 0 + + +def test_spec_voltage_tiers() -> None: + """T03: Verify voltage-level tier definitions.""" + spec = build_pass_condition_spec() + tiers = spec.voltage_level_tiers + + assert len(tiers) == 3 + + # Tier 1: transmission >= 230 kV + assert tiers[0].label == "transmission_230kv_plus" + assert tiers[0].min_kv == 230.0 + assert math.isinf(tiers[0].max_kv) + + # Tier 2: subtransmission 69-230 kV + assert tiers[1].label == "subtransmission_69_to_229kv" + assert tiers[1].min_kv == 69.0 + assert tiers[1].max_kv == 230.0 + + # Tier 3: distribution < 69 kV + assert tiers[2].label == "distribution_below_69kv" + assert tiers[2].min_kv == 0.0 + assert tiers[2].max_kv == 69.0 + + # Verify contiguity: no gap or overlap + # Tier 3 covers [0, 69), Tier 2 covers [69, 230), Tier 1 covers [230, inf) + assert tiers[2].max_kv == tiers[1].min_kv + assert tiers[1].max_kv == tiers[0].min_kv + + +def test_spec_reference_paths() -> None: + """T04: Verify reference paths.""" + spec = build_pass_condition_spec() + + assert spec.bus_exclusion_registry_path == "data/fnm/reference/excluded_buses.json" + assert spec.acpf_reference_dir == "data/fnm/reference/acpf/" + assert spec.dcpf_reference_dir == "data/fnm/reference/dcpf/" + + +# --------------------------------------------------------------------------- +# Serialization round-trip tests (T05-T07) +# --------------------------------------------------------------------------- + + +def test_json_roundtrip(tmp_path: Path) -> None: + """T05: JSON write + load round-trip preserves all values.""" + spec = build_pass_condition_spec() + json_path = tmp_path / "pass_conditions.json" + write_json(spec, json_path) + + loaded = load_spec(json_path) + + # ACPF aggregate + assert loaded.acpf_aggregate.min_passing_fraction == spec.acpf_aggregate.min_passing_fraction + assert loaded.acpf_aggregate.vm_tolerance_pu == spec.acpf_aggregate.vm_tolerance_pu + assert loaded.acpf_aggregate.va_tolerance_deg == spec.acpf_aggregate.va_tolerance_deg + + # DCPF aggregate + assert ( + loaded.dcpf_aggregate.min_bus_passing_fraction + == spec.dcpf_aggregate.min_bus_passing_fraction + ) + assert loaded.dcpf_aggregate.va_tolerance_deg == spec.dcpf_aggregate.va_tolerance_deg + assert ( + loaded.dcpf_aggregate.min_branch_passing_fraction + == spec.dcpf_aggregate.min_branch_passing_fraction + ) + assert loaded.dcpf_aggregate.p_tolerance_pct == spec.dcpf_aggregate.p_tolerance_pct + assert loaded.dcpf_aggregate.p_base_floor_mw == spec.dcpf_aggregate.p_base_floor_mw + + # ACPF hard-fail + assert loaded.acpf_hard_fail.max_failing_fraction == spec.acpf_hard_fail.max_failing_fraction + assert loaded.acpf_hard_fail.vm_max_deviation_pu == spec.acpf_hard_fail.vm_max_deviation_pu + assert loaded.acpf_hard_fail.va_max_deviation_deg == spec.acpf_hard_fail.va_max_deviation_deg + + # DCPF hard-fail + assert ( + loaded.dcpf_hard_fail.max_bus_failing_fraction + == spec.dcpf_hard_fail.max_bus_failing_fraction + ) + assert ( + loaded.dcpf_hard_fail.max_branch_failing_fraction + == spec.dcpf_hard_fail.max_branch_failing_fraction + ) + assert loaded.dcpf_hard_fail.p_max_deviation_pct == spec.dcpf_hard_fail.p_max_deviation_pct + + # Outlier rules + assert len(loaded.outlier_classification.rules) == len(spec.outlier_classification.rules) + + # Voltage tiers + assert len(loaded.voltage_level_tiers) == len(spec.voltage_level_tiers) + + # Version + assert loaded.version == spec.version + + +def test_json_schema_structure(tmp_path: Path) -> None: + """T06: Verify top-level JSON structure and numeric types.""" + spec = build_pass_condition_spec() + json_path = tmp_path / "pass_conditions.json" + write_json(spec, json_path) + + data = json.loads(json_path.read_text(encoding="utf-8")) + + # Top-level keys + assert "$schema_version" in data + assert "$description" in data + assert "bus_exclusion" in data + assert "acpf" in data + assert "dcpf" in data + assert "voltage_level_tiers" in data + + # ACPF sub-keys + assert "reference_dir" in data["acpf"] + assert "reference_files" in data["acpf"] + assert "aggregate" in data["acpf"] + assert "hard_fail" in data["acpf"] + assert "outlier_classification" in data["acpf"] + + # DCPF sub-keys + assert "reference_dir" in data["dcpf"] + assert "reference_files" in data["dcpf"] + assert "aggregate" in data["dcpf"] + assert "hard_fail" in data["dcpf"] + + # Check that numeric thresholds are numbers, not strings + assert isinstance(data["acpf"]["aggregate"]["min_passing_fraction"], (int, float)) + assert isinstance(data["acpf"]["aggregate"]["vm_tolerance_pu"], (int, float)) + assert isinstance(data["acpf"]["aggregate"]["va_tolerance_deg"], (int, float)) + assert isinstance( + data["dcpf"]["aggregate"]["bus_angle"]["min_passing_fraction"], + (int, float), + ) + assert isinstance( + data["dcpf"]["aggregate"]["branch_flow"]["p_tolerance_pct"], + (int, float), + ) + + +def test_markdown_generation(tmp_path: Path) -> None: + """T07: Verify markdown content and minimum size.""" + spec = build_pass_condition_spec() + md_path = tmp_path / "pass_conditions.md" + write_markdown(spec, md_path) + + content = md_path.read_text(encoding="utf-8") + + # Required strings + assert "0.005 p.u." in content or "0.005" in content + assert "0.5 degrees" in content or "0.5" in content + assert "95%" in content + assert "switched_shunt" in content + assert "hard-fail" in content or "Hard-Fail" in content or "Hard-fail" in content + assert "DCPF" in content + assert "ACPF" in content + assert "voltage level" in content.lower() or "Voltage Level" in content + assert "branch flow deviation" in content.lower() or "Branch Flow" in content + + # Non-trivial content + assert len(content.encode("utf-8")) > 1000 + + +# --------------------------------------------------------------------------- +# Helper: synthetic bus/branch generators +# --------------------------------------------------------------------------- + + +def _make_ref_buses( + n: int, + vm_base: float = 1.0, + va_base: float = 0.0, + start_bus: int = 1, +) -> list[dict]: + """Create n synthetic reference buses.""" + buses = [] + for i in range(n): + bus_num = start_bus + i + # Spread VM in [0.95, 1.05] and VA in [-15, 10] + vm = vm_base + 0.05 * (2 * (i % 10) / 9 - 1) + va = va_base + 25.0 * (i % 20) / 19 - 15.0 + buses.append({"bus": bus_num, "VM": vm, "VA": va}) + return buses + + +def _make_tool_buses_acpf( + ref_buses: list[dict], + vm_dev: float = 0.002, + va_dev: float = 0.1, + fail_indices: list[int] | None = None, + fail_vm_dev: float = 0.008, + fail_va_dev: float = 0.1, + hard_fail_index: int | None = None, + hard_fail_vm_dev: float = 0.0, + hard_fail_va_dev: float = 0.0, +) -> list[dict]: + """Create tool buses with controlled deviations.""" + fail_set = set(fail_indices) if fail_indices else set() + tool = [] + for i, rb in enumerate(ref_buses): + if i == hard_fail_index: + tool.append( + { + "bus": rb["bus"], + "VM": rb["VM"] + hard_fail_vm_dev, + "VA": rb["VA"] + hard_fail_va_dev, + } + ) + elif i in fail_set: + tool.append( + { + "bus": rb["bus"], + "VM": rb["VM"] + fail_vm_dev, + "VA": rb["VA"] + fail_va_dev, + } + ) + else: + tool.append( + { + "bus": rb["bus"], + "VM": rb["VM"] + vm_dev, + "VA": rb["VA"] + va_dev, + } + ) + return tool + + +def _make_base_kv_map( + buses: list[dict], + kv_values: list[float] | None = None, +) -> dict[int, float]: + """Create bus_base_kv mapping cycling through kv_values.""" + if kv_values is None: + kv_values = [69.0, 115.0, 138.0, 230.0, 500.0] + return {b["bus"]: kv_values[i % len(kv_values)] for i, b in enumerate(buses)} + + +# --------------------------------------------------------------------------- +# ACPF evaluation tests (T08-T11) +# --------------------------------------------------------------------------- + + +def test_acpf_all_pass() -> None: + """T08: 100% pass with small deviations.""" + spec = build_pass_condition_spec() + ref_buses = _make_ref_buses(100) + tool_buses = _make_tool_buses_acpf(ref_buses, vm_dev=0.003, va_dev=0.3) + bus_base_kv = _make_base_kv_map(ref_buses) + + verdict = evaluate_acpf( + spec=spec, + tool_buses=tool_buses, + ref_buses=ref_buses, + excluded_bus_numbers=set(), + bus_base_kv=bus_base_kv, + classify_outliers=False, + ) + + assert verdict.overall_pass is True + assert verdict.hard_fail is False + assert verdict.aggregate_metrics[0].passed is True + assert verdict.aggregate_metrics[0].value == 1.0 + + +def test_acpf_aggregate_fail() -> None: + """T09: 90% pass (below 95% threshold) — aggregate fail, no hard-fail.""" + spec = build_pass_condition_spec() + ref_buses = _make_ref_buses(100) + + # 10 buses fail with VM deviation of 0.008 (> 0.005) + fail_indices = list(range(10)) + tool_buses = _make_tool_buses_acpf( + ref_buses, + vm_dev=0.002, + va_dev=0.1, + fail_indices=fail_indices, + fail_vm_dev=0.008, + fail_va_dev=0.1, + ) + bus_base_kv = _make_base_kv_map(ref_buses) + + verdict = evaluate_acpf( + spec=spec, + tool_buses=tool_buses, + ref_buses=ref_buses, + excluded_bus_numbers=set(), + bus_base_kv=bus_base_kv, + classify_outliers=False, + ) + + assert verdict.overall_pass is False + assert verdict.hard_fail is False + assert verdict.aggregate_metrics[0].value == 0.90 + + +def test_acpf_hard_fail_extreme_vm() -> None: + """T10: One bus has VM deviation of 0.15 — hard-fail triggered.""" + spec = build_pass_condition_spec() + ref_buses = _make_ref_buses(100) + + # 99 buses pass, 1 bus has extreme VM deviation + tool_buses = _make_tool_buses_acpf( + ref_buses, + vm_dev=0.002, + va_dev=0.1, + hard_fail_index=50, + hard_fail_vm_dev=0.15, + hard_fail_va_dev=0.0, + ) + bus_base_kv = _make_base_kv_map(ref_buses) + + verdict = evaluate_acpf( + spec=spec, + tool_buses=tool_buses, + ref_buses=ref_buses, + excluded_bus_numbers=set(), + bus_base_kv=bus_base_kv, + classify_outliers=False, + ) + + assert verdict.overall_pass is False + assert verdict.hard_fail is True + + # Find the extreme_vm_deviation check + vm_check = next(c for c in verdict.hard_fail_checks if c.check_name == "extreme_vm_deviation") + assert vm_check.triggered is True + + +def test_acpf_hard_fail_excessive_fraction() -> None: + """T11: 25% of buses fail — exceeds 20% hard-fail threshold.""" + spec = build_pass_condition_spec() + ref_buses = _make_ref_buses(100) + + # 25 buses fail (VM deviation 0.008) + fail_indices = list(range(25)) + tool_buses = _make_tool_buses_acpf( + ref_buses, + vm_dev=0.002, + va_dev=0.1, + fail_indices=fail_indices, + fail_vm_dev=0.008, + fail_va_dev=0.1, + ) + bus_base_kv = _make_base_kv_map(ref_buses) + + verdict = evaluate_acpf( + spec=spec, + tool_buses=tool_buses, + ref_buses=ref_buses, + excluded_bus_numbers=set(), + bus_base_kv=bus_base_kv, + classify_outliers=False, + ) + + assert verdict.hard_fail is True + + fraction_check = next( + c for c in verdict.hard_fail_checks if c.check_name == "excessive_failing_fraction" + ) + assert fraction_check.triggered is True + + +# --------------------------------------------------------------------------- +# DCPF evaluation tests (T12-T14) +# --------------------------------------------------------------------------- + + +def _make_ref_buses_dcpf(n: int, start_bus: int = 1) -> list[dict]: + """Create n synthetic DCPF reference buses (VA only).""" + buses = [] + for i in range(n): + bus_num = start_bus + i + va = 40.0 * (i % 20) / 19 - 20.0 # VA in [-20, 20] + buses.append({"bus": bus_num, "VA": va}) + return buses + + +def _make_ref_branches(n: int) -> list[dict]: + """Create n synthetic reference branches.""" + branches = [] + for i in range(n): + from_bus = i + 1 + to_bus = i + 2 + p_flow = 1000.0 * (2 * (i % 50) / 49 - 1) # P in [-1000, 1000] + branches.append( + { + "from_bus": from_bus, + "to_bus": to_bus, + "ckt": "1", + "P_flow_MW": p_flow, + } + ) + return branches + + +def test_dcpf_all_pass() -> None: + """T12: All buses and branches pass.""" + spec = build_pass_condition_spec() + ref_buses = _make_ref_buses_dcpf(100) + ref_branches = _make_ref_branches(200) + + # Tool buses: VA deviation < 0.5 (well within 1.0 tolerance) + tool_buses = [{"bus": b["bus"], "VA": b["VA"] + 0.4} for b in ref_buses] + + # Tool branches: P deviation < 5% (well within 10% tolerance) + tool_branches = [] + for rb in ref_branches: + p_ref = rb["P_flow_MW"] + p_base = max(abs(p_ref), 1.0) + # 5% deviation from p_base + p_tool = p_ref + 0.04 * p_base # 4% deviation + tool_branches.append( + { + "from_bus": rb["from_bus"], + "to_bus": rb["to_bus"], + "ckt": rb["ckt"], + "P_flow_MW": p_tool, + } + ) + + bus_base_kv = {b["bus"]: 230.0 for b in ref_buses} + + verdict = evaluate_dcpf( + spec=spec, + tool_buses=tool_buses, + ref_buses=ref_buses, + tool_branches=tool_branches, + ref_branches=ref_branches, + excluded_bus_numbers=set(), + bus_base_kv=bus_base_kv, + ) + + assert verdict.overall_pass is True + assert verdict.hard_fail is False + + +def test_dcpf_branch_flow_fail() -> None: + """T13: Bus metric passes but branch metric fails (15% exceed P tolerance).""" + spec = build_pass_condition_spec() + ref_buses = _make_ref_buses_dcpf(100) + ref_branches = _make_ref_branches(200) + + # Tool buses: all pass VA tolerance + tool_buses = [{"bus": b["bus"], "VA": b["VA"] + 0.3} for b in ref_buses] + + # Tool branches: 170 pass, 30 fail (15% fail > 10% tolerance means + # branch_passing = 170/200 = 0.85 < 0.90) + tool_branches = [] + for i, rb in enumerate(ref_branches): + p_ref = rb["P_flow_MW"] + p_base = max(abs(p_ref), 1.0) + if i < 30: + # Make these fail: deviation > 10% but < 50% (no hard-fail) + p_tool = p_ref + 0.15 * p_base # 15% deviation + else: + # These pass: deviation < 10% + p_tool = p_ref + 0.03 * p_base # 3% deviation + tool_branches.append( + { + "from_bus": rb["from_bus"], + "to_bus": rb["to_bus"], + "ckt": rb["ckt"], + "P_flow_MW": p_tool, + } + ) + + bus_base_kv = {b["bus"]: 230.0 for b in ref_buses} + + verdict = evaluate_dcpf( + spec=spec, + tool_buses=tool_buses, + ref_buses=ref_buses, + tool_branches=tool_branches, + ref_branches=ref_branches, + excluded_bus_numbers=set(), + bus_base_kv=bus_base_kv, + ) + + assert verdict.overall_pass is False + + # Bus metric passes + bus_metric = next( + m for m in verdict.aggregate_metrics if m.metric_name == "dcpf_bus_va_aggregate" + ) + assert bus_metric.passed is True + + # Branch metric fails + branch_metric = next( + m for m in verdict.aggregate_metrics if m.metric_name == "dcpf_branch_p_aggregate" + ) + assert branch_metric.passed is False + + +def test_dcpf_p_base_floor() -> None: + """T14: Verify p_base_floor prevents inflated deviation on low-flow branches.""" + spec = build_pass_condition_spec() + + # Single reference bus (to avoid zero-denominator) + ref_buses = [{"bus": 1, "VA": 0.0}] + tool_buses = [{"bus": 1, "VA": 0.0}] + + # One reference branch with very small flow + ref_branches = [ + {"from_bus": 1, "to_bus": 2, "ckt": "1", "P_flow_MW": 0.1}, + ] + # Tool branch with slightly larger flow + tool_branches = [ + {"from_bus": 1, "to_bus": 2, "ckt": "1", "P_flow_MW": 0.5}, + ] + + bus_base_kv = {1: 230.0} + + verdict = evaluate_dcpf( + spec=spec, + tool_buses=tool_buses, + ref_buses=ref_buses, + tool_branches=tool_branches, + ref_branches=ref_branches, + excluded_bus_numbers=set(), + bus_base_kv=bus_base_kv, + ) + + # Without floor: deviation = |0.5-0.1|/|0.1| * 100 = 400% + # With floor (1.0 MW): deviation = |0.5-0.1|/1.0 * 100 = 40% + # The branch still fails (40 > 10) but does NOT trigger the 50% hard-fail + + # Check that hard-fail for extreme branch deviation is NOT triggered + extreme_check = next( + c for c in verdict.hard_fail_checks if c.check_name == "extreme_branch_flow_deviation" + ) + assert extreme_check.triggered is False + # The value should be 40.0, not 400.0 + assert abs(extreme_check.value - 40.0) < 0.01 diff --git a/data/fnm/tests/test_per_unit_conventions_doc.py b/data/fnm/tests/test_per_unit_conventions_doc.py new file mode 100644 index 00000000..56aabba4 --- /dev/null +++ b/data/fnm/tests/test_per_unit_conventions_doc.py @@ -0,0 +1,371 @@ +"""Structural validation tests for the per-unit conventions reference document. + +Tests verify that data/fnm/docs/per-unit-conventions.md contains all required +sections, worked examples, pitfalls coverage, and cross-references as specified +in PRD 02/03. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +DOC_PATH = REPO_ROOT / "data" / "fnm" / "docs" / "per-unit-conventions.md" + +DOMAIN_HEADINGS = [ + "1. System MVA Base", + "2. Bus Base Voltage", + "3. Branch (AC Line) Impedance", + "4. Two-Winding Transformer Impedance", + "5. Two-Winding Transformer Tap Ratios", + "6. Three-Winding Transformer Per-Unit Bases", + "7. Shunt Admittance", + "8. Generator Capability", + "9. Load Representation", +] + +TOOL_NAMES = ["MATPOWER", "pandapower", "PyPSA", "GridCal", "PowerModels", "PowerSimulations"] + + +@pytest.fixture(scope="module") +def doc_text() -> str: + """Read the per-unit conventions document and return its full text.""" + assert DOC_PATH.exists(), f"Document not found at {DOC_PATH}" + return DOC_PATH.read_text(encoding="utf-8") + + +@pytest.fixture(scope="module") +def doc_sections(doc_text: str) -> dict[str, str]: + """Split the document into top-level (##) sections keyed by heading.""" + sections: dict[str, str] = {} + current_heading = "" + current_lines: list[str] = [] + for line in doc_text.splitlines(): + if line.startswith("## ") and not line.startswith("### "): + if current_heading: + sections[current_heading] = "\n".join(current_lines) + current_heading = line[3:].strip() + current_lines = [] + else: + current_lines.append(line) + if current_heading: + sections[current_heading] = "\n".join(current_lines) + return sections + + +def _find_section(sections: dict[str, str], prefix: str) -> str: + """Find a section whose heading starts with the given prefix. + + Returns the section body text, or empty string if not found. + """ + for heading, body in sections.items(): + if heading.startswith(prefix): + return body + return "" + + +def _has_section(sections: dict[str, str], prefix: str) -> bool: + """Check whether a section whose heading starts with *prefix* exists.""" + return any(h.startswith(prefix) for h in sections) + + +# --------------------------------------------------------------------------- +# Document structure tests (T01-T04) +# --------------------------------------------------------------------------- + + +@pytest.mark.docs +class TestDocumentStructure: + """T01-T04: Verify the document has all required sections.""" + + def test_document_exists_at_expected_path(self) -> None: + """T01: Verify the document exists at the expected path.""" + assert DOC_PATH.exists(), f"Expected per-unit conventions document at {DOC_PATH}" + + def test_all_nine_domain_sections_present(self, doc_sections: dict[str, str]) -> None: + """T02: All nine numbered domain sections are present.""" + for heading in DOMAIN_HEADINGS: + assert _has_section(doc_sections, heading), f"Missing domain section: '## {heading}'" + + def test_required_subsections_per_domain(self, doc_sections: dict[str, str]) -> None: + """T03: Each domain has required subsections.""" + for heading in DOMAIN_HEADINGS: + section_text = _find_section(doc_sections, heading) + subsection_headings = re.findall(r"^### (.+)$", section_text, re.MULTILINE) + + # Every domain needs Worked Example and Common Pitfalls + assert any("Worked Example" in h for h in subsection_headings), ( + f"Domain '{heading}' missing '### Worked Example' subsection" + ) + assert any("Common Pitfalls" in h for h in subsection_headings), ( + f"Domain '{heading}' missing '### Common Pitfalls' subsection" + ) + + # Every domain needs at least one substantive subsection beyond + # Worked Example and Common Pitfalls (e.g., Definition, Base + # Impedance Formula, CW/CZ modes, Fixed/Switched Shunts, etc.) + non_example_subsections = [ + h for h in subsection_headings if h not in ("Worked Example", "Common Pitfalls") + ] + has_definition = len(non_example_subsections) > 0 + assert has_definition, ( + f"Domain '{heading}' missing a definition-type subsection " + "(e.g., '### Definition' or '### Base Impedance Formula')" + ) + + # Domains 4 and 5 need CZ/CW mode subsections + if "4." in heading: + for cz in ["CZ=1", "CZ=2", "CZ=3"]: + assert any(cz in h for h in subsection_headings), ( + f"Domain '{heading}' missing subsection for {cz}" + ) + if "5." in heading: + for cw in ["CW=1", "CW=2", "CW=3"]: + assert any(cw in h for h in subsection_headings), ( + f"Domain '{heading}' missing subsection for {cw}" + ) + + def test_summary_table_present(self, doc_text: str) -> None: + """T04: Summary Table section exists with 9+ data rows.""" + assert "## Summary Table" in doc_text, "Missing '## Summary Table' section" + + # Extract the summary table section + summary_start = doc_text.index("## Summary Table") + # Find the next ## section or end of file + next_section = doc_text.find("\n## ", summary_start + 1) + if next_section == -1: + summary_section = doc_text[summary_start:] + else: + summary_section = doc_text[summary_start:next_section] + + # Count table rows (lines starting with |, excluding header separator) + table_lines = [ + line.strip() + for line in summary_section.splitlines() + if line.strip().startswith("|") and not re.match(r"^\|[-\s|]+\|$", line.strip()) + ] + # First line is the header, rest are data rows + data_rows = [line for line in table_lines[1:] if line.startswith("|")] + assert len(data_rows) >= 9, ( + f"Summary table has {len(data_rows)} data rows, expected at least 9" + ) + + # Verify expected columns + header = table_lines[0] if table_lines else "" + for col in ["Domain", "PSS/E Fields", "Unit", "Per-Unit Base", "Conversion"]: + assert col.lower() in header.lower(), f"Summary table missing expected column: '{col}'" + + +# --------------------------------------------------------------------------- +# Worked example tests (T05-T07) +# --------------------------------------------------------------------------- + + +@pytest.mark.docs +class TestWorkedExamples: + """T05-T07: Verify worked examples use realistic values and both directions.""" + + def test_worked_examples_use_realistic_values(self, doc_sections: dict[str, str]) -> None: + """T05: Worked examples use transmission-scale values.""" + for heading in DOMAIN_HEADINGS: + section_text = _find_section(doc_sections, heading) + # Extract the Worked Example subsection + example_match = re.search( + r"### Worked Example\s*\n(.*?)(?=\n### |\Z)", + section_text, + re.DOTALL, + ) + assert example_match, f"Could not extract Worked Example from '{heading}'" + example_text = example_match.group(1) + + # Extract all numeric values from the example + numbers = [float(m) for m in re.findall(r"(? None: + """T06: Each worked example shows both pu-to-physical and physical-to-pu.""" + pu_to_phys_patterns = [ + r"[Pp]er.unit to physical", + r"pu\s*(?:to|->|-->|→)\s*(?:kV|ohm|MW|MVAR|siemens|A)", + r"[Pp]er-unit to physical", + r"_pu\s*\*", + r"_pu \*", + r"_ohm\s*=", # Calculating physical ohms from per-unit + r"_kV\s*=", # Calculating physical kV from per-unit + r"_MW\s*=", # Calculating physical MW from per-unit + ] + phys_to_pu_patterns = [ + r"[Pp]hysical to per.unit", + r"(?:kV|ohm|MW|MVAR|siemens|A)\s*(?:to|->|-->|→)\s*pu", + r"[Pp]hysical to per-unit", + r"/ (?:S_base|Z_base|BASKV|SBASE|V_base)", + r"_pu\s*=\s*\d", + r"_pu,\w+\s*=\s*\d", # e.g. R_pu,system = 0.001750 + r"_system\s*=\s*\d", # e.g. X12_system = 0.01417 + ] + + for heading in DOMAIN_HEADINGS: + section_text = _find_section(doc_sections, heading) + example_match = re.search( + r"### Worked Example\s*\n(.*?)(?=\n### |\Z)", + section_text, + re.DOTALL, + ) + assert example_match, f"No Worked Example in '{heading}'" + example_text = example_match.group(1) + + has_pu_to_phys = any(re.search(p, example_text) for p in pu_to_phys_patterns) + has_phys_to_pu = any(re.search(p, example_text) for p in phys_to_pu_patterns) + + assert has_pu_to_phys, ( + f"Domain '{heading}' worked example missing per-unit to physical conversion" + ) + assert has_phys_to_pu, ( + f"Domain '{heading}' worked example missing physical to per-unit conversion" + ) + + def test_cz_mode_examples_cover_all_three_modes(self, doc_sections: dict[str, str]) -> None: + """T07: 2W transformer impedance section covers CZ=1, CZ=2, CZ=3.""" + heading = "4. Two-Winding Transformer Impedance" + section_text = _find_section(doc_sections, heading) + assert section_text, f"Missing section '{heading}'" + + for cz_mode in ["CZ=1", "CZ=2", "CZ=3"]: + # Find the CZ mode subsection or worked example referencing it + assert cz_mode in section_text, f"Section '{heading}' does not mention {cz_mode}" + # Verify there is at least one numeric calculation for each mode + # Find text around each CZ mention and check for numbers + cz_positions = [m.start() for m in re.finditer(cz_mode, section_text)] + has_calculation = False + for pos in cz_positions: + surrounding = section_text[max(0, pos - 200) : pos + 500] + numbers = re.findall(r"\d+\.\d+", surrounding) + if len(numbers) >= 2: # At least 2 numeric values = a calculation + has_calculation = True + break + assert has_calculation, f"No numeric calculation found for {cz_mode} in '{heading}'" + + +# --------------------------------------------------------------------------- +# Pitfalls and tool coverage tests (T08-T10) +# --------------------------------------------------------------------------- + + +@pytest.mark.docs +class TestPitfallsAndToolCoverage: + """T08-T10: Verify pitfalls sections and tool coverage.""" + + def test_pitfalls_reference_at_least_three_tools(self, doc_sections: dict[str, str]) -> None: + """T08: Each Common Pitfalls subsection mentions at least 3 tools.""" + for heading in DOMAIN_HEADINGS: + section_text = _find_section(doc_sections, heading) + pitfalls_match = re.search( + r"### Common Pitfalls\s*\n(.*?)(?=\n### |\Z)", + section_text, + re.DOTALL, + ) + assert pitfalls_match, f"No Common Pitfalls subsection in '{heading}'" + pitfalls_text = pitfalls_match.group(1) + pitfalls_lower = pitfalls_text.lower() + + tool_count = sum(1 for tool in TOOL_NAMES if tool.lower() in pitfalls_lower) + assert tool_count >= 3, ( + f"Domain '{heading}' pitfalls mention only {tool_count} tools, " + f"expected at least 3. Tools found: " + f"{[t for t in TOOL_NAMES if t.lower() in pitfalls_lower]}" + ) + + def test_pitfalls_include_diagnostic_signature(self, doc_sections: dict[str, str]) -> None: + """T09: At least 6/9 pitfalls sections contain diagnostic signature language.""" + diagnostic_patterns = [ + r"off by a factor", + r"scaled by", + r"differs by", + r"consistent ratio", + r"symptom", + r"diagnostic signature", + r"factor of", + r"consistently scaled", + ] + + domains_with_diagnostic = 0 + for heading in DOMAIN_HEADINGS: + section_text = _find_section(doc_sections, heading) + pitfalls_match = re.search( + r"### Common Pitfalls\s*\n(.*?)(?=\n### |\Z)", + section_text, + re.DOTALL, + ) + if not pitfalls_match: + continue + pitfalls_text = pitfalls_match.group(1).lower() + if any(re.search(p, pitfalls_text) for p in diagnostic_patterns): + domains_with_diagnostic += 1 + + assert domains_with_diagnostic >= 6, ( + f"Only {domains_with_diagnostic}/9 pitfalls sections contain " + "diagnostic signature language, expected at least 6" + ) + + def test_all_six_tools_mentioned_in_document(self, doc_text: str) -> None: + """T10: All six tool names appear at least once in the document.""" + for tool in TOOL_NAMES: + assert tool in doc_text, f"Tool '{tool}' not mentioned anywhere in the document" + + +# --------------------------------------------------------------------------- +# Cross-reference and notation tests (T11-T12) +# --------------------------------------------------------------------------- + + +@pytest.mark.docs +class TestCrossReferencesAndNotation: + """T11-T12: Verify notation and cross-reference sections.""" + + def test_notation_conventions_section_present(self, doc_text: str) -> None: + """T11: Notation Conventions section defines S_base, V_base, Z_base, Y_base.""" + assert "## Notation Conventions" in doc_text, "Missing '## Notation Conventions' section" + # Extract the section + start = doc_text.index("## Notation Conventions") + next_section = doc_text.find("\n## ", start + 1) + if next_section == -1: + notation_text = doc_text[start:] + else: + notation_text = doc_text[start:next_section] + + for symbol in ["S_base", "V_base", "Z_base", "Y_base"]: + assert symbol in notation_text, ( + f"Notation Conventions section missing definition of '{symbol}'" + ) + + def test_cross_references_section_present(self, doc_text: str) -> None: + """T12: Cross-References section references D7, PRD 01, PRD 04.""" + assert "## Cross-References" in doc_text, "Missing '## Cross-References' section" + start = doc_text.index("## Cross-References") + xref_text = doc_text[start:] + + # Check for D7 reference + assert "D7" in xref_text, "Cross-References missing reference to Phase 1 D7" + # Check for PRD 01 reference + assert re.search(r"PRD[- ]01", xref_text), "Cross-References missing reference to PRD 01" + # Check for PRD 04 reference + assert re.search(r"PRD[- ]04", xref_text), "Cross-References missing reference to PRD 04" diff --git a/data/fnm/tests/test_raw_record_counter.py b/data/fnm/tests/test_raw_record_counter.py new file mode 100644 index 00000000..521d7b68 --- /dev/null +++ b/data/fnm/tests/test_raw_record_counter.py @@ -0,0 +1,284 @@ +"""Tests for the PSS/E v31 RAW file record counter (PRD 01/03). + +T01-T09: Synthetic tests (no FNM data required). +T10-T12: FNM integration tests (require FNM_PATH, skip if unset). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from fnm.scripts.raw_record_counter import ( + PSSE_V31_SECTION_NAMES, + count_raw_records, + count_section_records, + parse_header, + summary_to_dict, +) + +# --------------------------------------------------------------------------- +# Helpers for building synthetic RAW file content +# --------------------------------------------------------------------------- + +_VALID_HEADER_LINES = [ + " 0, 100.00, 31.0, 0, 0, 60.00 / PSS/E-31.0 test case", + "Test case identification line 1", + "Test case identification line 2", +] + + +def _make_raw_content( + header: list[str] | None = None, + sections: list[list[str]] | None = None, +) -> str: + """Build a minimal PSS/E v31 RAW file as a string. + + Args: + header: 3-line header. Defaults to _VALID_HEADER_LINES. + sections: A list of 17 section bodies. Each body is a list of data lines + (the ``0`` sentinel is appended automatically). If a section body is + empty, only the sentinel line is written. Defaults to 17 empty sections. + """ + hdr = header or _VALID_HEADER_LINES + secs = sections or [[] for _ in range(17)] + lines = list(hdr) + for body in secs: + lines.extend(body) + lines.append(" 0") + return "\n".join(lines) + "\n" + + +def _write_raw(tmp_path: Path, content: str) -> Path: + p = tmp_path / "test.raw" + p.write_text(content, encoding="utf-8") + return p + + +# --------------------------------------------------------------------------- +# T01: test_parse_header_valid +# --------------------------------------------------------------------------- + + +def test_parse_header_valid() -> None: + """Parse a synthetic 3-line v31 header and verify all fields.""" + hdr = parse_header(_VALID_HEADER_LINES) + assert hdr.ic == 0 + assert hdr.sbase == 100.0 + assert hdr.rev == 31.0 + assert hdr.xfrrat == 0.0 + assert hdr.nxfrat == 0.0 + assert hdr.basfrq == 60.0 + assert hdr.case_id == "Test case identification line 1" + assert hdr.case_id2 == "Test case identification line 2" + + +# --------------------------------------------------------------------------- +# T02: test_parse_header_rejects_non_v31 +# --------------------------------------------------------------------------- + + +def test_parse_header_rejects_non_v31() -> None: + """REV=30.0 should raise ValueError.""" + lines = [ + " 0, 100.00, 30.0, 0, 0, 60.00", + "Case id 1", + "Case id 2", + ] + with pytest.raises(ValueError, match="v31"): + parse_header(lines) + + +# --------------------------------------------------------------------------- +# T03: test_parse_header_rejects_malformed +# --------------------------------------------------------------------------- + + +def test_parse_header_rejects_malformed() -> None: + """Non-numeric line 1 should raise ValueError.""" + lines = [ + "this is not a valid header line", + "Case id 1", + "Case id 2", + ] + with pytest.raises(ValueError, match="[Mm]alformed"): + parse_header(lines) + + +# --------------------------------------------------------------------------- +# T04: test_count_section_simple +# --------------------------------------------------------------------------- + + +def test_count_section_simple() -> None: + """5 data lines + terminator -> count=5.""" + lines = [ + "1, 'BUS1', 138.0, 1", + "2, 'BUS2', 138.0, 1", + "3, 'BUS3', 138.0, 1", + "4, 'BUS4', 69.0, 1", + "5, 'BUS5', 69.0, 1", + " 0", + ] + count = count_section_records(iter(lines), section_index=0) + assert count == 5 + + +# --------------------------------------------------------------------------- +# T05: test_count_section_empty +# --------------------------------------------------------------------------- + + +def test_count_section_empty() -> None: + """Just a terminator -> count=0.""" + lines = [" 0"] + count = count_section_records(iter(lines), section_index=0) + assert count == 0 + + +# --------------------------------------------------------------------------- +# T06: test_count_section_transformer_2w_and_3w +# --------------------------------------------------------------------------- + + +def test_count_section_transformer_2w_and_3w() -> None: + """One 2W (4 lines, K=0) + one 3W (5 lines, K!=0) -> count=2.""" + lines = [ + # 2-winding transformer (K=0): 4 lines + " 1, 2, 0, '1', 1, 1, 1, 0.0, 0.0, 2, 'xfmr2w'", # line 1, K=0 + " 0.01, 0.10, 100.0", # line 2 + " 1.0, 0.0, 0.0, 138.0, 0.0, 0.0, 0, 0, 1.1, 0.9", # line 3 + " 1.0, 0.0, 0.0, 69.0, 0.0, 0.0", # line 4 + # 3-winding transformer (K!=0): 5 lines + " 1, 2, 3, '1', 1, 1, 1, 0.0, 0.0, 2, 'xfmr3w'", # line 1, K=3 + " 0.01, 0.10, 100.0", # line 2 + " 1.0, 0.0, 0.0, 138.0, 0.0, 0.0, 0, 0, 1.1, 0.9", # line 3 + " 1.0, 0.0, 0.0, 69.0, 0.0, 0.0", # line 4 + " 1.0, 0.0, 0.0, 34.5, 0.0, 0.0", # line 5 + # sentinel + " 0", + ] + count = count_section_records(iter(lines), section_index=5) + assert count == 2 + + +# --------------------------------------------------------------------------- +# T07: test_count_section_multi_terminal_dc +# --------------------------------------------------------------------------- + + +def test_count_section_multi_terminal_dc() -> None: + """One MTDC record with NCONV=2, NDCBS=3, NDCLN=1 -> count=1.""" + lines = [ + # Main record line: NCONV=2, NDCBS=3, NDCLN=1 + " 2, 3, 1, 'MTDC1', 0", + # 2 converter lines + " 1, 100, 1.0, 0.0, 0.0", + " 2, 200, 1.0, 0.0, 0.0", + # 3 DC bus lines + " 1, 1, 100.0, 1, 0", + " 2, 2, 100.0, 1, 0", + " 3, 3, 100.0, 1, 0", + # 1 DC link line + " 1, 2, '1', 0.01, 100.0", + # sentinel + " 0", + ] + count = count_section_records(iter(lines), section_index=10) + assert count == 1 + + +# --------------------------------------------------------------------------- +# T08: test_count_full_synthetic_file +# --------------------------------------------------------------------------- + + +def test_count_full_synthetic_file(tmp_path: Path) -> None: + """Full synthetic file: 3 buses, 2 loads, empty rest -> Bus=3, Load=2, total=5.""" + bus_data = [ + " 1, 'BUS1 ', 138.0, 1, 1, 1, 1, 1.05, 0.0", + " 2, 'BUS2 ', 138.0, 1, 1, 1, 1, 1.03, -2.1", + " 3, 'BUS3 ', 69.0, 1, 1, 1, 1, 1.01, -5.0", + ] + load_data = [ + " 1, '1', 1, 1, 1, 50.0, 25.0, 0.0, 0.0, 0.0, 0.0, 1", + " 2, '1', 1, 1, 1, 100.0, 50.0, 0.0, 0.0, 0.0, 0.0, 1", + ] + sections: list[list[str]] = [[] for _ in range(17)] + sections[0] = bus_data + sections[1] = load_data + + content = _make_raw_content(sections=sections) + raw_path = _write_raw(tmp_path, content) + + summary = count_raw_records(raw_path) + assert summary.section_counts["Bus"] == 3 + assert summary.section_counts["Load"] == 2 + assert summary.total_data_lines == 5 + assert summary.header.rev == 31.0 + assert len(summary.section_counts) == 17 + # All other sections should be 0 + for name in PSSE_V31_SECTION_NAMES[2:]: + assert summary.section_counts[name] == 0 + + +# --------------------------------------------------------------------------- +# T09: test_summary_to_dict_roundtrip +# --------------------------------------------------------------------------- + + +def test_summary_to_dict_roundtrip(tmp_path: Path) -> None: + """Build summary, convert to dict, verify JSON-serializable.""" + content = _make_raw_content() + raw_path = _write_raw(tmp_path, content) + summary = count_raw_records(raw_path) + + d = summary_to_dict(summary) + + # Must be JSON-serializable + json_str = json.dumps(d) + loaded = json.loads(json_str) + + assert loaded["header"]["rev"] == 31.0 + assert loaded["total_sections"] == 17 + assert isinstance(loaded["section_counts"], dict) + assert len(loaded["section_counts"]) == 17 + assert isinstance(loaded["hvdc_facts_present"], dict) + assert set(loaded["hvdc_facts_present"].keys()) == { + "Two-Terminal DC", + "VSC DC", + "Multi-Terminal DC", + "FACTS", + } + + +# --------------------------------------------------------------------------- +# FNM integration tests (T10-T12) — require FNM_PATH +# --------------------------------------------------------------------------- + + +@pytest.mark.fnm +def test_fnm_header_is_v31(require_fnm_raw: Path) -> None: + """T10: FNM header is v31 with sbase=100.0.""" + summary = count_raw_records(require_fnm_raw) + assert summary.header.rev == 31.0 + assert summary.header.sbase == 100.0 + + +@pytest.mark.fnm +def test_fnm_bus_count_scale(require_fnm_raw: Path) -> None: + """T11: Bus count in 25000-35000 range (production-scale network).""" + summary = count_raw_records(require_fnm_raw) + assert 25000 <= summary.section_counts["Bus"] <= 35000, ( + f"Bus count {summary.section_counts['Bus']} outside expected production-scale range" + ) + + +@pytest.mark.fnm +def test_fnm_all_17_sections_present(require_fnm_raw: Path) -> None: + """T12: All 17 sections present in section_counts, total_sections==17.""" + summary = count_raw_records(require_fnm_raw) + assert summary.total_sections == 17 + assert set(summary.section_counts.keys()) == set(PSSE_V31_SECTION_NAMES) diff --git a/data/fnm/tests/test_rubric_v4_justification.py b/data/fnm/tests/test_rubric_v4_justification.py new file mode 100644 index 00000000..d69f3ff1 --- /dev/null +++ b/data/fnm/tests/test_rubric_v4_justification.py @@ -0,0 +1,360 @@ +"""Structural validation tests for the rubric v4 amendment justification document. + +Tests verify that data/fnm/docs/rubric-v4-justification.md contains all required +sections, covers all required arguments, respects grading boundary constraints, +and documents the FNM_PATH gating contract as specified in PRD 04/05. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +# --------------------------------------------------------------------------- +# Paths and constants +# --------------------------------------------------------------------------- + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +DOC_PATH = REPO_ROOT / "data" / "fnm" / "docs" / "rubric-v4-justification.md" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _load_document() -> str: + """Load the justification document as a string.""" + return DOC_PATH.read_text(encoding="utf-8") + + +def _split_sections(text: str) -> list[tuple[str, str]]: + """Split a markdown document into (header, body) tuples by H2 headers. + + The first tuple has an empty header string and contains content before the + first H2. Subsequent tuples use the H2 text (without ``## ``) as header. + """ + parts = re.split(r"^## (.+)$", text, flags=re.MULTILINE) + sections: list[tuple[str, str]] = [] + # parts[0] is content before first H2 + sections.append(("", parts[0])) + # remaining parts alternate: header, body, header, body, ... + for i in range(1, len(parts), 2): + header = parts[i].strip() + body = parts[i + 1] if i + 1 < len(parts) else "" + sections.append((header, body)) + return sections + + +def _find_section(sections: list[tuple[str, str]], *keywords: str) -> tuple[str, str] | None: + """Find the first section whose header contains any of the keywords (case-insensitive).""" + for header, body in sections: + for kw in keywords: + if kw.lower() in header.lower(): + return header, body + return None + + +def _get_executive_summary(text: str, sections: list[tuple[str, str]]) -> str: + """Extract the executive summary content. + + The executive summary is either the content before the first H2 (if it + contains substantive text) or the first H2 section if the pre-H2 content + is just a title / front-matter. + """ + # Content before first H2 (excluding leading H1 title line) + pre_h2 = sections[0][1] + # Strip the H1 title line(s) and any blank lines + lines = pre_h2.strip().splitlines() + content_lines = [ln for ln in lines if not ln.startswith("# ")] + content = "\n".join(content_lines).strip() + if len(content) > 100: + return content + # Fall back to first H2 section + if len(sections) > 1: + return sections[1][1] + return content + + +def _count_sentences(text: str) -> int: + """Count sentences in text by splitting on sentence-ending punctuation.""" + # Remove markdown table rows and blank lines + lines = [ln for ln in text.strip().splitlines() if not ln.strip().startswith("|")] + prose = " ".join(ln.strip() for ln in lines if ln.strip()) + # Split on period/question-mark/exclamation followed by space or end of string + sentences = re.split(r"(?<=[.?!])\s+", prose) + return len([s for s in sentences if len(s.strip()) > 10]) + + +# --------------------------------------------------------------------------- +# Structural validation tests +# --------------------------------------------------------------------------- + + +@pytest.mark.docs +class TestStructuralValidation: + """T01-T04: Structural completeness tests.""" + + def test_document_exists(self) -> None: + """T01: Verify the justification document exists and is non-empty.""" + assert DOC_PATH.exists(), f"Document not found at {DOC_PATH}" + content = _load_document() + assert len(content.strip()) > 0, "Document is empty" + + def test_all_required_sections_present(self) -> None: + """T02: Verify all 8 required sections exist by matching H2 headers. + + Required sections (by key phrases in H2 headers): + - S1: Executive summary (content before first H2 or first H2) + - S2: "Phase 1" or "Data Model Fidelity" + - S3: "Evidence" or "Synthetic" + - S4: "Precedent" + - S5: "Grading Impact" + - S6: "Gating" or "FNM_PATH" + - S7: "Fail" or "Asymmetry" + - S8: "Cross-Reference" + """ + text = _load_document() + sections = _split_sections(text) + + # S1: executive summary -- either pre-H2 content or first H2 + exec_summary = _get_executive_summary(text, sections) + assert len(exec_summary.strip()) > 100, ( + "S1 (Executive Summary): no substantive content found before or at first H2" + ) + + # S2-S8: check H2 headers by key phrases + section_checks = { + "S2 (Data Model Fidelity)": ("Phase 1", "Data Model Fidelity"), + "S3 (FNM vs Synthetic Evidence)": ("Evidence", "Synthetic"), + "S4 (Precedent)": ("Precedent",), + "S5 (Grading Impact)": ("Grading Impact",), + "S6 (FNM_PATH Gating)": ("Gating", "FNM_PATH"), + "S7 (Failure Asymmetry)": ("Fail", "Asymmetry"), + "S8 (Cross-References)": ("Cross-Reference",), + } + + missing = [] + for label, keywords in section_checks.items(): + result = _find_section(sections, *keywords) + if result is None: + missing.append(label) + + assert not missing, f"Missing required sections: {', '.join(missing)}" + + def test_executive_summary_is_concise(self) -> None: + """T03: Verify executive summary is between 3 and 10 sentences.""" + text = _load_document() + sections = _split_sections(text) + exec_summary = _get_executive_summary(text, sections) + sentence_count = _count_sentences(exec_summary) + assert 3 <= sentence_count <= 10, ( + f"Executive summary has {sentence_count} sentences, expected 3-10" + ) + + def test_document_word_count_in_range(self) -> None: + """T04: Verify total document word count is between 1,500 and 5,000.""" + text = _load_document() + word_count = len(text.split()) + assert 1500 <= word_count <= 5000, f"Document has {word_count} words, expected 1,500-5,000" + + +# --------------------------------------------------------------------------- +# Argument coverage tests +# --------------------------------------------------------------------------- + + +@pytest.mark.docs +class TestArgumentCoverage: + """T05-T07: Argument coverage tests.""" + + def test_record_type_gap_table_present(self) -> None: + """T05: Verify S3 contains a markdown table with key record types. + + The table must have at least 5 data rows and mention both + '3-winding' and 'switched shunt' (case-insensitive). + """ + text = _load_document() + sections = _split_sections(text) + result = _find_section(sections, "Evidence", "Synthetic") + assert result is not None, "S3 section not found" + _, body = result + + # Find markdown table rows (lines starting with |, excluding header separator) + table_rows = [ + ln + for ln in body.strip().splitlines() + if ln.strip().startswith("|") and not re.match(r"^\|[\s\-:|]+\|$", ln.strip()) + ] + # Exclude header rows (first row of each table) + # Heuristic: data rows contain at least one non-title-case cell + data_rows = [ + r for r in table_rows if not all(c.strip().istitle() for c in r.split("|")[1:-1]) + ] + assert len(data_rows) >= 5, ( + f"S3 record type gap table has {len(data_rows)} data rows, expected >= 5" + ) + + body_lower = body.lower() + assert "3-winding" in body_lower, ( + "S3 table must mention '3-winding' or '3-Winding' transformers" + ) + assert "switched shunt" in body_lower, ( + "S3 table must mention 'switched shunt' or 'Switched Shunt'" + ) + + def test_field_coverage_gap_referenced(self) -> None: + """T06: Verify S3 references field-level coverage analysis. + + Must contain either: + (a) a markdown table with 'DCPF-critical' and a numeric value, or + (b) prose with both 'DCPF-critical' and 'field' in the same paragraph. + """ + text = _load_document() + sections = _split_sections(text) + result = _find_section(sections, "Evidence", "Synthetic") + assert result is not None, "S3 section not found" + _, body = result + + # Check for table with DCPF-critical and a number + has_table = False + for line in body.splitlines(): + if "DCPF-critical" in line and re.search(r"\d+", line): + has_table = True + break + + # Check for prose with both terms in same paragraph + has_prose = False + paragraphs = re.split(r"\n\s*\n", body) + for para in paragraphs: + if "DCPF-critical" in para and "field" in para.lower(): + has_prose = True + break + + assert has_table or has_prose, ( + "S3 must reference field-level coverage with 'DCPF-critical' " + "in either a table with numeric values or prose with 'field'" + ) + + def test_v2_precedent_cited(self) -> None: + """T07: Verify S4 cites the rubric v2 precedent specifically. + + Must contain all of: + - 'sub-question' or 'sub-questions' + - 'SCOPF' + - 'inform' or 'readiness indicator' + """ + text = _load_document() + sections = _split_sections(text) + result = _find_section(sections, "Precedent") + assert result is not None, "S4 (Precedent) section not found" + _, body = result + + body_lower = body.lower() + + assert "sub-question" in body_lower or "sub-questions" in body_lower, ( + "S4 must mention 'sub-question' or 'sub-questions' (the v2 additions)" + ) + assert "scopf" in body_lower, ( + "S4 must mention 'SCOPF' (the most significant v2 sub-question)" + ) + assert "inform" in body_lower or "readiness indicator" in body_lower, ( + "S4 must contain 'inform' or 'readiness indicator' (grading mechanism language)" + ) + + +# --------------------------------------------------------------------------- +# Boundary and constraint tests +# --------------------------------------------------------------------------- + + +@pytest.mark.docs +class TestBoundaryConstraints: + """T08-T10: Boundary and constraint validation tests.""" + + def test_no_threshold_changes_claimed(self) -> None: + """T08: Verify S5 confirms grade boundaries are unchanged. + + At least one sentence must contain both a negation word and a + grade-boundary term. + """ + text = _load_document() + sections = _split_sections(text) + result = _find_section(sections, "Grading Impact") + assert result is not None, "S5 (Grading Impact) section not found" + _, body = result + + negation_words = {"no", "not", "unchanged", "unaffected", "invariant"} + boundary_terms = {"threshold", "boundary", "a/b/c", "grade boundary", "grade boundaries"} + + # Check sentence by sentence + sentences = re.split(r"(?<=[.?!])\s+", body) + found = False + for sentence in sentences: + sentence_lower = sentence.lower() + has_negation = any( + re.search(rf"\b{re.escape(w)}\b", sentence_lower) for w in negation_words + ) + has_boundary = any(term in sentence_lower for term in boundary_terms) + if has_negation and has_boundary: + found = True + break + + assert found, ( + "S5 must contain at least one sentence with both a negation word " + "(no/not/unchanged/unaffected/invariant) and a grade-boundary term " + "(threshold/boundary/A/B/C/grade boundary)" + ) + + def test_fnm_path_gating_documented(self) -> None: + """T09: Verify S6 documents the FNM_PATH gating contract. + + Must contain all of: + - 'FNM_PATH' + - 'skip' or 'skipped' + - 'additive' or 'complete grades' or 'without FNM' + """ + text = _load_document() + sections = _split_sections(text) + result = _find_section(sections, "Gating", "FNM_PATH") + assert result is not None, "S6 (FNM_PATH Gating) section not found" + _, body = result + + body_lower = body.lower() + + assert "fnm_path" in body_lower, "S6 must mention 'FNM_PATH'" + assert "skip" in body_lower or "skipped" in body_lower, ( + "S6 must mention 'skip' or 'skipped'" + ) + assert ( + "additive" in body_lower + or "complete grades" in body_lower + or "without fnm" in body_lower + ), "S6 must mention 'additive', 'complete grades', or 'without FNM'" + + def test_no_tool_rankings_or_recommendations(self) -> None: + """T10: Verify the document contains no tool-ranking language. + + The justification must be tool-agnostic and not contain any patterns + that rank, recommend, or compare tools by quality. + """ + text = _load_document() + text_lower = text.lower() + + forbidden_patterns = [ + "best tool", + "worst tool", + "recommended tool", + "winning tool", + "ranks higher", + "ranks lower", + "should choose", + "should select", + ] + + found = [p for p in forbidden_patterns if p in text_lower] + assert not found, ( + f"Document contains tool-ranking language: {found}. " + "The justification must be tool-agnostic." + ) diff --git a/data/fnm/tests/test_solved_snapshot.py b/data/fnm/tests/test_solved_snapshot.py new file mode 100644 index 00000000..c7dae8aa --- /dev/null +++ b/data/fnm/tests/test_solved_snapshot.py @@ -0,0 +1,436 @@ +"""Tests for solved-snapshot confirmation (PRD 08). + +Tests T01-T06 are synthetic unit tests requiring no FNM data. +Tests T07-T08 are integration tests using synthetic CSV fixtures. +Tests T09-T10 are FNM integration tests requiring FNM_PATH. +""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path + +import pytest + +from fnm.scripts.solved_snapshot import ( + DistributionStats, + IndicatorResult, + IndicatorSignal, + SnapshotClassification, + build_confirmation, + classify_overall, + classify_va, + classify_vm, + main, +) + +# --------------------------------------------------------------------------- +# T01: test_classify_vm_solved +# --------------------------------------------------------------------------- + + +def test_classify_vm_solved() -> None: + """VM with mean=1.01, std=0.03, 10% exact 1.0 -> SOLVED_SIGNAL.""" + stats = DistributionStats( + count=100, + mean=1.01, + std=0.03, + min=0.95, + max=1.06, + pct_exact_reference=10.0, + ) + result = classify_vm(stats) + assert result.signal == IndicatorSignal.SOLVED_SIGNAL + assert result.name == "VM" + + +# --------------------------------------------------------------------------- +# T02: test_classify_vm_flat +# --------------------------------------------------------------------------- + + +def test_classify_vm_flat() -> None: + """VM all exactly 1.0 -> FLAT_SIGNAL.""" + stats = DistributionStats( + count=100, + mean=1.0, + std=0.0, + min=1.0, + max=1.0, + pct_exact_reference=100.0, + ) + result = classify_vm(stats) + assert result.signal == IndicatorSignal.FLAT_SIGNAL + assert result.name == "VM" + + +# --------------------------------------------------------------------------- +# T03: test_classify_va_solved +# --------------------------------------------------------------------------- + + +def test_classify_va_solved() -> None: + """VA with mean=-5.2, std=8.3, 2% exactly 0.0 -> SOLVED_SIGNAL.""" + stats = DistributionStats( + count=100, + mean=-5.2, + std=8.3, + min=-30.0, + max=15.0, + pct_exact_reference=2.0, + ) + result = classify_va(stats) + assert result.signal == IndicatorSignal.SOLVED_SIGNAL + assert result.name == "VA" + + +# --------------------------------------------------------------------------- +# T04: test_classify_overall_solved +# --------------------------------------------------------------------------- + + +def test_classify_overall_solved() -> None: + """All three SOLVED_SIGNAL -> SOLVED.""" + vm = IndicatorResult("VM", IndicatorSignal.SOLVED_SIGNAL, "solved") + va = IndicatorResult("VA", IndicatorSignal.SOLVED_SIGNAL, "solved") + qg = IndicatorResult("Qg", IndicatorSignal.SOLVED_SIGNAL, "solved") + assert classify_overall(vm, va, qg) == SnapshotClassification.SOLVED + + +# --------------------------------------------------------------------------- +# T05: test_classify_overall_flat_start +# --------------------------------------------------------------------------- + + +def test_classify_overall_flat_start() -> None: + """All three FLAT_SIGNAL -> FLAT_START.""" + vm = IndicatorResult("VM", IndicatorSignal.FLAT_SIGNAL, "flat") + va = IndicatorResult("VA", IndicatorSignal.FLAT_SIGNAL, "flat") + qg = IndicatorResult("Qg", IndicatorSignal.FLAT_SIGNAL, "flat") + assert classify_overall(vm, va, qg) == SnapshotClassification.FLAT_START + + +# --------------------------------------------------------------------------- +# T06: test_classify_overall_indeterminate_mixed +# --------------------------------------------------------------------------- + + +def test_classify_overall_indeterminate_mixed() -> None: + """VM SOLVED, VA FLAT, Qg AMBIGUOUS -> INDETERMINATE.""" + vm = IndicatorResult("VM", IndicatorSignal.SOLVED_SIGNAL, "solved") + va = IndicatorResult("VA", IndicatorSignal.FLAT_SIGNAL, "flat") + qg = IndicatorResult("Qg", IndicatorSignal.AMBIGUOUS, "ambiguous") + assert classify_overall(vm, va, qg) == SnapshotClassification.INDETERMINATE + + +# --------------------------------------------------------------------------- +# Synthetic CSV helpers +# --------------------------------------------------------------------------- + + +def _write_bus_csv_with_header( + path: Path, + rows: list[dict[str, str | float]], +) -> None: + """Write a bus CSV with header row.""" + fieldnames = [ + "bus_i", + "type", + "Pd", + "Qd", + "Gs", + "Bs", + "area", + "vm", + "va", + "base_kv", + "zone", + "Vmax", + "Vmin", + ] + with open(path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + for row in rows: + writer.writerow(row) + + +def _write_gen_csv_with_header( + path: Path, + rows: list[dict[str, str | float]], +) -> None: + """Write a generator CSV with header row.""" + fieldnames = ["bus", "pg", "qg", "qmax", "qmin", "vs", "mbase", "status", "Pmax", "Pmin"] + with open(path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + for row in rows: + writer.writerow(row) + + +# --------------------------------------------------------------------------- +# T07: test_build_confirmation_solved_synthetic +# --------------------------------------------------------------------------- + + +def test_build_confirmation_solved_synthetic(tmp_path: Path) -> None: + """Synthetic CSV with solved values -> SOLVED, isolated bus excluded.""" + import random + + random.seed(42) + + bus_rows: list[dict[str, str | float]] = [] + # 50 non-isolated buses with varied VM (0.95-1.05) and VA (-20 to +15) + for i in range(1, 51): + vm = 0.95 + random.random() * 0.10 # 0.95 to 1.05 + va = -20.0 + random.random() * 35.0 # -20 to +15 + bus_rows.append( + { + "bus_i": i, + "type": 1, + "Pd": 100.0, + "Qd": 50.0, + "Gs": 0.0, + "Bs": 0.0, + "area": 1, + "vm": f"{vm:.6f}", + "va": f"{va:.6f}", + "base_kv": 230.0, + "zone": 1, + "Vmax": 1.1, + "Vmin": 0.9, + } + ) + + # 1 isolated bus (type=4) — should be excluded + bus_rows.append( + { + "bus_i": 99, + "type": 4, + "Pd": 0.0, + "Qd": 0.0, + "Gs": 0.0, + "Bs": 0.0, + "area": 1, + "vm": "1.000000", + "va": "0.000000", + "base_kv": 230.0, + "zone": 1, + "Vmax": 1.1, + "Vmin": 0.9, + } + ) + + bus_csv = tmp_path / "bus.csv" + _write_bus_csv_with_header(bus_csv, bus_rows) + + # 20 generators with non-zero Qg + gen_rows: list[dict[str, str | float]] = [] + for i in range(1, 21): + qg = -50.0 + random.random() * 100.0 # -50 to +50 MVAr + gen_rows.append( + { + "bus": i, + "pg": 200.0, + "qg": f"{qg:.4f}", + "qmax": 100.0, + "qmin": -100.0, + "vs": 1.0, + "mbase": 100.0, + "status": 1, + "Pmax": 500.0, + "Pmin": 0.0, + } + ) + + gen_csv = tmp_path / "gen.csv" + _write_gen_csv_with_header(gen_csv, gen_rows) + + confirmation = build_confirmation(bus_csv, gen_csv, canonical_parser="TEST") + + assert confirmation.classification == SnapshotClassification.SOLVED + assert confirmation.buses_excluded_isolated == 1 + assert confirmation.buses_analyzed == 50 + assert confirmation.vm_stats.std > 0.01 + assert confirmation.vm_indicator.signal == IndicatorSignal.SOLVED_SIGNAL + assert confirmation.va_indicator.signal == IndicatorSignal.SOLVED_SIGNAL + assert confirmation.qg_indicator.signal == IndicatorSignal.SOLVED_SIGNAL + + +# --------------------------------------------------------------------------- +# T08: test_build_confirmation_flat_start_synthetic +# --------------------------------------------------------------------------- + + +def test_build_confirmation_flat_start_synthetic(tmp_path: Path) -> None: + """Flat-start synthetic CSVs -> FLAT_START.""" + bus_rows: list[dict[str, str | float]] = [] + for i in range(1, 101): + bus_rows.append( + { + "bus_i": i, + "type": 1, + "Pd": 50.0, + "Qd": 20.0, + "Gs": 0.0, + "Bs": 0.0, + "area": 1, + "vm": "1.000000", + "va": "0.000000", + "base_kv": 345.0, + "zone": 1, + "Vmax": 1.1, + "Vmin": 0.9, + } + ) + + bus_csv = tmp_path / "bus.csv" + _write_bus_csv_with_header(bus_csv, bus_rows) + + gen_rows: list[dict[str, str | float]] = [] + for i in range(1, 31): + gen_rows.append( + { + "bus": i, + "pg": 100.0, + "qg": "0.0000", + "qmax": 50.0, + "qmin": -50.0, + "vs": 1.0, + "mbase": 100.0, + "status": 1, + "Pmax": 300.0, + "Pmin": 0.0, + } + ) + + gen_csv = tmp_path / "gen.csv" + _write_gen_csv_with_header(gen_csv, gen_rows) + + confirmation = build_confirmation(bus_csv, gen_csv, canonical_parser="TEST") + + assert confirmation.classification == SnapshotClassification.FLAT_START + assert confirmation.buses_analyzed == 100 + assert confirmation.buses_excluded_isolated == 0 + assert confirmation.vm_indicator.signal == IndicatorSignal.FLAT_SIGNAL + assert confirmation.va_indicator.signal == IndicatorSignal.FLAT_SIGNAL + assert confirmation.qg_indicator.signal == IndicatorSignal.FLAT_SIGNAL + + +# --------------------------------------------------------------------------- +# T09: test_fnm_snapshot_produces_classification (FNM required) +# --------------------------------------------------------------------------- + + +@pytest.mark.fnm +def test_fnm_snapshot_produces_classification(require_fnm) -> None: + """Run with actual parser output -> valid SnapshotClassification.""" + fnm_path = require_fnm.fnm_path + assert fnm_path is not None + + # Locate canonical parser CSV outputs in intermediate directory + repo_root = Path(__file__).resolve().parent.parent.parent.parent + intermediate = repo_root / "data" / "fnm" / "intermediate" + + # Try MATPOWER first, then GridCal + bus_csv = None + gen_csv = None + parser_name = "" + + mpc_bus = intermediate / "matpower" / "mpc_bus.csv" + mpc_gen = intermediate / "matpower" / "mpc_gen.csv" + gc_bus = intermediate / "gridcal" / "gridcal_buses.csv" + gc_gen = intermediate / "gridcal" / "gridcal_generators.csv" + + if mpc_bus.exists() and mpc_gen.exists(): + bus_csv = mpc_bus + gen_csv = mpc_gen + parser_name = "MATPOWER" + elif gc_bus.exists() and gc_gen.exists(): + bus_csv = gc_bus + gen_csv = gc_gen + parser_name = "GRIDCAL" + else: + pytest.skip( + "No canonical parser CSV outputs found in intermediate/. Run parser pipeline first." + ) + + confirmation = build_confirmation(bus_csv, gen_csv, canonical_parser=parser_name) + + assert confirmation.classification in ( + SnapshotClassification.SOLVED, + SnapshotClassification.FLAT_START, + SnapshotClassification.INDETERMINATE, + ) + assert confirmation.buses_analyzed > 0 + assert confirmation.qg_stats.total_generators > 0 + + +# --------------------------------------------------------------------------- +# T10: test_fnm_snapshot_report_files_written (FNM required) +# --------------------------------------------------------------------------- + + +@pytest.mark.fnm +def test_fnm_snapshot_report_files_written(require_fnm, tmp_path: Path) -> None: + """Run main() with actual CSVs -> verify JSON and markdown files exist.""" + fnm_path = require_fnm.fnm_path + assert fnm_path is not None + + repo_root = Path(__file__).resolve().parent.parent.parent.parent + intermediate = repo_root / "data" / "fnm" / "intermediate" + + mpc_bus = intermediate / "matpower" / "mpc_bus.csv" + mpc_gen = intermediate / "matpower" / "mpc_gen.csv" + gc_bus = intermediate / "gridcal" / "gridcal_buses.csv" + gc_gen = intermediate / "gridcal" / "gridcal_generators.csv" + + if mpc_bus.exists() and mpc_gen.exists(): + bus_csv = mpc_bus + gen_csv = mpc_gen + parser_name = "MATPOWER" + elif gc_bus.exists() and gc_gen.exists(): + bus_csv = gc_bus + gen_csv = gc_gen + parser_name = "GRIDCAL" + else: + pytest.skip( + "No canonical parser CSV outputs found in intermediate/. Run parser pipeline first." + ) + + main( + [ + "--bus-csv", + str(bus_csv), + "--gen-csv", + str(gen_csv), + "--parser", + parser_name, + "--output-dir", + str(tmp_path), + ] + ) + + json_path = tmp_path / "solved_snapshot_report.json" + md_path = tmp_path / "solved_snapshot_report.md" + + assert json_path.exists(), "JSON report was not created" + assert md_path.exists(), "Markdown report was not created" + + # Validate JSON structure + report = json.loads(json_path.read_text(encoding="utf-8")) + assert "classification" in report + assert report["classification"] in ("solved", "flat_start", "indeterminate") + assert "vm_stats" in report + assert "va_stats" in report + assert "qg_stats" in report + assert "vm_indicator" in report + assert "va_indicator" in report + assert "qg_indicator" in report + assert "phase3_implications" in report + + # Validate markdown has key sections + md_text = md_path.read_text(encoding="utf-8") + assert "# Solved-Snapshot Confirmation Report" in md_text + assert "## Phase 3 Implications" in md_text diff --git a/data/fnm/tests/test_three_winding_transformers_doc.py b/data/fnm/tests/test_three_winding_transformers_doc.py new file mode 100644 index 00000000..b8dcfd12 --- /dev/null +++ b/data/fnm/tests/test_three_winding_transformers_doc.py @@ -0,0 +1,353 @@ +"""Structural validation tests for the 3-winding transformer reference document. + +Tests verify that data/fnm/docs/three-winding-transformers.md contains all required +sections, field coverage, topology diagrams, impedance formulas, tool handling +documentation, and worked examples as specified in PRD 02/04. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +# --------------------------------------------------------------------------- +# Constants and fixtures +# --------------------------------------------------------------------------- + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +DOC_PATH = REPO_ROOT / "data" / "fnm" / "docs" / "three-winding-transformers.md" + +REQUIRED_H2_SECTIONS = [ + "Purpose", + "Audience", + "PSS/E v31 Record Structure", + "Star-Bus Equivalent Topology", + "Winding Parameters", + "Intermediate Format Representation", + "Tool Handling", + "Worked Example", + "Common Pitfalls", + "Cross-References", +] + +# All 83 PSS/E v31 3-winding transformer field names +DATA_LINE_1_FIELDS = [ + "I", + "J", + "K", + "CKT", + "CW", + "CZ", + "CM", + "MAG1", + "MAG2", + "NMETR", + "NAME", + "STAT", + "O1", + "F1", + "O2", + "F2", + "O3", + "F3", + "O4", + "F4", + "VECGRP", +] + +DATA_LINE_2_FIELDS = [ + "R1-2", + "X1-2", + "SBASE1-2", + "R2-3", + "X2-3", + "SBASE2-3", + "R3-1", + "X3-1", + "SBASE3-1", + "VMSTAR", + "ANSTAR", +] + +WINDING_FIELDS_TEMPLATE = [ + "WINDV{n}", + "NOMV{n}", + "ANG{n}", + "RATA{n}", + "RATB{n}", + "RATC{n}", + "COD{n}", + "CONT{n}", + "RMA{n}", + "RMI{n}", + "VMA{n}", + "VMI{n}", + "NTP{n}", + "TAB{n}", + "CR{n}", + "CX{n}", + "CNXA{n}", +] + +ALL_83_FIELDS: list[str] = list(DATA_LINE_1_FIELDS) + list(DATA_LINE_2_FIELDS) +for winding_num in (1, 2, 3): + ALL_83_FIELDS.extend(f.format(n=winding_num) for f in WINDING_FIELDS_TEMPLATE) + +TOOL_NAMES = ["PyPSA", "pandapower", "GridCal", "PowerModels", "PowerSimulations", "MATPOWER"] + + +@pytest.fixture(scope="module") +def doc_text() -> str: + """Read the 3-winding transformer document and return its full text.""" + assert DOC_PATH.exists(), f"Document not found at {DOC_PATH}" + text = DOC_PATH.read_text(encoding="utf-8") + assert len(text) > 0, "Document is empty" + return text + + +@pytest.fixture(scope="module") +def doc_sections(doc_text: str) -> dict[str, str]: + """Split the document into top-level (##) sections keyed by heading.""" + sections: dict[str, str] = {} + current_heading = "" + current_lines: list[str] = [] + for line in doc_text.splitlines(): + if line.startswith("## ") and not line.startswith("### "): + if current_heading: + sections[current_heading] = "\n".join(current_lines) + current_heading = line[3:].strip() + current_lines = [] + else: + current_lines.append(line) + if current_heading: + sections[current_heading] = "\n".join(current_lines) + return sections + + +def _find_section(sections: dict[str, str], substring: str) -> str: + """Find a section whose heading contains the given substring (case-insensitive). + + Returns the section body text, or empty string if not found. + """ + sub_lower = substring.lower() + for heading, body in sections.items(): + if sub_lower in heading.lower(): + return body + return "" + + +# --------------------------------------------------------------------------- +# Document structure tests -- T01-T03 +# --------------------------------------------------------------------------- + + +@pytest.mark.docs +class TestDocumentStructure: + """T01-T03: Verify the document has all required sections.""" + + def test_document_exists_at_expected_path(self) -> None: + """T01: Verify document exists at data/fnm/docs/three-winding-transformers.md.""" + assert DOC_PATH.exists(), f"Expected document at {DOC_PATH}" + content = DOC_PATH.read_text(encoding="utf-8") + assert len(content) > 0, "Document exists but is empty" + + def test_required_top_level_sections_present(self, doc_sections: dict[str, str]) -> None: + """T02: All required H2 sections exist.""" + for section_name in REQUIRED_H2_SECTIONS: + found = any(section_name.lower() in heading.lower() for heading in doc_sections) + assert found, ( + f"Missing required H2 section: '## {section_name}'. " + f"Found sections: {list(doc_sections.keys())}" + ) + + def test_data_line_subsections_present(self, doc_sections: dict[str, str]) -> None: + """T03: Within PSS/E v31 Record Structure, subsections for all 5 data lines.""" + record_section = _find_section(doc_sections, "PSS/E v31 Record Structure") + assert record_section, "Could not find 'PSS/E v31 Record Structure' section" + + for line_num in range(1, 6): + patterns = [ + f"Data Line {line_num}", + f"Line {line_num}", + ] + found = any(p in record_section for p in patterns) + assert found, ( + f"Missing subsection for Data Line {line_num} in 'PSS/E v31 Record Structure'" + ) + + +# --------------------------------------------------------------------------- +# Field coverage tests -- T04-T06 +# --------------------------------------------------------------------------- + + +@pytest.mark.docs +class TestFieldCoverage: + """T04-T06: Verify all 83 fields are documented.""" + + def test_all_83_fields_documented(self, doc_text: str) -> None: + """T04: All 83 PSS/E field names appear at least once in the document.""" + missing_fields: list[str] = [] + for field in ALL_83_FIELDS: + # Check for backtick-delimited or plain occurrence + if f"`{field}`" not in doc_text and field not in doc_text: + missing_fields.append(field) + assert not missing_fields, ( + f"Missing {len(missing_fields)} of 83 fields in the document: {missing_fields}" + ) + + def test_all_three_windings_covered(self, doc_text: str) -> None: + """T05: Per-winding fields documented for all three windings.""" + winding_field_families = { + "WINDV": ["WINDV1", "WINDV2", "WINDV3"], + "COD": ["COD1", "COD2", "COD3"], + "RATA": ["RATA1", "RATA2", "RATA3"], + "NTP": ["NTP1", "NTP2", "NTP3"], + } + for family, fields in winding_field_families.items(): + for field in fields: + assert field in doc_text, ( + f"Winding field '{field}' (family '{family}') not found in document" + ) + + def test_field_count_summary_states_83(self, doc_text: str) -> None: + """T06: Document contains a field count summary stating 83 fields total.""" + pattern = re.compile(r"83\s*(?:fields|total\s*fields)", re.IGNORECASE) + alt_pattern = re.compile(r"total[:\s]*83", re.IGNORECASE) + assert pattern.search(doc_text) or alt_pattern.search(doc_text), ( + "Document does not contain a field count summary stating '83 fields' or 'total: 83'" + ) + + +# --------------------------------------------------------------------------- +# Topology and formula tests -- T07-T08 +# --------------------------------------------------------------------------- + + +@pytest.mark.docs +class TestTopologyAndFormulas: + """T07-T08: Verify star-bus diagram and impedance conversion formulas.""" + + def test_star_bus_diagram_present(self, doc_sections: dict[str, str]) -> None: + """T07: Star-Bus Equivalent Topology section contains a topology diagram.""" + topology_section = _find_section(doc_sections, "Star-Bus Equivalent Topology") + assert topology_section, "Could not find 'Star-Bus Equivalent Topology' section" + + # Check for a fenced code block or indented block with key elements + diagram_elements = [ + ("Bus I", "Winding 1"), + ("Bus J", "Winding 2"), + ("Bus K", "Winding 3"), + ("Star", "star bus"), + ] + + matches = 0 + for primary, alt in diagram_elements: + if primary in topology_section or alt in topology_section: + matches += 1 + + assert matches >= 3, ( + f"Topology diagram should contain at least 3 of the 4 key elements " + f"(Bus I/Winding 1, Bus J/Winding 2, Bus K/Winding 3, Star/star bus). " + f"Found {matches}." + ) + + # Verify there is a fenced code block (```) in the section + assert "```" in topology_section, ( + "Topology section should contain a fenced code block for the diagram" + ) + + def test_impedance_conversion_formulas_present(self, doc_text: str) -> None: + """T08: All three star-leg impedance conversion formulas are present.""" + # Z1 = (Z1-2 + Z3-1 - Z2-3) / 2 (or equivalent notation) + # Z2 = (Z1-2 + Z2-3 - Z3-1) / 2 + # Z3 = (Z2-3 + Z3-1 - Z1-2) / 2 + + # Flexible patterns to match various notations: + # Z_1, Z1, Z_12, Z12, Z1-2, etc. + z1_pattern = re.compile( + r"Z_?1\s*=\s*\(.*(Z_?1[-_]?2|Z_?12).*" + r"(Z_?3[-_]?1|Z_?31).*" + r"(Z_?2[-_]?3|Z_?23).*\)\s*/\s*2", + re.IGNORECASE, + ) + z2_pattern = re.compile( + r"Z_?2\s*=\s*\(.*(Z_?1[-_]?2|Z_?12).*" + r"(Z_?2[-_]?3|Z_?23).*" + r"(Z_?3[-_]?1|Z_?31).*\)\s*/\s*2", + re.IGNORECASE, + ) + z3_pattern = re.compile( + r"Z_?3\s*=\s*\(.*(Z_?2[-_]?3|Z_?23).*" + r"(Z_?3[-_]?1|Z_?31).*" + r"(Z_?1[-_]?2|Z_?12).*\)\s*/\s*2", + re.IGNORECASE, + ) + + assert z1_pattern.search(doc_text), ( + "Missing Z1 star-leg impedance formula: Z1 = (Z1-2 + Z3-1 - Z2-3) / 2" + ) + assert z2_pattern.search(doc_text), ( + "Missing Z2 star-leg impedance formula: Z2 = (Z1-2 + Z2-3 - Z3-1) / 2" + ) + assert z3_pattern.search(doc_text), ( + "Missing Z3 star-leg impedance formula: Z3 = (Z2-3 + Z3-1 - Z1-2) / 2" + ) + + +# --------------------------------------------------------------------------- +# Tool handling test -- T09 +# --------------------------------------------------------------------------- + + +@pytest.mark.docs +class TestToolHandling: + """T09: Verify all six tools are documented in the Tool Handling section.""" + + def test_all_six_tools_documented(self, doc_sections: dict[str, str]) -> None: + """T09: All six tool names appear in the Tool Handling section.""" + tool_section = _find_section(doc_sections, "Tool Handling") + assert tool_section, "Could not find 'Tool Handling' section" + + tool_section_lower = tool_section.lower() + missing_tools: list[str] = [] + for tool in TOOL_NAMES: + if tool.lower() not in tool_section_lower: + missing_tools.append(tool) + + assert not missing_tools, f"Missing tools in 'Tool Handling' section: {missing_tools}" + + +# --------------------------------------------------------------------------- +# Worked example test -- T10 +# --------------------------------------------------------------------------- + + +@pytest.mark.docs +class TestWorkedExample: + """T10: Verify worked example uses realistic values.""" + + def test_worked_example_uses_realistic_values(self, doc_sections: dict[str, str]) -> None: + """T10: Worked Example section contains realistic transmission-scale values.""" + example_section = _find_section(doc_sections, "Worked Example") + assert example_section, "Could not find 'Worked Example' section" + + # Extract numeric values from the section + numbers = [float(m) for m in re.findall(r"(? None: + """Write a list of dicts as a CSV file with header.""" + if not rows: + path.write_text("", encoding="utf-8") + return + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=list(rows[0].keys())) + writer.writeheader() + writer.writerows(rows) + + +def _write_json(path: Path, data: dict) -> None: + """Write a dict as a JSON file.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2), encoding="utf-8") + + +# =========================================================================== +# T01 / T02 -- ACPF Power Balance +# =========================================================================== + + +class TestAcpfPowerBalance: + """Tests for check_acpf_power_balance (ACPF Check A).""" + + def test_acpf_power_balance_pass(self) -> None: + """T01: Power balance within tolerance -> PASS.""" + summary = { + "system_summary": { + "total_gen_mw": 10000.0, + "total_load_mw": 9700.0, + "total_loss_mw": 300.0, + "slack_bus": 1, + } + } + result = check_acpf_power_balance(summary) + assert result.status == CheckStatus.PASS + assert result.metric_value is not None + assert result.metric_value < 0.01 + assert result.tolerance == 1.0 + + def test_acpf_power_balance_fail(self) -> None: + """T02: 2 MW residual -> FAIL.""" + summary = { + "system_summary": { + "total_gen_mw": 10000.0, + "total_load_mw": 9700.0, + "total_loss_mw": 298.0, + "slack_bus": 1, + } + } + result = check_acpf_power_balance(summary) + assert result.status == CheckStatus.FAIL + assert result.metric_value is not None + assert abs(result.metric_value - 2.0) < 0.01 + + +# =========================================================================== +# T03 / T04 / T05 -- ACPF Per-Bus KCL +# =========================================================================== + + +def _make_balanced_5bus() -> tuple[list[dict], list[dict], list[dict], list[dict]]: + """Build a balanced 5-bus synthetic network. + + Bus 1: Generator 100 MW, 20 MVAr, load 0 + Bus 2: Generator 50 MW, 10 MVAr, load 0 + Bus 3: Load 60 MW, 12 MVAr + Bus 4: Load 50 MW, 10 MVAr + Bus 5: Load 40 MW, 8 MVAr + + Branches carry flows such that KCL is exactly satisfied. + We assign flows manually to make KCL hold: + + Branch 1->3: P_from=40, Q_from=8, P_to=-40, Q_to=-8 + Branch 1->4: P_from=30, Q_from=6, P_to=-30, Q_to=-6 + Branch 1->5: P_from=30, Q_from=6, P_to=-30, Q_to=-6 + Branch 2->3: P_from=20, Q_from=4, P_to=-20, Q_to=-4 + Branch 2->4: P_from=20, Q_from=4, P_to=-20, Q_to=-4 + Branch 2->5: P_from=10, Q_from=2, P_to=-10, Q_to=-2 + + KCL at bus 1: Gen(100,20) - Load(0,0) - BranchOut(40+30+30, 8+6+6) = 0,0 + KCL at bus 2: Gen(50,10) - Load(0,0) - BranchOut(20+20+10, 4+4+2) = 0,0 + KCL at bus 3: Gen(0,0) - Load(60,12) - BranchOut(-40-20, -8-4) = 0,0 + KCL at bus 4: Gen(0,0) - Load(50,10) - BranchOut(-30-20, -6-4) = 0,0 + KCL at bus 5: Gen(0,0) - Load(40,8) - BranchOut(-30-10, -6-2) = 0,0 + """ + acpf_buses = [ + {"bus": 1, "VM": 1.0, "VA": 0.0}, + {"bus": 2, "VM": 1.0, "VA": -1.0}, + {"bus": 3, "VM": 0.98, "VA": -2.0}, + {"bus": 4, "VM": 0.99, "VA": -1.5}, + {"bus": 5, "VM": 0.97, "VA": -2.5}, + ] + + acpf_branches = [ + { + "from_bus": 1, + "to_bus": 3, + "ckt": "1", + "P_from": 40.0, + "Q_from": 8.0, + "P_to": -40.0, + "Q_to": -8.0, + }, + { + "from_bus": 1, + "to_bus": 4, + "ckt": "1", + "P_from": 30.0, + "Q_from": 6.0, + "P_to": -30.0, + "Q_to": -6.0, + }, + { + "from_bus": 1, + "to_bus": 5, + "ckt": "1", + "P_from": 30.0, + "Q_from": 6.0, + "P_to": -30.0, + "Q_to": -6.0, + }, + { + "from_bus": 2, + "to_bus": 3, + "ckt": "1", + "P_from": 20.0, + "Q_from": 4.0, + "P_to": -20.0, + "Q_to": -4.0, + }, + { + "from_bus": 2, + "to_bus": 4, + "ckt": "1", + "P_from": 20.0, + "Q_from": 4.0, + "P_to": -20.0, + "Q_to": -4.0, + }, + { + "from_bus": 2, + "to_bus": 5, + "ckt": "1", + "P_from": 10.0, + "Q_from": 2.0, + "P_to": -10.0, + "Q_to": -2.0, + }, + ] + + acpf_generators = [ + {"bus": 1, "machine_id": "1", "P": 100.0, "Q": 20.0}, + {"bus": 2, "machine_id": "1", "P": 50.0, "Q": 10.0}, + ] + + intermediate_buses = [ + {"bus": 1, "bus_type": 3, "PD": 0.0, "QD": 0.0}, + {"bus": 2, "bus_type": 2, "PD": 0.0, "QD": 0.0}, + {"bus": 3, "bus_type": 1, "PD": 60.0, "QD": 12.0}, + {"bus": 4, "bus_type": 1, "PD": 50.0, "QD": 10.0}, + {"bus": 5, "bus_type": 1, "PD": 40.0, "QD": 8.0}, + ] + + return acpf_buses, acpf_branches, acpf_generators, intermediate_buses + + +class TestAcpfKcl: + """Tests for check_acpf_kcl (ACPF Check B).""" + + def test_acpf_kcl_balanced_network(self) -> None: + """T03: Balanced 5-bus network -> PASS.""" + acpf_buses, acpf_branches, acpf_generators, intermediate_buses = _make_balanced_5bus() + result = check_acpf_kcl( + acpf_buses, + acpf_branches, + acpf_generators, + intermediate_buses, + excluded_buses=set(), + ) + assert result.status == CheckStatus.PASS + assert result.metric_value is not None + assert result.metric_value < 0.01 + assert result.failing_elements == 0 + + def test_acpf_kcl_single_bus_violation(self) -> None: + """T04: Perturbed branch flow at bus 1 -> FAIL.""" + acpf_buses, acpf_branches, acpf_generators, intermediate_buses = _make_balanced_5bus() + # Perturb branch 1->3 P_from by +0.2 MW + acpf_branches[0]["P_from"] = 40.2 + + result = check_acpf_kcl( + acpf_buses, + acpf_branches, + acpf_generators, + intermediate_buses, + excluded_buses=set(), + ) + assert result.status == CheckStatus.FAIL + assert result.failing_elements >= 1 + assert result.detail is not None + bus1_entries = [d for d in result.detail if d["bus"] == 1] + assert len(bus1_entries) >= 1 + assert bus1_entries[0]["mismatch_mva"] > 0.1 + + def test_acpf_kcl_excludes_registry_buses(self) -> None: + """T05: Excluded bus 6 is not checked.""" + acpf_buses, acpf_branches, acpf_generators, intermediate_buses = _make_balanced_5bus() + # Add bus 6 (excluded) with a load that would cause mismatch + acpf_buses.append({"bus": 6, "VM": 1.0, "VA": 0.0}) + intermediate_buses.append({"bus": 6, "bus_type": 1, "PD": 100.0, "QD": 50.0}) + + result = check_acpf_kcl( + acpf_buses, + acpf_branches, + acpf_generators, + intermediate_buses, + excluded_buses={6}, + ) + # Bus 6 should not appear in detail + if result.detail: + bus6_entries = [d for d in result.detail if d["bus"] == 6] + assert len(bus6_entries) == 0 + # total_elements should be 5, not 6 + assert result.total_elements == 5 + + +# =========================================================================== +# T06 / T07 -- ACPF Voltage Plausibility +# =========================================================================== + + +class TestAcpfVmPlausibility: + """Tests for check_acpf_vm_plausibility (ACPF Check C).""" + + def test_acpf_vm_all_plausible(self) -> None: + """T06: All 20 buses in [0.95, 1.05] -> PASS.""" + buses = [{"bus": i, "VM": 0.95 + (i - 1) * 0.1 / 19, "VA": 0.0} for i in range(1, 21)] + result = check_acpf_vm_plausibility(buses, excluded_buses=set()) + assert result.status == CheckStatus.PASS + assert result.failing_elements == 0 + + def test_acpf_vm_out_of_range(self) -> None: + """T07: Two buses out of range -> FAIL.""" + buses = [{"bus": i, "VM": 1.0, "VA": 0.0} for i in range(1, 19)] + buses.append({"bus": 19, "VM": 0.75, "VA": 0.0}) + buses.append({"bus": 20, "VM": 1.25, "VA": 0.0}) + + result = check_acpf_vm_plausibility(buses, excluded_buses=set()) + assert result.status == CheckStatus.FAIL + assert result.failing_elements == 2 + assert result.detail is not None + detail_buses = {d["bus"] for d in result.detail} + assert 19 in detail_buses + assert 20 in detail_buses + + +# =========================================================================== +# T08 / T09 / T10 -- ACPF Generator Limits +# =========================================================================== + + +def _make_generators( + n: int, *, p_offset: float = 0.0, q_offset: float = 0.0 +) -> tuple[list[dict], list[dict]]: + """Create n generators within limits, returning (acpf, intermediate) lists.""" + acpf_gens = [] + int_gens = [] + for i in range(1, n + 1): + p_val = 50.0 + q_val = 10.0 + acpf_gens.append({"bus": i, "machine_id": "1", "P": p_val, "Q": q_val}) + int_gens.append( + { + "bus": i, + "machine_id": "1", + "status": 1, + "PG": p_val, + "QG": q_val, + "PT": 100.0, + "PB": 0.0, + "QT": 50.0, + "QB": -50.0, + } + ) + return acpf_gens, int_gens + + +class TestAcpfGeneratorLimits: + """Tests for check_acpf_generator_limits (ACPF Check D).""" + + def test_acpf_gen_limits_all_within(self) -> None: + """T08: All 10 generators within limits -> PASS.""" + acpf_gens, int_gens = _make_generators(10) + summary = {"system_summary": {"slack_bus": 999}} # No match -> no exemption + + result = check_acpf_generator_limits(acpf_gens, int_gens, summary) + assert result.status == CheckStatus.PASS + assert result.failing_elements == 0 + + def test_acpf_gen_limits_violation(self) -> None: + """T09: Two generators with violations -> FAIL.""" + acpf_gens, int_gens = _make_generators(10) + summary = {"system_summary": {"slack_bus": 999}} + + # Generator at bus 3: P above PT by 0.5 MW + acpf_gens[2]["P"] = 100.6 # PT=100, tolerance=0.1 -> violation + # Generator at bus 7: Q below QB by 0.5 MVAr + acpf_gens[6]["Q"] = -50.6 # QB=-50, tolerance=0.1 -> violation + + result = check_acpf_generator_limits(acpf_gens, int_gens, summary) + assert result.status == CheckStatus.FAIL + assert result.failing_elements == 2 + assert result.detail is not None + + violation_types = set() + for d in result.detail: + for vt in d["violation_type"]: + violation_types.add(vt) + assert "P_above_PT" in violation_types + assert "Q_below_QB" in violation_types + + def test_acpf_gen_limits_slack_exempt(self) -> None: + """T10: Slack bus generator exempt from P-limit check.""" + acpf_gens, int_gens = _make_generators(5) + # The slack bus is bus 1, set its P way above PT + acpf_gens[0]["P"] = 150.0 # PT=100, +50 MW above limit + summary = {"system_summary": {"slack_bus": 1}} + + result = check_acpf_generator_limits(acpf_gens, int_gens, summary) + + # Slack generator should NOT appear as a P-limit violator + if result.detail: + for d in result.detail: + if d["bus"] == 1: + # Should not have P_above_PT violation + assert "P_above_PT" not in d["violation_type"] + + # Notes should mention slack exemption + assert any("Slack bus generator" in n and "exempt" in n for n in result.notes) + + +# =========================================================================== +# T11 / T12 -- DCPF Power Balance +# =========================================================================== + + +class TestDcpfPowerBalance: + """Tests for check_dcpf_power_balance (DCPF Check A).""" + + def test_dcpf_power_balance_pass(self) -> None: + """T11: Balanced lossless network -> PASS.""" + summary = { + "power_summary": { + "total_generation_mw": 5000.0, + "total_load_mw": 5000.0, + "slack_injection_mw": 0.0, + } + } + result = check_dcpf_power_balance(summary) + assert result.status == CheckStatus.PASS + assert result.metric_value is not None + assert result.metric_value < 0.01 + + def test_dcpf_power_balance_fail(self) -> None: + """T12: 0.5 MW residual -> FAIL.""" + summary = { + "power_summary": { + "total_generation_mw": 5000.0, + "total_load_mw": 4999.0, + "slack_injection_mw": 0.5, + } + } + result = check_dcpf_power_balance(summary) + assert result.status == CheckStatus.FAIL + assert result.metric_value is not None + assert abs(result.metric_value - 0.5) < 0.01 + + +# =========================================================================== +# T13 / T14 -- DCPF Flow-Angle Consistency +# =========================================================================== + + +def _make_3bus_dc_network() -> tuple[list[dict], list[dict], list[dict], dict]: + """Build a 3-bus DC network for flow-angle tests. + + Bus 1 (slack, VA=0), Bus 2 (VA=-2.0 deg), Bus 3 (VA=-3.5 deg). + Branches: 1-2 (X=0.01), 2-3 (X=0.02), 1-3 (X=0.03). + baseMVA=100. + """ + dcpf_buses = [ + {"bus": 1, "VA": 0.0}, + {"bus": 2, "VA": -2.0}, + {"bus": 3, "VA": -3.5}, + ] + + intermediate_branches = [ + { + "from_bus": 1, + "to_bus": 2, + "ckt": "1", + "x_pu": 0.01, + "tap_ratio": 1.0, + "shift_deg": 0.0, + "status": 1, + }, + { + "from_bus": 2, + "to_bus": 3, + "ckt": "1", + "x_pu": 0.02, + "tap_ratio": 1.0, + "shift_deg": 0.0, + "status": 1, + }, + { + "from_bus": 1, + "to_bus": 3, + "ckt": "1", + "x_pu": 0.03, + "tap_ratio": 1.0, + "shift_deg": 0.0, + "status": 1, + }, + ] + + base_mva = 100.0 + dcpf_summary = {"base_mva": base_mva, "settings": {"slack_bus": 1}} + + # Compute expected flows: P = (VA_from - VA_to) * pi/180 / X * baseMVA + dcpf_branches = [] + for br in intermediate_branches: + va_from = next(b["VA"] for b in dcpf_buses if b["bus"] == br["from_bus"]) + va_to = next(b["VA"] for b in dcpf_buses if b["bus"] == br["to_bus"]) + p_expected = (va_from - va_to) * math.pi / 180.0 / br["x_pu"] * base_mva + dcpf_branches.append( + { + "from_bus": br["from_bus"], + "to_bus": br["to_bus"], + "ckt": br["ckt"], + "P_flow_MW": p_expected, + } + ) + + return dcpf_buses, dcpf_branches, intermediate_branches, dcpf_summary + + +class TestDcpfFlowAngleConsistency: + """Tests for check_dcpf_flow_angle_consistency (DCPF Check B).""" + + def test_dcpf_flow_angle_consistent(self) -> None: + """T13: Consistent 3-bus network -> PASS.""" + dcpf_buses, dcpf_branches, int_branches, dcpf_summary = _make_3bus_dc_network() + result = check_dcpf_flow_angle_consistency( + dcpf_buses, + dcpf_branches, + int_branches, + dcpf_summary, + ) + assert result.status == CheckStatus.PASS + assert result.metric_value is not None + assert result.metric_value < 0.01 + + def test_dcpf_flow_angle_inconsistent(self) -> None: + """T14: Branch 1-2 flow perturbed by 0.5 MW -> FAIL.""" + dcpf_buses, dcpf_branches, int_branches, dcpf_summary = _make_3bus_dc_network() + # Add 0.5 MW to branch 1-2 stored flow + dcpf_branches[0]["P_flow_MW"] += 0.5 + + result = check_dcpf_flow_angle_consistency( + dcpf_buses, + dcpf_branches, + int_branches, + dcpf_summary, + ) + assert result.status == CheckStatus.FAIL + assert result.failing_elements >= 1 + assert result.detail is not None + br12 = [d for d in result.detail if d["from_bus"] == 1 and d["to_bus"] == 2] + assert len(br12) == 1 + assert abs(br12[0]["deviation_mw"] - 0.5) < 0.01 + + +# =========================================================================== +# T15 -- DCPF Slack Angle +# =========================================================================== + + +class TestDcpfSlackAngle: + """Tests for check_dcpf_slack_angle (DCPF Check C).""" + + def test_dcpf_slack_angle_zero(self) -> None: + """T15: Slack bus angle is exactly 0.0 -> PASS.""" + dcpf_buses = [ + {"bus": 1, "VA": 0.0}, + {"bus": 2, "VA": -5.0}, + {"bus": 3, "VA": -8.0}, + ] + dcpf_summary = {"settings": {"slack_bus": 1}} + + result = check_dcpf_slack_angle(dcpf_buses, dcpf_summary) + assert result.status == CheckStatus.PASS + assert result.metric_value == 0.0 + + +# =========================================================================== +# T16 -- Integration test (requires FNM_PATH + D2/D3 outputs) +# =========================================================================== + + +@pytest.mark.fnm +class TestFnmValidationReport: + """Integration test using real FNM reference data.""" + + def test_fnm_validation_report_produces_outputs( + self, + require_fnm: dict, + tmp_path: Path, + ) -> None: + """T16: Run full validation on real FNM reference data.""" + repo_root = Path(__file__).resolve().parent.parent.parent.parent + acpf_dir = repo_root / "data" / "fnm" / "reference" / "acpf" + dcpf_dir = repo_root / "data" / "fnm" / "reference" / "dcpf" + intermediate_dir = repo_root / "data" / "fnm" / "intermediate" / "canonical" + reference_dir = repo_root / "data" / "fnm" / "reference" + + run_validation( + acpf_dir=acpf_dir, + dcpf_dir=dcpf_dir, + intermediate_dir=intermediate_dir, + reference_dir=reference_dir, + output_dir=tmp_path, + ) + + # Both output files should exist + json_path = tmp_path / "validation_report.json" + md_path = tmp_path / "validation_report.md" + assert json_path.exists() + assert md_path.exists() + + # Read and verify JSON structure + with open(json_path, encoding="utf-8") as f: + data = json.load(f) + + assert data["summary"]["total_checks"] == 7 + + # No check should be skipped (all inputs should be present) + for check in data["checks"]: + assert check["status"] != "skip", ( + f"Check {check['check_id']} was skipped: {check.get('skip_reason')}" + ) + + # All required fields present in each check + required_fields = { + "check_id", + "check_name", + "status", + "metric_value", + "metric_unit", + "tolerance", + "tolerance_unit", + "total_elements", + "passing_elements", + "failing_elements", + "detail", + "notes", + "skip_reason", + } + for check in data["checks"]: + missing = required_fields - set(check.keys()) + assert not missing, f"Check {check['check_id']} missing fields: {missing}" + + # Log per-check results for manual review + for check in data["checks"]: + print(f" {check['check_id']}: {check['status']} (metric={check['metric_value']})") diff --git a/report/docs/assets/grid-primer-diagram-validation.md b/report/docs/assets/grid-primer-diagram-validation.md deleted file mode 100644 index d3e18c87..00000000 --- a/report/docs/assets/grid-primer-diagram-validation.md +++ /dev/null @@ -1,148 +0,0 @@ -# Grid Primer Diagram Validation Report - -> Auto-generated by `report/scripts/validate_diagram_style.py`. - -Overall status: **PASS** - ---- - -## Check Summary - -| Check | Status | Issues | -|-------|--------|--------| -| Color consistency | PASS | 0 | -| Symbol consistency | PASS | 0 | -| Sizing consistency | PASS | 0 | -| Cumulative layering | PASS | 0 | -| Legend presence | PASS | 0 | -| ViewBox consistency | PASS | 0 | -| No raster images | PASS | 0 | -| No external dependencies | PASS | 0 | -| File sizes | PASS | 0 | -| Visual progression | PASS | 0 | - -## Per-Diagram Results - -### Stage 1: `stage-1_single-bus.svg` -- PASS - -| Check | Status | Evidence | -|-------|--------|----------| -| Color consistency | PASS | OK | -| Symbol consistency | PASS | OK | -| Sizing consistency | PASS | OK | -| Cumulative layering | PASS | OK | -| Legend presence | PASS | OK | -| ViewBox consistency | PASS | OK | -| No raster images | PASS | OK | -| No external dependencies | PASS | OK | -| File sizes | PASS | OK | -| Visual progression | PASS | OK | - -### Stage 2: `stage-2_two-bus.svg` -- PASS - -| Check | Status | Evidence | -|-------|--------|----------| -| Color consistency | PASS | OK | -| Symbol consistency | PASS | OK | -| Sizing consistency | PASS | OK | -| Cumulative layering | PASS | OK | -| Legend presence | PASS | OK | -| ViewBox consistency | PASS | OK | -| No raster images | PASS | OK | -| No external dependencies | PASS | OK | -| File sizes | PASS | OK | -| Visual progression | PASS | OK | - -### Stage 3: `stage-3_meshed-network.svg` -- PASS - -| Check | Status | Evidence | -|-------|--------|----------| -| Color consistency | PASS | OK | -| Symbol consistency | PASS | OK | -| Sizing consistency | PASS | OK | -| Cumulative layering | PASS | OK | -| Legend presence | PASS | OK | -| ViewBox consistency | PASS | OK | -| No raster images | PASS | OK | -| No external dependencies | PASS | OK | -| File sizes | PASS | OK | -| Visual progression | PASS | OK | - -### Stage 4: `stage-4_opf-dispatch.svg` -- PASS - -| Check | Status | Evidence | -|-------|--------|----------| -| Color consistency | PASS | OK | -| Symbol consistency | PASS | OK | -| Sizing consistency | PASS | OK | -| Cumulative layering | PASS | OK | -| Legend presence | PASS | OK | -| ViewBox consistency | PASS | OK | -| No raster images | PASS | OK | -| No external dependencies | PASS | OK | -| File sizes | PASS | OK | -| Visual progression | PASS | OK | - -### Stage 5: `stage-5_congestion.svg` -- PASS - -| Check | Status | Evidence | -|-------|--------|----------| -| Color consistency | PASS | OK | -| Symbol consistency | PASS | OK | -| Sizing consistency | PASS | OK | -| Cumulative layering | PASS | OK | -| Legend presence | PASS | OK | -| ViewBox consistency | PASS | OK | -| No raster images | PASS | OK | -| No external dependencies | PASS | OK | -| File sizes | PASS | OK | -| Visual progression | PASS | OK | - -### Stage 6: `stage-6_scopf.svg` -- PASS - -| Check | Status | Evidence | -|-------|--------|----------| -| Color consistency | PASS | OK | -| Symbol consistency | PASS | OK | -| Sizing consistency | PASS | OK | -| Cumulative layering | PASS | OK | -| Legend presence | PASS | OK | -| ViewBox consistency | PASS | OK | -| No raster images | PASS | OK | -| No external dependencies | PASS | OK | -| File sizes | PASS | OK | -| Visual progression | PASS | OK | - -## File Sizes - -| File | Size (KB) | Under 50 KB? | -|------|-----------|-------------| -| stage-1_single-bus.svg | 1.5 | Yes | -| stage-2_two-bus.svg | 2.6 | Yes | -| stage-3_meshed-network.svg | 5.1 | Yes | -| stage-4_opf-dispatch.svg | 6.1 | Yes | -| stage-5_congestion.svg | 6.2 | Yes | -| stage-6_scopf.svg | 6.9 | Yes | - -## Visual Progression (Element Counts) - -| Stage | File | Elements | -|-------|------|----------| -| 1 | stage-1_single-bus.svg | 11 | -| 2 | stage-2_two-bus.svg | 17 | -| 3 | stage-3_meshed-network.svg | 37 | -| 4 | stage-4_opf-dispatch.svg | 45 | -| 5 | stage-5_congestion.svg | 44 | -| 6 | stage-6_scopf.svg | 49 | - -## Hygiene Summary - -No raster images found. No external dependencies found. All files under 50 KB. - -## Consistency Summary - -All colors are on-palette. Symbol usage is consistent. Sizing is consistent. - -## Progression Summary - -Element count is non-decreasing from Stage 1 through Stage 6. diff --git a/report/docs/assets/grid-primer-style-guide.md b/report/docs/assets/grid-primer-style-guide.md deleted file mode 100644 index 15ebbdc1..00000000 --- a/report/docs/assets/grid-primer-style-guide.md +++ /dev/null @@ -1,148 +0,0 @@ -# Grid Primer Diagram Style Guide - -This document defines the visual vocabulary, color palette, sizing conventions, -cumulative-layering treatment, and legend requirements for the six Grid Primer -SVG diagrams (Stages 1 through 6). All diagrams must conform to these rules so -the sequence reads as a single, coherent visual narrative. - ---- - -## Color Palette - -Every `fill` and `stroke` value in the diagrams must come from the palette -below. No `inherit`, `currentColor`, or unnamed colors are allowed. - -### Element Colors - -| Role | Hex | Usage | -|------|-----|-------| -| Bus fill | `none` | Bus circles are unfilled | -| Bus stroke | `#333333` | Dark gray outline for bus nodes | -| Generator stroke / fill | `#2e7d32` | Green for generator circles, lines, labels | -| Generator background | `#e8f5e9` | Light green fill for dispatch annotation boxes | -| Load stroke | `#c62828` | Red for load arrows, lines, labels | -| Load background | `#ffebee` | Light red fill for load annotation boxes | -| Line normal | `#1565c0` | Blue for uncongested transmission lines and flow arrows | -| Line background | `#e3f2fd` | Light blue fill for annotation boxes (e.g., N-1 box) | -| Line congested | `#e65100` | Orange for congested lines and thermal-limit indicators | -| Congestion background | `#ffccbc` | Light orange fill for thermal-limit bar background | -| Line tripped | `#b71c1c` | Dark red for tripped/contingency lines and X markers | -| Trip background | `#ffcdd2` | Light red fill for trip label boxes | -| LMP / annotation stroke | `#ff8f00` | Amber for LMP boxes and PTDF labels | -| LMP background | `#fff8e1` | Light amber fill for LMP annotation boxes | - -### Neutral Colors - -| Role | Hex | Usage | -|------|-----|-------| -| Primary text / bus labels | `#333333` | Bus label text and annotation body text | -| Secondary text / legend | `#666666` | Annotation and legend caption text | -| Tertiary text | `#999999` | Subtle secondary captions | -| Meter background | `#e0e0e0` | Gray fill for unfilled meter bars | - -### Special Values - -| Value | Usage | -|-------|-------| -| `none` | Explicit "no fill" on bus circles and load arrows | - ---- - -## Symbol Vocabulary - -Each element type uses a consistent visual shape across all six diagrams. - -| Element | Shape | Notes | -|---------|-------|-------| -| **Bus** | Circle (unfilled, stroked `#333`) | Bus label (B1, B2, ...) centered inside | -| **Generator** | Circle with tilde (`~`) inside | Connected to bus by a horizontal line | -| **Load** | Downward-pointing triangle (polygon) | Connected to bus by a horizontal line, with a vertical stub above | -| **Transmission line** | Solid `` element | Color depends on state (normal, congested, tripped) | -| **Flow arrow** | Filled `` arrowhead | Placed on transmission lines to indicate direction | -| **Congestion marker** | Filled `` bar across congested line | Orange fill with binding-limit label | -| **Trip marker** | Two crossed `` elements forming an X | Dark red, placed at midpoint of tripped line | -| **Annotation box** | `` with rounded corners (`rx`) | Color-coded by element type (green/red/amber/blue) | -| **Legend / caption** | `` element(s) near bottom of diagram | Uses secondary text color `#666` | - ---- - -## Sizing Conventions - -| Property | Value | Notes | -|----------|-------|-------| -| Bus circle radius | 25 px | `r="25"` (diameter = 50 px) | -| Generator circle radius | 25--30 px | Slightly variable; 25 px in later stages, 30 px in Stage 1 | -| Line stroke width (normal) | 2 px | Highlighted (new) transmission lines | -| Line stroke width (dimmed) | 1 px | Prior-stage elements | -| Line stroke width (congested) | 3 px | Congested line in Stage 5 | -| Trip line stroke width | 2.5 px | Tripped line in Stage 6 | -| X-marker stroke width | 3 px | Trip X marker in Stage 6 | -| Label font size | 14 px | Bus labels (B1, B2, ...) | -| Generator label font size | 12--14 px | Generator name labels (G1, G2) | -| Tilde font size | 18--20 px | Generator tilde symbol | -| Annotation font size | 10--12 px | Dispatch values, LMP, captions | -| Font family | `Arial, sans-serif` | System-safe stack; no external fonts | -| viewBox | `0 0 800 500` | All diagrams use identical 800x500 landscape viewBox | - ---- - -## Cumulative Layering Rules - -The diagrams form a progressive sequence. Each stage highlights its new elements -at full opacity while dimming elements carried forward from prior stages. - -### Opacity and Stroke Treatment - -| Layer | Opacity | Stroke Width | Description | -|-------|---------|-------------|-------------| -| **Highlighted** (new) | `1.0` | Normal (2 px) or bold (3 px for congestion) | New elements introduced at this stage | -| **Dimmed** (prior) | `0.35` | Reduced (1 px) | Elements from earlier stages | - -### Stage-Specific Rules - -| Stage | Treatment | -|-------|-----------| -| **Stage 1** | All elements at full opacity (no dimming -- everything is new) | -| **Stage 2** | Stage 1 bus + generator dimmed; new bus, line, load, flow arrow highlighted | -| **Stage 3** | Stages 1-2 elements dimmed; new buses, generators, lines, PTDF labels highlighted | -| **Stage 4** | All topology dimmed; OPF annotations (dispatch boxes, LMP boxes) highlighted | -| **Stage 5** | All topology dimmed; congestion line, thermal bars, LMP decomposition highlighted | -| **Stage 6** | All topology dimmed; tripped line, X marker, re-dispatch boxes, N-1 box highlighted | - -### Dimming Implementation - -Dimmed elements use the `opacity` attribute on each individual SVG element: - -```xml - - - - - -``` - ---- - -## Legend Requirements - -| Stage | Legend | -|-------|--------| -| **Stage 1** | Caption text describing the single-bus concept | -| **Stages 2--6** | Caption text at the bottom of the diagram explaining the stage's key concept. Stages 4--6 additionally include in-diagram annotation boxes that serve as implicit legends for new visual elements (dispatch boxes, LMP boxes, thermal bars, trip markers). | - -All legend/caption text uses font size 12 px in color `#666666` and is -positioned near the bottom of the viewBox (y >= 430). - ---- - -## SVG Hygiene Rules - -1. **No raster images.** No `` elements with `data:` URIs or external - raster file references. -2. **No external dependencies.** No ``, `@import`, external `@font-face`, - or external `` references. -3. **File size limit.** Each SVG must be under 50 KB. -4. **Consistent viewBox.** All six diagrams use `viewBox="0 0 800 500"` - (landscape, 8:5 aspect ratio). -5. **Self-contained.** Each SVG must render correctly without any external - resources. diff --git a/report/docs/grid-primer.mdx b/report/docs/grid-primer.mdx deleted file mode 100644 index 1d517059..00000000 --- a/report/docs/grid-primer.mdx +++ /dev/null @@ -1,287 +0,0 @@ ---- -sidebar_position: 2 -title: "Grid Operations Primer" -description: "A practitioner-level walkthrough of power grid operations, building from a single bus to contingency analysis in six progressive stages." ---- - -import Placeholder from '@site/src/components/Placeholder'; - -## Introduction - -If you work in energy markets, you already understand that electricity prices emerge from the -physical interaction between generators, transmission lines, and load. But the modeling tools -evaluated in this report operate at a level of detail that can feel opaque without shared -vocabulary. This primer exists to bridge that gap: to give energy market practitioners the -physical intuition needed to read the evaluation results and understand what each tool is -actually doing when it solves a power flow or dispatches generation. - -This page is not a power systems textbook. It introduces exactly the concepts you need to -interpret the evaluation criteria, and no more. The target audience is someone who trades -energy, manages a portfolio, or builds analytics, someone comfortable with market mechanics -but not necessarily with the transmission physics underneath. If you already build power flow -models for a living, you can safely skip to the -[evaluation results](/results/expressiveness). - -The primer is organized as six progressive stages. Each stage adds one new element to the -grid, building cumulatively from the simplest possible system to full contingency analysis. -Along the way, you will see references to specific evaluation test IDs (like A-1 or A-3); -these connect each concept to the criteria we used to evaluate the six modeling tools. Gray placeholder boxes mark spots where future interactive demonstrations will let you manipulate -the grid directly. - -## Stage 1: Single Generator, Single Load - -Every power system, no matter how large, rests on a single inviolable constraint: at every -bus, in every instant, the power injected by generators must exactly equal the power consumed -by loads plus any losses. This is the power balance equation, and it is the foundation on -which everything else in this primer is built. - -![Stage 1 diagram showing a single bus with one generator and one load connected by power flow arrows](/img/grid-primer/stage-1_single-bus.svg) - -A bus is the fundamental node in a power system model, a point where equipment connects. In -the simplest possible grid, a single bus has one generator (a source of power) and one load -(a sink of power). There is no transmission network, no optimization, and no market. The only -question is whether supply matches demand. - -This deceptively simple setup already exposes a key difference between electricity and other -commodities: there is no inventory. Natural gas can be stored in pipelines; crude oil sits in -tanks. Electricity must be produced and consumed simultaneously. When a generator's output -does not match the load's demand, frequency deviates from its nominal value, a condition -that, if sustained, damages equipment and cascades into blackouts. Every modeling tool must -enforce this balance constraint, and how faithfully it does so is the most basic test of -correctness (this is what Expressiveness tests A-1 and A-2 verify). - -The bus concept also introduces the distinction between a physical substation and a model -node. Real substations have switchgear, transformers, and multiple voltage levels. In a -model, all of that collapses into a single bus with aggregate injection and withdrawal values. -The art of power system modeling is choosing the right level of abstraction: enough detail -to capture the physics that matter, without drowning in complexity that does not. - - - -With power balance established at a single bus, the natural next question is: what happens -when we need to move power from where it is generated to where it is consumed? - -## Stage 2: Two Buses Connected by a Line - -Adding a second bus and a transmission line between them introduces the physics that make -power systems fundamentally different from other network optimization problems. Power does not -flow like water through a pipe or data through a cable. It obeys Kirchhoff's laws, and its -behavior is governed by a property of the line called impedance, the combination of -resistance (which dissipates energy as heat) and reactance (which stores and releases energy -in electromagnetic fields). - -![Stage 2 diagram showing two buses connected by a transmission line with impedance label](/img/grid-primer/stage-2_two-bus.svg) - -When a generator at bus 1 pushes power toward a load at bus 2, the voltage angle at bus 1 -leads the angle at bus 2. The greater the angle difference, the more power flows. This -relationship is the core of power flow analysis: given the network topology and the injection -pattern (which buses generate, which consume), solve for the voltage angles and magnitudes at -every bus, and from those compute the flow on every line. - -Two formulations of this problem dominate the tools under evaluation. The AC power flow -(ACPF) solves the full nonlinear equations that govern real and reactive power, voltage -magnitudes, and angles. It is accurate but computationally expensive, requiring iterative -numerical methods like Newton-Raphson. The DC power flow (DCPF) linearizes the problem by -assuming voltage magnitudes are all 1.0 per unit, angle differences are small, and line -resistance is negligible compared to reactance. The result is a system of linear equations -that can be solved directly, fast enough for real-time market clearing and accurate enough for -most energy trading applications (this is what Expressiveness tests A-1 and A-2 measure: -A-1 for DCPF, A-2 for ACPF). - -The tradeoff between AC and DC formulations is not academic. Real-time operations often use a -DC approximation for speed, while planning studies use full AC models for accuracy. A modeling -tool that supports only one formulation limits the analyses you can perform. - -
-Mathematical formulation: DC power flow - -In the DC approximation, the power flow on a line from bus $i$ to bus $j$ reduces to: - -$$P_\{ij\} = \frac\{\theta_i - \theta_j\}\{x_\{ij\}\}$$ - -Where $P_\{ij\}$ is the real power flow on the line, $\theta_i$ and $\theta_j$ are the voltage -angles at buses $i$ and $j$ (in radians), and $x_\{ij\}$ is the line's reactance in per-unit. -This linear relationship is what makes DC power flow solvable as a matrix equation and is -the foundation of shift factors used in market clearing. - -
- - - -A two-bus system, while illustrative, has only one path for power to flow. Real grids are -meshed, and that changes everything. - -## Stage 3: Meshed Network - -When a third bus joins the network and multiple paths connect the buses, power flow becomes -genuinely non-trivial. Unlike a transportation network where a shipper chooses a route, power -splits across all available paths simultaneously according to the impedance of each path. You -cannot direct a megawatt to take the northern route instead of the southern one; physics -makes that choice for you. - -![Stage 3 diagram showing a meshed three-bus network with parallel paths and PTDF annotations](/img/grid-primer/stage-3_meshed-network.svg) - -This parallel-path behavior leads to one of power systems' most counterintuitive phenomena. -Adding a new transmission line to a congested network can actually increase the loading on -existing lines, a result known as Braess's paradox. The mechanism is straightforward once you -internalize the impedance-based flow distribution from Stage 2: a new line changes the -impedance ratios of all parallel paths, redistributing flow in ways that may worsen -bottlenecks even as total transfer capacity increases. - -To quantify how power redistributes, operators use Power Transfer Distribution Factors -(PTDFs). A PTDF answers a specific question: if one additional megawatt is injected at bus A -and withdrawn at bus B, what fraction of that megawatt flows on each line in the network? -PTDFs are the backbone of congestion management in organized markets. When an operator -calculates whether a proposed trade would violate a transmission constraint, they multiply the -trade quantity by the relevant PTDF to determine the trade's impact on each monitored line. - -For modeling tools, meshed networks are where the bus-branch formulation introduced in Stage 1 -proves its worth. The bus admittance matrix (a square matrix whose dimensions equal the -number of buses) encodes the entire network topology and impedance structure. Solving the -power flow on a meshed network is fundamentally the same matrix operation as on two buses, -just larger (this is what Expressiveness tests A-1 through A-3 validate across increasingly -complex topologies). - - - -So far, we have treated generators as fixed injections. In reality, someone must decide how -much each generator produces, and that decision is an optimization problem. - -## Stage 4: Economic Dispatch and OPF - -Given a meshed network with multiple generators, each having a different cost to produce -power, the operator must decide how much output to assign to each unit. The simplest version -of this problem, economic dispatch, stacks generators in order of their marginal cost (the -merit order) and dispatches them from cheapest to most expensive until total generation meets -total demand. This is the optimization that runs every five minutes in organized electricity -markets. - -![Stage 4 diagram showing a multi-bus network with generator cost curves and LMP labels at each bus](/img/grid-primer/stage-4_opf-dispatch.svg) - -But economic dispatch alone ignores the network. If the cheapest generator is behind a -congested transmission line, dispatching it at full output would overload that line. Optimal -Power Flow (OPF) combines economic dispatch with the power flow constraints from Stages 2 -and 3: minimize total generation cost subject to power balance at every bus, generator output -limits, and transmission line flow limits. The solution tells you not just how much each -generator produces, but also the shadow price of the power balance constraint at each bus. - -That shadow price is the Locational Marginal Price (LMP), the cost of serving one additional -megawatt of load at a specific bus. In an uncongested network, all LMPs are equal (every bus -sees the same marginal generator). When transmission constraints bind, LMPs diverge: buses -behind the congestion see higher prices because cheap remote generation cannot reach them. -LMPs are the price signals that drive energy trading, generator siting decisions, and -transmission investment. Understanding how they emerge from the OPF is essential for -interpreting the evaluation results (this is what Expressiveness tests A-3 and A-4 measure: -A-3 for DC-OPF, A-4 for AC-OPF). - -The DC versus AC distinction from Stage 2 reappears here. A DC-OPF uses the linearized power -flow equations, producing approximate LMPs quickly. An AC-OPF uses the full nonlinear -equations, capturing voltage magnitude effects and reactive power costs, but requires -significantly more computation. Most day-ahead markets use a form of DC-OPF; voltage-related -constraints are handled through supplemental studies. - -
-Mathematical formulation: OPF objective - -The optimal power flow minimizes total generation cost subject to network constraints: - -$$\min \sum_\{g \in \mathcal\{G\}\} C_g(P_g) \quad \text\{s.t.\} \quad \sum_\{g \in \mathcal\{G\}_n\} P_g - P_n^d = \sum_\{(n,m) \in \mathcal\{L\}\} P_\{nm\} \quad \forall n$$ - -Where $C_g(P_g)$ is the cost function of generator $g$, $P_g$ is its output, -$P_n^d$ is the demand at bus $n$, and $P_\{nm\}$ is the flow on line $(n, m)$. The constraint -enforces power balance at every bus. Additional constraints on generator limits -($P_g^\{\min\} \leq P_g \leq P_g^\{\max\}$) and line flow limits -($|P_\{nm\}| \leq P_\{nm\}^\{\max\}$) complete the formulation. The dual variable of each bus's -power balance constraint is that bus's LMP. - -
- - - -The OPF determines the dispatch and the prices. But what happens when a line is pushed to its -limit? - -## Stage 5: Transmission Limits and Congestion - -Every transmission line has a thermal rating, a maximum power flow determined by how much -current the conductor can carry before it heats to the point of sagging into trees or losing -tensile strength. When the OPF solution from Stage 4 pushes a line to this limit, the -constraint binds, and the network is congested. - -![Stage 5 diagram showing a network with one congested line highlighted in red and divergent LMP values at buses](/img/grid-primer/stage-5_congestion.svg) - -Congestion is not a failure. It is the normal operating state of a stressed grid, and it is -the mechanism that creates price separation between locations. Without congestion, the energy -market would need no locational component; a single system-wide price would suffice. It is -precisely because transmission capacity is scarce that LMPs differ across buses, creating the -arbitrage opportunities that define energy trading. - -The LMP at any bus can be decomposed into three components: the energy component (the -system-wide marginal cost of generation), the loss component (the marginal cost of -transmission losses to deliver power to that bus), and the congestion component (the marginal -cost of the binding transmission constraints that affect delivery to that bus). In DC-OPF -models, the loss component is typically zero because the DC approximation neglects resistance. -The congestion component is what traders care about most: it determines the spread between -source and sink prices that drives Congestion Revenue Rights (CRRs) and Point-to-Point (PTP) -trading strategies. - -For modeling tools, transmission limits test whether the solver can handle inequality -constraints that may or may not be active depending on the operating point. A tool that only -solves unconstrained power flow cannot produce LMPs and therefore cannot model the market -outcomes that matter to traders. Tools that solve the constrained OPF but do not expose the -LMP decomposition hide the economic signal in aggregate prices (this is what Expressiveness -tests A-5 and A-6 assess). - - - -In real operations, the grid must not only handle the current state; it must survive -unexpected equipment failures. That requirement is the subject of the final stage. - -## Stage 6: Contingency Analysis and SCOPF - -The power grid operates under a principle called N-1 security: the system must remain stable -and within limits if any single element (a generator, a transmission line, a transformer) -trips offline unexpectedly. This is not a theoretical concern. Equipment failures happen daily -on large grids, and the consequences of an unprepared system range from localized price spikes -to cascading blackouts. - -![Stage 6 diagram showing a network with a tripped line drawn as dashed and re-dispatch arrows indicating preventive action](/img/grid-primer/stage-6_scopf.svg) - -Contingency analysis evaluates every credible N-1 scenario: for each element that could fail, -re-solve the power flow with that element removed and check whether any remaining line exceeds -its emergency rating. If a contingency would cause a violation, the operator must take -preventive action, typically re-dispatching generation away from the pattern that Stage 4's -OPF would otherwise choose. This preventive re-dispatch costs money, because it moves -generation away from the economic optimum to maintain security. - -Security-Constrained Optimal Power Flow (SCOPF) formalizes this tradeoff. It solves the OPF -from Stage 4 while simultaneously enforcing the constraint that the solution must remain -feasible under every contingency. The result is a dispatch that is more expensive than the -unconstrained optimum but survives any single equipment failure. In organized markets, this -is exactly what the system operator solves every five minutes; the LMPs that traders see -already embed the cost of N-1 security. - -SCOPF is the most computationally demanding problem in this primer. A network with $L$ lines -has $L$ contingencies, and each contingency requires evaluating the full network power flow. -For a realistic grid with thousands of lines, the problem scales rapidly. This is where -modeling tool performance differences become stark: a tool that handles a five-bus SCOPF in -milliseconds may struggle or fail on a realistic network. The Scalability dimension of our -evaluation measures exactly this progression (this is what Expressiveness tests A-7 through -A-11 evaluate, covering unit commitment, SCUC, and security-constrained formulations). - - - -## From Primer to Evaluation - -You now have the conceptual vocabulary for every analysis type in our evaluation: from basic -power balance (Stage 1) through DC and AC power flow (Stage 2), meshed network flow -distribution (Stage 3), economic dispatch and OPF (Stage 4), congestion and LMP decomposition -(Stage 5), to contingency-constrained security analysis (Stage 6). Each stage maps directly -to one or more Expressiveness test cases that we ran against all six tools. - -The [Expressiveness criterion page](/results/expressiveness) presents the results of -those tests. When you see that a tool passes test A-3 (DC-OPF) but fails A-4 (AC-OPF), you -can now map that result back to the physics: the tool can optimize dispatch using the linear -DC approximation from Stage 2, but cannot handle the full nonlinear AC formulation. That -distinction matters when you need to model voltage-dependent phenomena or validate DC results -against a full AC baseline. diff --git a/report/scripts/smoke_test.py b/report/scripts/smoke_test.py index 15c4689e..508b8b73 100644 --- a/report/scripts/smoke_test.py +++ b/report/scripts/smoke_test.py @@ -65,11 +65,6 @@ class PageExpectation: description="Expressiveness Results", title_fragment="Expressiveness", ), - PageExpectation( - path="/grid-primer/", - description="Grid Operations Primer", - title_fragment="Grid Operations Primer", - ), PageExpectation( path="/tools-evaluated/", description="Tools Evaluated", diff --git a/report/sidebars.js b/report/sidebars.js index 15e48842..d8485d9d 100644 --- a/report/sidebars.js +++ b/report/sidebars.js @@ -2,7 +2,6 @@ const sidebars = { reportSidebar: [ 'index', - 'grid-primer', 'use-cases-criteria', 'tools-evaluated', 'contract-traceability', diff --git a/report/static/img/grid-primer/stage-1_single-bus.excalidraw b/report/static/img/grid-primer/stage-1_single-bus.excalidraw deleted file mode 100644 index b82cdf63..00000000 --- a/report/static/img/grid-primer/stage-1_single-bus.excalidraw +++ /dev/null @@ -1,41 +0,0 @@ -{ - "type": "excalidraw", - "version": 2, - "source": "hand-crafted", - "elements": [ - { - "type": "ellipse", - "id": "bus1", - "x": 375, - "y": 225, - "width": 50, - "height": 50, - "strokeColor": "#333333", - "label": { "text": "B1" } - }, - { - "type": "ellipse", - "id": "gen1", - "x": 180, - "y": 220, - "width": 60, - "height": 60, - "strokeColor": "#2e7d32", - "label": { "text": "G1 ~" } - }, - { - "type": "diamond", - "id": "load1", - "x": 550, - "y": 220, - "width": 40, - "height": 50, - "strokeColor": "#c62828", - "label": { "text": "L1" } - } - ], - "appState": { - "viewBackgroundColor": "#ffffff" - }, - "files": {} -} diff --git a/report/static/img/grid-primer/stage-1_single-bus.svg b/report/static/img/grid-primer/stage-1_single-bus.svg deleted file mode 100644 index d47d9f70..00000000 --- a/report/static/img/grid-primer/stage-1_single-bus.svg +++ /dev/null @@ -1,24 +0,0 @@ - - Stage 1: Single Generator, Single Load - A single bus connected to one generator and one load, illustrating the simplest possible power system: generation equals demand at a single node. - - - - B1 - - - - - ~ - G1 - - - - - - - L1 - - - P_gen = P_load (power balance at single node) - diff --git a/report/static/img/grid-primer/stage-2_two-bus.excalidraw b/report/static/img/grid-primer/stage-2_two-bus.excalidraw deleted file mode 100644 index bd8b1cc8..00000000 --- a/report/static/img/grid-primer/stage-2_two-bus.excalidraw +++ /dev/null @@ -1,62 +0,0 @@ -{ - "type": "excalidraw", - "version": 2, - "source": "hand-crafted", - "elements": [ - { - "type": "ellipse", - "id": "bus1", - "x": 175, - "y": 225, - "width": 50, - "height": 50, - "strokeColor": "#333333", - "opacity": 35, - "label": { "text": "B1" } - }, - { - "type": "ellipse", - "id": "gen1", - "x": 20, - "y": 220, - "width": 60, - "height": 60, - "strokeColor": "#2e7d32", - "opacity": 35, - "label": { "text": "G1 ~" } - }, - { - "type": "ellipse", - "id": "bus2", - "x": 575, - "y": 225, - "width": 50, - "height": 50, - "strokeColor": "#333333", - "label": { "text": "B2" } - }, - { - "type": "line", - "id": "line12", - "x": 225, - "y": 250, - "width": 350, - "height": 0, - "strokeColor": "#1565c0" - }, - { - "type": "diamond", - "id": "load1", - "x": 700, - "y": 220, - "width": 40, - "height": 50, - "strokeColor": "#c62828", - "label": { "text": "L1" } - } - ], - "appState": { - "viewBackgroundColor": "#ffffff" - }, - "files": {} -} diff --git a/report/static/img/grid-primer/stage-2_two-bus.svg b/report/static/img/grid-primer/stage-2_two-bus.svg deleted file mode 100644 index fbdca6f2..00000000 --- a/report/static/img/grid-primer/stage-2_two-bus.svg +++ /dev/null @@ -1,41 +0,0 @@ - - Stage 2: Two-Bus System with Transmission - Two buses connected by a transmission line, with a generator at bus 1 and a load at bus 2. Power flows from the generator bus to the load bus through the line. Elements from stage 1 are dimmed while the new bus, line, and flow arrows are highlighted. - - - - - - B1 - - - - - ~ - G1 - - - - - - - - - P flow → - - - Line 1-2 - - - - B2 - - - - - - L1 - - - Power flows from generation (B1) to load (B2) through transmission line - diff --git a/report/static/img/grid-primer/stage-3_meshed-network.excalidraw b/report/static/img/grid-primer/stage-3_meshed-network.excalidraw deleted file mode 100644 index f1c26708..00000000 --- a/report/static/img/grid-primer/stage-3_meshed-network.excalidraw +++ /dev/null @@ -1,81 +0,0 @@ -{ - "type": "excalidraw", - "version": 2, - "source": "hand-crafted", - "elements": [ - { - "type": "ellipse", - "id": "bus1", - "x": 175, - "y": 125, - "width": 50, - "height": 50, - "strokeColor": "#333333", - "opacity": 35, - "label": { "text": "B1" } - }, - { - "type": "ellipse", - "id": "bus2", - "x": 575, - "y": 125, - "width": 50, - "height": 50, - "strokeColor": "#333333", - "opacity": 35, - "label": { "text": "B2" } - }, - { - "type": "ellipse", - "id": "bus3", - "x": 175, - "y": 325, - "width": 50, - "height": 50, - "strokeColor": "#333333", - "label": { "text": "B3" } - }, - { - "type": "ellipse", - "id": "bus4", - "x": 575, - "y": 325, - "width": 50, - "height": 50, - "strokeColor": "#333333", - "label": { "text": "B4" } - }, - { - "type": "ellipse", - "id": "gen2", - "x": 20, - "y": 320, - "width": 60, - "height": 60, - "strokeColor": "#2e7d32", - "label": { "text": "G2 ~" } - }, - { - "type": "diamond", - "id": "load2", - "x": 700, - "y": 320, - "width": 40, - "height": 50, - "strokeColor": "#c62828", - "label": { "text": "L2" } - }, - { - "type": "text", - "id": "ptdf-label", - "x": 350, - "y": 130, - "text": "PTDF = 0.40", - "strokeColor": "#ff8f00" - } - ], - "appState": { - "viewBackgroundColor": "#ffffff" - }, - "files": {} -} diff --git a/report/static/img/grid-primer/stage-3_meshed-network.svg b/report/static/img/grid-primer/stage-3_meshed-network.svg deleted file mode 100644 index db9df1fd..00000000 --- a/report/static/img/grid-primer/stage-3_meshed-network.svg +++ /dev/null @@ -1,75 +0,0 @@ - - Stage 3: Meshed Network with Loop Topology - A four-bus meshed network with multiple transmission paths forming loops. Generators at buses 1 and 3, loads at buses 2 and 4. Power Transfer Distribution Factors (PTDFs) determine how flow distributes across parallel paths. Prior stage elements are dimmed; new buses, lines, and PTDF labels are highlighted. - - - - - - B1 - - - - - ~ - G1 - - - - B2 - - - - - - L1 - - - - - - - - - B3 - - - - - ~ - G2 - - - - B4 - - - - - - L2 - - - - Line 1-3 - - - - Line 2-4 - - - - Line 3-4 - - - - Line 1-4 - - - PTDF = 0.40 - PTDF = 0.35 - PTDF = 0.25 - - - Flow distributes across parallel paths per Power Transfer Distribution Factors - diff --git a/report/static/img/grid-primer/stage-4_opf-dispatch.excalidraw b/report/static/img/grid-primer/stage-4_opf-dispatch.excalidraw deleted file mode 100644 index 3970494b..00000000 --- a/report/static/img/grid-primer/stage-4_opf-dispatch.excalidraw +++ /dev/null @@ -1,87 +0,0 @@ -{ - "type": "excalidraw", - "version": 2, - "source": "hand-crafted", - "elements": [ - { - "type": "ellipse", - "id": "bus1", - "x": 175, - "y": 125, - "width": 50, - "height": 50, - "strokeColor": "#333333", - "opacity": 35, - "label": { "text": "B1" } - }, - { - "type": "ellipse", - "id": "bus2", - "x": 575, - "y": 125, - "width": 50, - "height": 50, - "strokeColor": "#333333", - "opacity": 35, - "label": { "text": "B2" } - }, - { - "type": "ellipse", - "id": "bus3", - "x": 175, - "y": 325, - "width": 50, - "height": 50, - "strokeColor": "#333333", - "opacity": 35, - "label": { "text": "B3" } - }, - { - "type": "ellipse", - "id": "bus4", - "x": 575, - "y": 325, - "width": 50, - "height": 50, - "strokeColor": "#333333", - "opacity": 35, - "label": { "text": "B4" } - }, - { - "type": "text", - "id": "g1-dispatch", - "x": 20, - "y": 100, - "text": "G1: 180 MW @ $25/MWh", - "strokeColor": "#2e7d32" - }, - { - "type": "text", - "id": "g2-dispatch", - "x": 20, - "y": 300, - "text": "G2: 120 MW @ $35/MWh", - "strokeColor": "#2e7d32" - }, - { - "type": "text", - "id": "lmp-b1", - "x": 175, - "y": 90, - "text": "LMP $25", - "strokeColor": "#ff8f00" - }, - { - "type": "text", - "id": "lmp-b2", - "x": 575, - "y": 90, - "text": "LMP $30", - "strokeColor": "#ff8f00" - } - ], - "appState": { - "viewBackgroundColor": "#ffffff" - }, - "files": {} -} diff --git a/report/static/img/grid-primer/stage-4_opf-dispatch.svg b/report/static/img/grid-primer/stage-4_opf-dispatch.svg deleted file mode 100644 index 7c262b56..00000000 --- a/report/static/img/grid-primer/stage-4_opf-dispatch.svg +++ /dev/null @@ -1,91 +0,0 @@ - - Stage 4: Optimal Power Flow Dispatch - The same four-bus meshed network with cost curve annotations on generators, MW dispatch values, and Locational Marginal Price (LMP) values at each bus. The network topology is dimmed while economic dispatch annotations are highlighted. - - - - - - B1 - - - - - ~ - - - - B2 - - - - - - - - - B3 - - - - - ~ - - - - B4 - - - - - - - - - - - - - - - - - - G1: 180 MW - $25/MWh - - - - G2: 120 MW - $35/MWh - - - - 200 MW - - - - 100 MW - - - - $25 - - - - $30 - - - - $35 - - - - $32 - - - LMP ($/MWh) - - - Cheapest generator dispatched first; LMPs reflect marginal cost at each bus - diff --git a/report/static/img/grid-primer/stage-5_congestion.excalidraw b/report/static/img/grid-primer/stage-5_congestion.excalidraw deleted file mode 100644 index 8164fc38..00000000 --- a/report/static/img/grid-primer/stage-5_congestion.excalidraw +++ /dev/null @@ -1,67 +0,0 @@ -{ - "type": "excalidraw", - "version": 2, - "source": "hand-crafted", - "elements": [ - { - "type": "ellipse", - "id": "bus1", - "x": 175, - "y": 125, - "width": 50, - "height": 50, - "strokeColor": "#333333", - "opacity": 35, - "label": { "text": "B1" } - }, - { - "type": "ellipse", - "id": "bus2", - "x": 575, - "y": 125, - "width": 50, - "height": 50, - "strokeColor": "#333333", - "opacity": 35, - "label": { "text": "B2" } - }, - { - "type": "line", - "id": "line12-congested", - "x": 225, - "y": 150, - "width": 350, - "height": 0, - "strokeColor": "#e65100", - "strokeWidth": 3 - }, - { - "type": "text", - "id": "binding-label", - "x": 330, - "y": 120, - "text": "100/100 MW (BINDING)", - "strokeColor": "#e65100" - }, - { - "type": "text", - "id": "lmp-decomp-b1", - "x": 145, - "y": 60, - "text": "LMP=$25\nSMEC=$30\nMCC=-$5", - "strokeColor": "#ff8f00" - }, - { - "type": "text", - "id": "lmp-decomp-b2", - "x": 545, - "y": 60, - "text": "LMP=$45\nSMEC=$30\nMCC=+$15", - "strokeColor": "#ff8f00" - } - ], - "appState": { - "viewBackgroundColor": "#ffffff" - }, - "files": {} -} diff --git a/report/static/img/grid-primer/stage-5_congestion.svg b/report/static/img/grid-primer/stage-5_congestion.svg deleted file mode 100644 index 9f85f9ce..00000000 --- a/report/static/img/grid-primer/stage-5_congestion.svg +++ /dev/null @@ -1,88 +0,0 @@ - - Stage 5: Transmission Congestion and LMP Decomposition - The four-bus network showing congested transmission lines with thermal limit indicators. Line 1-2 is at its thermal limit (binding constraint), causing LMP separation between buses. LMPs are decomposed into System Marginal Energy Cost (SMEC) and Marginal Congestion Cost (MCC) components. - - - - - - B1 - - - - - ~ - - - - B2 - - - - - - - - - B3 - - - - - ~ - - - - B4 - - - - - - - - - - - - - - - - - - - - - 100/100 MW (BINDING) - - - - - 60/150 MW - - - - LMP = $25 - SMEC = $30 - MCC = -$5 - - - - LMP = $45 - SMEC = $30 - MCC = +$15 - - - - - - - - - Congestion rent = (LMP2 - LMP1) x flow - - - Binding line creates LMP spread: LMP = SMEC + MCC (+ MLC) - SMEC = System Marginal Energy Cost, MCC = Marginal Congestion Cost - diff --git a/report/static/img/grid-primer/stage-6_scopf.excalidraw b/report/static/img/grid-primer/stage-6_scopf.excalidraw deleted file mode 100644 index be086a4b..00000000 --- a/report/static/img/grid-primer/stage-6_scopf.excalidraw +++ /dev/null @@ -1,78 +0,0 @@ -{ - "type": "excalidraw", - "version": 2, - "source": "hand-crafted", - "elements": [ - { - "type": "ellipse", - "id": "bus1", - "x": 175, - "y": 125, - "width": 50, - "height": 50, - "strokeColor": "#333333", - "opacity": 35, - "label": { "text": "B1" } - }, - { - "type": "ellipse", - "id": "bus2", - "x": 575, - "y": 125, - "width": 50, - "height": 50, - "strokeColor": "#333333", - "opacity": 35, - "label": { "text": "B2" } - }, - { - "type": "line", - "id": "line12-tripped", - "x": 225, - "y": 150, - "width": 350, - "height": 0, - "strokeColor": "#b71c1c", - "strokeStyle": "dashed" - }, - { - "type": "text", - "id": "tripped-label", - "x": 365, - "y": 170, - "text": "TRIPPED", - "strokeColor": "#b71c1c" - }, - { - "type": "text", - "id": "g1-redispatch", - "x": 20, - "y": 100, - "text": "G1: 180 → 140 MW", - "strokeColor": "#b71c1c" - }, - { - "type": "text", - "id": "g2-redispatch", - "x": 20, - "y": 300, - "text": "G2: 120 → 160 MW", - "strokeColor": "#2e7d32" - }, - { - "type": "rectangle", - "id": "n1-box", - "x": 540, - "y": 400, - "width": 230, - "height": 60, - "strokeColor": "#1565c0", - "backgroundColor": "#e3f2fd", - "label": { "text": "N-1 Security Criterion" } - } - ], - "appState": { - "viewBackgroundColor": "#ffffff" - }, - "files": {} -} diff --git a/report/static/img/grid-primer/stage-6_scopf.svg b/report/static/img/grid-primer/stage-6_scopf.svg deleted file mode 100644 index b5a2deb8..00000000 --- a/report/static/img/grid-primer/stage-6_scopf.svg +++ /dev/null @@ -1,105 +0,0 @@ - - Stage 6: Security-Constrained OPF (SCOPF) with N-1 Contingency - The four-bus network under a contingency scenario. Line 1-2 is tripped (shown as dashed with an X), forcing power to reroute through remaining paths. Generators re-dispatch to maintain reliability under the N-1 criterion. Prior elements are dimmed while contingency indicators, re-dispatch arrows, and N-1 annotations are highlighted. - - - - - - B1 - - - - - ~ - - - - B2 - - - - - - - - - B3 - - - - - ~ - - - - B4 - - - - - - - - - - - - - - - - - - - - - - - - TRIPPED - - - - - - - - - - - - - - - G1: 180 → 140 MW - ▼ -40 MW - - - - G2: 120 → 160 MW - ▲ +40 MW - - - - - ↑ flow - - - - → rerouted flow - - - - ↑ flow - - - - N-1 Security Criterion - System must survive loss of - any single element - - - SCOPF pre-positions dispatch so no - single outage overloads the network - diff --git a/report/tests/test_grid_primer_prose.py b/report/tests/test_grid_primer_prose.py deleted file mode 100644 index 1a8df354..00000000 --- a/report/tests/test_grid_primer_prose.py +++ /dev/null @@ -1,427 +0,0 @@ -"""Tests for PRD 04/03 — Grid Primer MDX Prose (SC-01 through SC-18).""" - -from __future__ import annotations - -import re -from pathlib import Path - -import pytest - -REPORT_DIR = Path(__file__).resolve().parent.parent -MDX_PATH = REPORT_DIR / "docs" / "grid-primer.mdx" -IMG_DIR = REPORT_DIR / "static" / "img" / "grid-primer" - -STAGE_HEADINGS = [ - "## Stage 1: Single Generator, Single Load", - "## Stage 2: Two Buses Connected by a Line", - "## Stage 3: Meshed Network", - "## Stage 4: Economic Dispatch and OPF", - "## Stage 5: Transmission Limits and Congestion", - "## Stage 6: Contingency Analysis and SCOPF", -] - -PLACEHOLDER_TITLES = [ - "Set generator output and load demand; see real-time power balance", - "Adjust line impedance and see how power flow changes", - "Change injection at one bus and watch power redistribute across parallel paths", - "Set generator costs and load level; see the optimal dispatch and resulting LMPs", - "Toggle line limits on and off; watch LMPs diverge as congestion binds", - "Trip a line and see how preventive SCOPF re-dispatches to maintain N-1 security", -] - -SVG_SLUGS = [ - "stage-1_single-bus", - "stage-2_two-bus", - "stage-3_meshed-network", - "stage-4_opf-dispatch", - "stage-5_congestion", - "stage-6_scopf", -] - - -@pytest.fixture(scope="module") -def mdx_text() -> str: - return MDX_PATH.read_text(encoding="utf-8") - - -@pytest.fixture(scope="module") -def mdx_body(mdx_text: str) -> str: - """MDX content without frontmatter.""" - parts = mdx_text.split("---", 2) - assert len(parts) >= 3, "MDX file must have YAML frontmatter delimited by ---" - return parts[2] - - -@pytest.fixture(scope="module") -def stage_sections(mdx_body: str) -> list[str]: - """Extract the six stage sections from the MDX body.""" - pattern = r"(## Stage \d:.*?)(?=## (?:Stage \d|From Primer)|\Z)" - sections = re.findall(pattern, mdx_body, re.DOTALL) - return sections - - -def _count_prose_words(text: str) -> int: - """Count words in prose, excluding frontmatter, MDX/JSX tags, code blocks, and math.""" - # Remove frontmatter - text = re.sub(r"^---.*?---", "", text, count=1, flags=re.DOTALL) - # Remove import lines - text = re.sub(r"^import .*$", "", text, flags=re.MULTILINE) - # Remove JSX comments - text = re.sub(r"\{/\*.*?\*/\}", "", text, flags=re.DOTALL) - # Remove code blocks - text = re.sub(r"```.*?```", "", text, flags=re.DOTALL) - # Remove display math - text = re.sub(r"\$\$.*?\$\$", "", text, flags=re.DOTALL) - # Remove inline math - text = re.sub(r"\$[^$]+?\$", "", text) - # Remove JSX/HTML tags but keep text content - text = re.sub(r"<[^>]+>", "", text) - # Remove image references - text = re.sub(r"!\[.*?\]\(.*?\)", "", text) - # Remove markdown heading markers - text = re.sub(r"^#{1,6}\s+", "", text, flags=re.MULTILINE) - return len(text.split()) - - -def _get_section_between(body: str, start_heading: str, end_pattern: str) -> str: - """Get text between a heading and the next matching pattern.""" - start_idx = body.find(start_heading) - if start_idx == -1: - return "" - after_start = body[start_idx + len(start_heading) :] - end_match = re.search(end_pattern, after_start, re.MULTILINE) - if end_match: - return after_start[: end_match.start()] - return after_start - - -# --- SC-01: Word count in range (2000-3000) --- - - -def test_sc01_word_count_in_range(mdx_text: str) -> None: - """SC-01: Total prose word count is between 2000 and 3000.""" - count = _count_prose_words(mdx_text) - assert 2000 <= count <= 3000, f"Word count {count} not in [2000, 3000]" - - -# --- SC-02: Introduction present --- - - -def test_sc02_introduction_present(mdx_body: str) -> None: - """SC-02: Introduction has 2-3 paragraphs mentioning audience, evaluation, structure.""" - intro = _get_section_between(mdx_body, "## Introduction", r"^## Stage 1") - # Extract non-empty paragraphs (blocks of text separated by blank lines) - paragraphs = [p.strip() for p in re.split(r"\n\s*\n", intro) if p.strip()] - assert 2 <= len(paragraphs) <= 3, ( - f"Intro has {len(paragraphs)} paragraphs, expected 2-3" - ) - intro_lower = intro.lower() - assert any( - w in intro_lower for w in ["practitioner", "trader", "portfolio", "audience"] - ), "Introduction must mention the target audience" - assert any(w in intro_lower for w in ["evaluation", "evaluate", "criteria"]), ( - "Introduction must mention the evaluation context" - ) - assert any( - w in intro_lower for w in ["cumulative", "progressive", "stage", "build"] - ), "Introduction must mention the cumulative structure" - - -# --- SC-03: All six stages populated --- - - -def test_sc03_all_six_stages_populated(stage_sections: list[str]) -> None: - """SC-03: Each of the six stages contains at least 2 paragraphs of explanation.""" - assert len(stage_sections) == 6, f"Expected 6 stages, found {len(stage_sections)}" - for i, section in enumerate(stage_sections, 1): - # Remove image refs, placeholders, details blocks, headings - prose = re.sub(r"!\[.*?\]\(.*?\)", "", section) - prose = re.sub(r"\s*", "", prose) - prose = re.sub(r"
.*?
", "", prose, flags=re.DOTALL) - prose = re.sub(r"^## .*$", "", prose, flags=re.MULTILINE) - paragraphs = [p.strip() for p in re.split(r"\n\s*\n", prose) if p.strip()] - assert len(paragraphs) >= 2, ( - f"Stage {i} has {len(paragraphs)} paragraphs, expected at least 2" - ) - - -# --- SC-04: SVG diagram references correct --- - - -def test_sc04_svg_diagram_references(stage_sections: list[str]) -> None: - """SC-04: Each stage has an image reference to the correct SVG path that exists on disk.""" - for i, (section, slug) in enumerate(zip(stage_sections, SVG_SLUGS), 1): - expected_path = f"/img/grid-primer/{slug}.svg" - assert expected_path in section, ( - f"Stage {i} missing image ref to {expected_path}" - ) - fs_path = IMG_DIR / f"{slug}.svg" - assert fs_path.exists(), f"SVG file not found: {fs_path}" - - -# --- SC-05: Placeholder components present --- - - -def test_sc05_placeholder_components(stage_sections: list[str]) -> None: - """SC-05: Each stage has exactly one Placeholder with the correct title.""" - for i, (section, title) in enumerate(zip(stage_sections, PLACEHOLDER_TITLES), 1): - placeholders = re.findall(r"", section) - assert len(placeholders) == 1, ( - f"Stage {i}: expected 1 Placeholder, found {len(placeholders)}" - ) - assert placeholders[0] == title, ( - f"Stage {i}: Placeholder title mismatch: {placeholders[0]!r} != {title!r}" - ) - - -# --- SC-06: Placeholder import present --- - - -def test_sc06_placeholder_import(mdx_text: str) -> None: - """SC-06: The MDX file imports the Placeholder component.""" - assert re.search(r"^import\s+Placeholder\s+from\s+", mdx_text, re.MULTILINE), ( - "Missing Placeholder import" - ) - - -# --- SC-07: KaTeX equations present --- - - -def test_sc07_katex_equations_present(mdx_body: str) -> None: - """SC-07: The page contains 1-2 KaTeX equations (display math $$...$$).""" - display_eqs = re.findall(r"\$\$.*?\$\$", mdx_body, re.DOTALL) - assert 1 <= len(display_eqs) <= 2, ( - f"Expected 1-2 display equations, found {len(display_eqs)}" - ) - - -# --- SC-08: Equations in collapsible sections --- - - -def test_sc08_equations_in_details(mdx_body: str) -> None: - """SC-08: Every display KaTeX equation is inside a
block with a .""" - # Find all details blocks - details_blocks = re.findall(r"
.*?
", mdx_body, re.DOTALL) - # Find all display equations - all_eqs = re.findall(r"\$\$.*?\$\$", mdx_body, re.DOTALL) - assert len(all_eqs) > 0, "No display equations found" - # Every equation must be inside a details block - for eq in all_eqs: - in_details = any(eq in block for block in details_blocks) - assert in_details, f"Display equation not inside
: {eq[:60]}..." - # Each details block with equations must have a summary - for block in details_blocks: - if "$$" in block: - assert "" in block, "Details block with equation missing " - - -# --- SC-09: Rubric cross-references present --- - - -def test_sc09_rubric_crossrefs(stage_sections: list[str]) -> None: - """SC-09: Each stage has at least one parenthetical rubric cross-reference (A-N pattern).""" - for i, section in enumerate(stage_sections, 1): - assert re.search(r"A-\d+", section), ( - f"Stage {i} missing rubric cross-reference (A-N pattern)" - ) - - -# --- SC-10: Bridge section present --- - - -def test_sc10_bridge_section(mdx_body: str) -> None: - """SC-10: A bridge section after Stage 6 links to the Expressiveness criterion page.""" - bridge_match = re.search(r"## From Primer to Evaluation", mdx_body) - assert bridge_match, "Missing 'From Primer to Evaluation' bridge section" - bridge_text = mdx_body[bridge_match.start() :] - assert ( - "/docs/criteria/expressiveness" in bridge_text - or "expressiveness" in bridge_text.lower() - ) - # Must contain a markdown link - assert re.search(r"\[.*?\]\(.*?expressiveness.*?\)", bridge_text), ( - "Bridge section must contain a link to the Expressiveness criterion page" - ) - - -# --- SC-11: Cumulative prose --- - - -def test_sc11_cumulative_prose(stage_sections: list[str]) -> None: - """SC-11: Stages 2-6 reference concepts from earlier stages.""" - # Stage 2+ should reference concepts from Stage 1 (bus, power balance, generator, load) - for i, section in enumerate(stage_sections[1:], 2): - section_lower = section.lower() - has_back_ref = any( - term in section_lower - for term in [ - "bus", - "power balance", - "impedance", - "lmp", - "opf", - "stage", - "earlier", - "from stage", - "generator", - "congestion", - ] - ) - assert has_back_ref, ( - f"Stage {i} does not reference concepts from earlier stages" - ) - - -# --- SC-12: Valid frontmatter --- - - -def test_sc12_valid_frontmatter(mdx_text: str) -> None: - """SC-12: Frontmatter has title, sidebar_position: 2, and non-empty description.""" - fm_match = re.match(r"^---\s*\n(.*?)\n---", mdx_text, re.DOTALL) - assert fm_match, "Missing frontmatter" - fm = fm_match.group(1) - assert 'title: "Grid Operations Primer"' in fm - assert "sidebar_position: 2" in fm - desc_match = re.search(r'description:\s*"(.+?)"', fm) - assert desc_match and len(desc_match.group(1).strip()) > 0, ( - "Frontmatter must have a non-empty description" - ) - - -# --- SC-13: Practitioner tone --- - - -def test_sc13_practitioner_tone(mdx_body: str) -> None: - """SC-13: Technical terms are defined on first use with plain-language explanation.""" - body_lower = mdx_body.lower() - # Key terms that should be accompanied by explanation - terms_with_definitions = { - "impedance": ["resistance", "reactance"], - "lmp": ["locational marginal price", "shadow price", "cost"], - "ptdf": ["power transfer distribution factor", "fraction", "how"], - "scopf": ["security-constrained", "contingency"], - "n-1": ["single element", "any single", "loss of"], - } - for term, explanations in terms_with_definitions.items(): - if term in body_lower: - has_explanation = any(exp in body_lower for exp in explanations) - assert has_explanation, ( - f"Term '{term}' used without plain-language explanation" - ) - - -# --- SC-14: No broken MDX syntax --- - - -def test_sc14_no_broken_mdx_syntax(mdx_text: str) -> None: - """SC-14: Basic MDX syntax checks — balanced tags, valid image refs, imports.""" - # All
are closed - opens = len(re.findall(r"
", mdx_text)) - closes = len(re.findall(r"
", mdx_text)) - assert opens == closes, f"Unbalanced
tags: {opens} opens, {closes} closes" - - # All are closed - opens = len(re.findall(r"", mdx_text)) - closes = len(re.findall(r"", mdx_text)) - assert opens == closes, f"Unbalanced tags: {opens} opens, {closes} closes" - - # Placeholder tags are self-closing - assert not re.search(r"", mdx_text), ( - "Placeholder should be self-closing" - ) - - # Image references use valid MDX syntax - for img in re.finditer(r"!\[.*?\]\((.*?)\)", mdx_text): - path = img.group(1) - assert path.startswith("/img/"), f"Image path should be absolute: {path}" - - -# --- SC-15: Transition sentences --- - - -def test_sc15_transition_sentences(stage_sections: list[str]) -> None: - """SC-15: Stages 1-5 end with a transition; Stage 6 does not have a forward transition.""" - for i, section in enumerate(stage_sections[:5], 1): - # Get text after the last Placeholder - after_placeholder = section.rsplit("", 1) - assert len(trailing) == 2, f"Stage {i}: Malformed Placeholder" - transition_text = trailing[1].strip() - assert len(transition_text) > 10, ( - f"Stage {i}: Missing transition sentence after Placeholder" - ) - - # Stage 6 should NOT have significant text after Placeholder - stage6 = stage_sections[5] - after_ph = stage6.rsplit("", 1) - if len(trailing) == 2: - remaining = trailing[1].strip() - # Should have no substantial transition text - assert len(remaining) < 50, ( - f"Stage 6 should not have a forward transition, found: {remaining[:80]!r}" - ) - - -# --- SC-16: Image alt text --- - - -def test_sc16_image_alt_text(mdx_body: str) -> None: - """SC-16: Every SVG image reference includes descriptive alt text.""" - images = re.findall(r"!\[(.*?)\]\(.*?\)", mdx_body) - assert len(images) == 6, f"Expected 6 image references, found {len(images)}" - for i, alt in enumerate(images, 1): - assert len(alt) >= 10, f"Image {i} has insufficient alt text: {alt!r}" - - -# --- SC-17: Stage headings match Deliverable 1 --- - - -def test_sc17_stage_headings(mdx_body: str) -> None: - """SC-17: The six H2 headings match the Deliverable 1 specification.""" - for heading in STAGE_HEADINGS: - assert heading in mdx_body, f"Missing expected heading: {heading}" - - -# --- SC-18: No orphaned concepts --- - - -def test_sc18_no_orphaned_concepts(mdx_body: str) -> None: - """SC-18: Terms referenced in later stages are defined where first introduced.""" - body_lower = mdx_body.lower() - - # Map of terms to the stage section where they should first appear - # (using heading text as anchor) - term_first_stage = { - "power balance": "stage 1", - "impedance": "stage 2", - "ptdf": "stage 3", - "lmp": "stage 4", - "n-1": "stage 6", - } - - stage_positions = {} - for n in range(1, 7): - marker = f"## stage {n}:" - pos = body_lower.find(marker) - if pos >= 0: - stage_positions[f"stage {n}"] = pos - - for term, expected_stage in term_first_stage.items(): - first_use = body_lower.find(term) - if first_use == -1: - continue # Term not used at all — not an error for this test - stage_start = stage_positions.get(expected_stage, 0) - # Allow the term to appear at or after its expected stage start - # (or in the introduction if it's a general term) - intro_pos = body_lower.find("## introduction") - assert first_use >= intro_pos, ( - f"Term '{term}' appears before the Introduction section" - ) - # Term should not appear before its defining stage (unless in intro) - if first_use > intro_pos + len("## introduction"): - assert first_use >= stage_start, ( - f"Term '{term}' first used before {expected_stage}" - )