diff --git a/tests/skill/understand/test_merge_batch_graphs.py b/tests/skill/understand/test_merge_batch_graphs.py index 38f3fd28c..a0efcb5ba 100644 --- a/tests/skill/understand/test_merge_batch_graphs.py +++ b/tests/skill/understand/test_merge_batch_graphs.py @@ -60,6 +60,433 @@ def _file_node(path: str, **extra: Any) -> dict[str, Any]: return node +def _function_node(path: str, name: str, **extra: Any) -> dict[str, Any]: + """Build a minimal normalized function node.""" + node: dict[str, Any] = { + "id": f"function:{path}:{name}", + "type": "function", + "name": name, + "filePath": path, + "summary": "", + "tags": [], + "complexity": "simple", + } + node.update(extra) + return node + + +def _extraction_record( + path: str, + caller: str, + callee: str, + imports: list[dict[str, Any]] | None = None, + call_line: int = 10, +) -> dict[str, Any]: + """Build a minimal Python extraction record containing one call.""" + return { + "path": path, + "language": "python", + "callGraph": [{"caller": caller, "callee": callee, "lineNumber": call_line}], + "imports": imports or [], + } + + +# ── deterministic Python call linker ───────────────────────────────────── + +class PythonCallLinkerTests(unittest.TestCase): + def test_links_same_file_call(self) -> None: + nodes = { + "function:service.py:run": _function_node("service.py", "run"), + "function:service.py:expand": _function_node("service.py", "expand"), + } + extracted = [_extraction_record("service.py", "run", "expand")] + edges: list[dict[str, Any]] = [] + + added, skipped = mbg.link_python_calls(nodes, edges, extracted) + + self.assertEqual((added, skipped), (1, 0)) + self.assertEqual(edges, [{ + "source": "function:service.py:run", + "target": "function:service.py:expand", + "type": "calls", + "direction": "forward", + "weight": 0.8, + "description": "Python call graph (deterministic)", + }]) + + def test_links_direct_import(self) -> None: + nodes = { + "function:service.py:run": _function_node("service.py", "run"), + "function:package/helper.py:expand": _function_node( + "package/helper.py", "expand" + ), + } + extracted = [_extraction_record( + "service.py", + "run", + "expand", + [{"source": "package.helper", "specifiers": ["expand"], "lineNumber": 1}], + )] + edges: list[dict[str, Any]] = [] + + self.assertEqual(mbg.link_python_calls(nodes, edges, extracted), (1, 0)) + self.assertEqual(edges[0]["target"], "function:package/helper.py:expand") + + def test_links_aliased_direct_import(self) -> None: + nodes = { + "function:service.py:run": _function_node("service.py", "run"), + "function:package/helper.py:expand": _function_node( + "package/helper.py", "expand" + ), + } + extracted = [_extraction_record( + "service.py", + "run", + "fill", + [{ + "source": "package.helper", + "specifiers": ["fill"], + "aliases": {"fill": "expand"}, + "lineNumber": 1, + }], + )] + edges: list[dict[str, Any]] = [] + + self.assertEqual(mbg.link_python_calls(nodes, edges, extracted), (1, 0)) + self.assertEqual(edges[0]["target"], "function:package/helper.py:expand") + + def test_links_module_qualified_import(self) -> None: + nodes = { + "function:service.py:run": _function_node("service.py", "run"), + "function:package/helper.py:expand": _function_node( + "package/helper.py", "expand" + ), + } + extracted = [_extraction_record( + "service.py", + "run", + "helper.expand", + [{ + "source": "package.helper", + "specifiers": ["helper"], + "aliases": {"helper": "package.helper"}, + "lineNumber": 1, + }], + )] + edges: list[dict[str, Any]] = [] + + self.assertEqual(mbg.link_python_calls(nodes, edges, extracted), (1, 0)) + self.assertEqual(edges[0]["target"], "function:package/helper.py:expand") + + def test_skips_ambiguous_function_targets(self) -> None: + nodes = { + "function:service.py:run": _function_node("service.py", "run"), + "function:package/helper.py:expand": _function_node( + "package/helper.py", "expand" + ), + "function:package/helper/__init__.py:expand": _function_node( + "package/helper/__init__.py", "expand" + ), + } + extracted = [_extraction_record( + "service.py", + "run", + "expand", + [{"source": "package.helper", "specifiers": ["expand"], "lineNumber": 1}], + )] + edges: list[dict[str, Any]] = [] + + self.assertEqual(mbg.link_python_calls(nodes, edges, extracted), (0, 1)) + self.assertEqual(edges, []) + + def test_skips_absent_caller_node(self) -> None: + nodes = { + "function:service.py:expand": _function_node("service.py", "expand"), + } + extracted = [_extraction_record("service.py", "run", "expand")] + edges: list[dict[str, Any]] = [] + + self.assertEqual(mbg.link_python_calls(nodes, edges, extracted), (0, 1)) + self.assertEqual(edges, []) + + def test_suppresses_duplicate_pre_existing_calls_edge(self) -> None: + nodes = { + "function:service.py:run": _function_node("service.py", "run"), + "function:service.py:expand": _function_node("service.py", "expand"), + } + extracted = [_extraction_record("service.py", "run", "expand")] + edges: list[dict[str, Any]] = [{ + "source": "function:service.py:run", + "target": "function:service.py:expand", + "type": "calls", + "direction": "forward", + "weight": 0.5, + }] + + self.assertEqual(mbg.link_python_calls(nodes, edges, extracted), (0, 0)) + self.assertEqual(len(edges), 1) + + def test_latest_import_binding_wins(self) -> None: + nodes = { + "function:service.py:run": _function_node("service.py", "run"), + "function:a.py:expand": _function_node("a.py", "expand"), + "function:b.py:expand": _function_node("b.py", "expand"), + } + extracted = [_extraction_record( + "service.py", + "run", + "expand", + [ + {"source": "a", "specifiers": ["expand"], "lineNumber": 1}, + {"source": "b", "specifiers": ["expand"], "lineNumber": 2}, + ], + )] + edges: list[dict[str, Any]] = [] + + self.assertEqual(mbg.link_python_calls(nodes, edges, extracted), (1, 0)) + self.assertEqual(edges[0]["target"], "function:b.py:expand") + + def test_does_not_link_shadowed_target_when_active_binding_is_absent(self) -> None: + nodes = { + "function:service.py:run": _function_node("service.py", "run"), + "function:a.py:expand": _function_node("a.py", "expand"), + } + extracted = [_extraction_record( + "service.py", + "run", + "expand", + [ + {"source": "a", "specifiers": ["expand"], "lineNumber": 1}, + {"source": "b", "specifiers": ["expand"], "lineNumber": 2}, + ], + )] + edges: list[dict[str, Any]] = [] + + self.assertEqual(mbg.link_python_calls(nodes, edges, extracted), (0, 1)) + self.assertEqual(edges, []) + + def test_import_after_call_does_not_rebind_call_site(self) -> None: + nodes = { + "function:service.py:run": _function_node("service.py", "run"), + "function:a.py:expand": _function_node("a.py", "expand"), + "function:b.py:expand": _function_node("b.py", "expand"), + } + extracted = [_extraction_record( + "service.py", + "run", + "expand", + [ + {"source": "a", "specifiers": ["expand"], "lineNumber": 1}, + {"source": "b", "specifiers": ["expand"], "lineNumber": 20}, + ], + )] + edges: list[dict[str, Any]] = [] + + self.assertEqual(mbg.link_python_calls(nodes, edges, extracted), (1, 0)) + self.assertEqual(edges[0]["target"], "function:a.py:expand") + + def test_skips_when_matching_import_order_is_unknown(self) -> None: + nodes = { + "function:service.py:run": _function_node("service.py", "run"), + "function:a.py:expand": _function_node("a.py", "expand"), + } + extracted = [_extraction_record( + "service.py", + "run", + "expand", + [{"source": "a", "specifiers": ["expand"]}], + )] + edges: list[dict[str, Any]] = [] + + self.assertEqual(mbg.link_python_calls(nodes, edges, extracted), (0, 1)) + self.assertEqual(edges, []) + + def test_links_from_import_submodule_attribute_call(self) -> None: + """from package import helper; helper.expand()""" + nodes = { + "function:service.py:run": _function_node("service.py", "run"), + "function:package/helper.py:expand": _function_node( + "package/helper.py", "expand" + ), + } + extracted = [_extraction_record( + "service.py", + "run", + "helper.expand", + [{"source": "package", "specifiers": ["helper"], "lineNumber": 1}], + )] + edges: list[dict[str, Any]] = [] + + self.assertEqual(mbg.link_python_calls(nodes, edges, extracted), (1, 0)) + self.assertEqual(edges[0]["target"], "function:package/helper.py:expand") + + def test_links_relative_from_import_submodule_attribute_call(self) -> None: + """from . import helper; helper.expand()""" + nodes = { + "function:package/service.py:run": _function_node( + "package/service.py", "run" + ), + "function:package/helper.py:expand": _function_node( + "package/helper.py", "expand" + ), + } + extracted = [_extraction_record( + "package/service.py", + "run", + "helper.expand", + [{"source": ".", "specifiers": ["helper"], "lineNumber": 1}], + )] + edges: list[dict[str, Any]] = [] + + self.assertEqual(mbg.link_python_calls(nodes, edges, extracted), (1, 0)) + self.assertEqual(edges[0]["target"], "function:package/helper.py:expand") + + def test_skips_submodule_call_when_imported_name_is_a_function(self) -> None: + """from package import helper; helper.expand() when helper is a function.""" + nodes = { + "function:service.py:run": _function_node("service.py", "run"), + "function:package/__init__.py:helper": _function_node( + "package/__init__.py", "helper" + ), + "function:package/helper.py:expand": _function_node( + "package/helper.py", "expand" + ), + } + extracted = [_extraction_record( + "service.py", + "run", + "helper.expand", + [{"source": "package", "specifiers": ["helper"], "lineNumber": 1}], + )] + edges: list[dict[str, Any]] = [] + + self.assertEqual(mbg.link_python_calls(nodes, edges, extracted), (0, 1)) + self.assertEqual(edges, []) + + def test_skips_empty_caller_or_callee_names(self) -> None: + nodes = { + "function:service.py:run": _function_node("service.py", "run"), + "function:service.py:expand": _function_node("service.py", "expand"), + } + extracted = [{ + "path": "service.py", + "language": "python", + "callGraph": [ + {"caller": "", "callee": "expand", "lineNumber": 10}, + {"caller": "run", "callee": "", "lineNumber": 11}, + ], + "imports": [], + }] + edges: list[dict[str, Any]] = [] + + self.assertEqual(mbg.link_python_calls(nodes, edges, extracted), (0, 2)) + self.assertEqual(edges, []) + + +class PythonModuleCandidatesTests(unittest.TestCase): + def test_resolves_absolute_and_relative_modules(self) -> None: + self.assertEqual( + mbg.python_module_candidates("ai/service.py", "package.helper"), + ("package/helper.py", "package/helper/__init__.py"), + ) + self.assertEqual( + mbg.python_module_candidates("ai/post/service.py", ".helper"), + ("ai/post/helper.py", "ai/post/helper/__init__.py"), + ) + self.assertEqual( + mbg.python_module_candidates("ai/post/service.py", "..shared.helper"), + ("ai/shared/helper.py", "ai/shared/helper/__init__.py"), + ) + self.assertEqual( + mbg._python_imported_submodule("package", "helper"), + "package.helper", + ) + self.assertEqual( + mbg._python_imported_submodule(".", "helper"), + ".helper", + ) + self.assertEqual( + mbg._python_imported_submodule(".pkg", "helper"), + ".pkg.helper", + ) + + def test_rejects_relative_module_escaping_project_root(self) -> None: + self.assertEqual( + mbg.python_module_candidates("service.py", "..shared.helper"), + (), + ) + self.assertEqual( + mbg.python_module_candidates("ai/service.py", "..shared.helper"), + (), + ) + + def test_rejects_absolute_source_paths(self) -> None: + self.assertEqual( + mbg.python_module_candidates("/tmp/service.py", "package.helper"), + (), + ) + + +class PythonExtractionLoaderTests(unittest.TestCase): + def test_loads_results_and_reports_malformed_artifacts(self) -> None: + import json as _json + import tempfile + + with tempfile.TemporaryDirectory(prefix="ua-python-calls-") as tmp: + ua_dir = Path(tmp) + artifact_dir = ua_dir / "tmp" + artifact_dir.mkdir() + (artifact_dir / "ua-file-extract-results-1.json").write_text( + _json.dumps({"results": [{"path": "service.py"}]}), + encoding="utf-8", + ) + (artifact_dir / "ua-file-extract-results-2.json").write_text( + _json.dumps({"notResults": []}), + encoding="utf-8", + ) + (artifact_dir / "ua-file-extract-results-3.json").write_text( + "{broken", + encoding="utf-8", + ) + + results, report = mbg.load_python_extraction_results(ua_dir) + + self.assertEqual(results, [{"path": "service.py"}]) + self.assertTrue(any("no results array" in line for line in report)) + self.assertTrue(any("could not parse" in line for line in report)) + + def test_missing_artifacts_are_non_fatal_and_reported(self) -> None: + import tempfile + + with tempfile.TemporaryDirectory(prefix="ua-python-calls-") as tmp: + results, report = mbg.load_python_extraction_results(Path(tmp)) + + self.assertEqual(results, []) + self.assertTrue(any("not found" in line for line in report)) + + +class PythonCallMergeIntegrationTests(unittest.TestCase): + def test_linker_runs_after_node_deduplication_and_reports_counts(self) -> None: + batch = { + "nodes": [ + _function_node("service.py", "run"), + _function_node("service.py", "expand"), + ], + "edges": [], + } + extracted = [_extraction_record("service.py", "run", "expand")] + + assembled, report = mbg.merge_and_normalize([batch], extracted) + + calls = [edge for edge in assembled["edges"] if edge["type"] == "calls"] + self.assertEqual(len(calls), 1) + self.assertIn("Python call linker:", report) + self.assertTrue(any("1 × calls edges produced" in line for line in report)) + self.assertTrue( + any("0 × unresolved or ambiguous calls skipped" in line for line in report) + ) + + # ── is_test_path ────────────────────────────────────────────────────────── class IsTestPathTests(unittest.TestCase): diff --git a/understand-anything-plugin/packages/core/src/plugins/extractors/__tests__/python-extractor.test.ts b/understand-anything-plugin/packages/core/src/plugins/extractors/__tests__/python-extractor.test.ts index 3929b05da..0bb7ac1b3 100644 --- a/understand-anything-plugin/packages/core/src/plugins/extractors/__tests__/python-extractor.test.ts +++ b/understand-anything-plugin/packages/core/src/plugins/extractors/__tests__/python-extractor.test.ts @@ -331,6 +331,50 @@ from foo import bar as baz parser.delete(); }); + it("preserves aliased import bindings", () => { + const { tree, parser, root } = parse(` +from package.service import run as execute +import package.helpers as helpers +`); + const result = extractor.extractStructure(root); + + expect(result.imports).toEqual([ + { + source: "package.service", + specifiers: ["execute"], + aliases: { execute: "run" }, + lineNumber: 2, + }, + { + source: "package.helpers", + specifiers: ["helpers"], + aliases: { helpers: "package.helpers" }, + lineNumber: 3, + }, + ]); + + tree.delete(); + parser.delete(); + }); + + it("preserves __proto__ alias as an own property", () => { + const { tree, parser, root } = parse(` +from package.service import run as __proto__ +`); + const result = extractor.extractStructure(root); + + expect(result.imports).toHaveLength(1); + expect(result.imports[0].specifiers).toEqual(["__proto__"]); + expect(Object.prototype.hasOwnProperty.call(result.imports[0].aliases, "__proto__")).toBe( + true, + ); + expect(result.imports[0].aliases?.["__proto__"]).toBe("run"); + expect((result.imports[0].aliases as any).polluted).toBeUndefined(); + + tree.delete(); + parser.delete(); + }); + it("extracts dotted module imports", () => { const { tree, parser, root } = parse(` import os.path diff --git a/understand-anything-plugin/packages/core/src/plugins/extractors/python-extractor.ts b/understand-anything-plugin/packages/core/src/plugins/extractors/python-extractor.ts index 83ae76cb0..b445d8af0 100644 --- a/understand-anything-plugin/packages/core/src/plugins/extractors/python-extractor.ts +++ b/understand-anything-plugin/packages/core/src/plugins/extractors/python-extractor.ts @@ -284,10 +284,17 @@ export class PythonExtractor implements LanguageExtractor { const alias = ai.children.find( (c) => c.type === "identifier", ); - if (dottedName) { + if (dottedName && alias) { + // Build via null-prototype map so keys like `__proto__` stay own + // properties, then copy to a plain object for JSON-safe output. + const aliasMap = Object.create(null) as Record; + aliasMap[alias.text] = dottedName.text; imports.push({ source: dottedName.text, - specifiers: [alias ? alias.text : dottedName.text], + specifiers: [alias.text], + aliases: Object.fromEntries( + Object.keys(aliasMap).map((k) => [k, aliasMap[k]]), + ), lineNumber: node.startPosition.row + 1, }); } @@ -304,6 +311,8 @@ export class PythonExtractor implements LanguageExtractor { const moduleNodeId = moduleNode?.id; const specifiers: string[] = []; + // null-prototype map so keys like `__proto__` stay own properties + const aliases = Object.create(null) as Record; // Collect dotted_name specifiers (non-aliased) // Skip the module_name dotted_name (compare by node id, not reference) @@ -316,12 +325,14 @@ export class PythonExtractor implements LanguageExtractor { // Collect aliased imports: `from foo import bar as baz` const aliasedImports = findChildren(node, "aliased_import"); for (const ai of aliasedImports) { + const original = findChild(ai, "dotted_name"); // The alias identifier follows the `as` keyword const alias = ai.children.find( (c) => c.type === "identifier", ); - if (alias) { + if (original && alias) { specifiers.push(alias.text); + aliases[alias.text] = original.text; } } @@ -330,9 +341,13 @@ export class PythonExtractor implements LanguageExtractor { specifiers.push("*"); } + const aliasKeys = Object.keys(aliases); imports.push({ source, specifiers, + ...(aliasKeys.length + ? { aliases: Object.fromEntries(aliasKeys.map((k) => [k, aliases[k]])) } + : {}), lineNumber: node.startPosition.row + 1, }); } diff --git a/understand-anything-plugin/packages/core/src/types.ts b/understand-anything-plugin/packages/core/src/types.ts index 28cadca0f..717f1947b 100644 --- a/understand-anything-plugin/packages/core/src/types.ts +++ b/understand-anything-plugin/packages/core/src/types.ts @@ -185,7 +185,12 @@ export interface ReferenceResolution { export interface StructuralAnalysis { functions: Array<{ name: string; lineRange: [number, number]; params: string[]; returnType?: string }>; classes: Array<{ name: string; lineRange: [number, number]; methods: string[]; properties: string[] }>; - imports: Array<{ source: string; specifiers: string[]; lineNumber: number }>; + imports: Array<{ + source: string; + specifiers: string[]; + lineNumber: number; + aliases?: Record; + }>; exports: Array<{ name: string; lineNumber: number; isDefault?: boolean }>; // Non-code structural data (all optional for backward compat) sections?: SectionInfo[]; diff --git a/understand-anything-plugin/skills/understand/extract-structure-result.mjs b/understand-anything-plugin/skills/understand/extract-structure-result.mjs index 5ab1d9446..784fe1c0d 100644 --- a/understand-anything-plugin/skills/understand/extract-structure-result.mjs +++ b/understand-anything-plugin/skills/understand/extract-structure-result.mjs @@ -33,6 +33,24 @@ export function buildResult(file, totalLines, nonEmptyLines, analysis, callGraph })); } + if (analysis.imports?.length) { + base.imports = analysis.imports.map(imp => { + const aliasEntries = imp.aliases + ? Object.entries(imp.aliases).filter( + ([key, value]) => typeof key === "string" && typeof value === "string", + ) + : []; + return { + source: imp.source, + specifiers: Array.isArray(imp.specifiers) ? [...imp.specifiers] : [], + ...(aliasEntries.length + ? { aliases: Object.fromEntries(aliasEntries) } + : {}), + lineNumber: imp.lineNumber, + }; + }); + } + if (analysis.exports && analysis.exports.length > 0) { base.exports = analysis.exports.map(exp => ({ name: exp.name, diff --git a/understand-anything-plugin/skills/understand/merge-batch-graphs.py b/understand-anything-plugin/skills/understand/merge-batch-graphs.py index 1c7746bfb..9752c6414 100644 --- a/understand-anything-plugin/skills/understand/merge-batch-graphs.py +++ b/understand-anything-plugin/skills/understand/merge-batch-graphs.py @@ -19,6 +19,7 @@ import json import os +import posixpath import re import sys from collections import Counter @@ -290,6 +291,336 @@ def normalize_complexity(value: Any) -> tuple[str, str]: return "moderate", "unknown" +# ── Deterministic Python call linker ────────────────────────────────────── + +def _normalized_project_path(path: str) -> str | None: + """Return a relative POSIX project path, or None if it escapes the root.""" + posix_path = path.replace("\\", "/") + if posix_path.startswith("/") or re.match(r"^[A-Za-z]:/", posix_path): + return None + normalized = posixpath.normpath(posix_path) + if normalized in ("", ".", "..") or normalized.startswith("../"): + return None + return normalized + + +def python_module_candidates(source_file: str, module: str) -> tuple[str, ...]: + """Map a Python import module to possible project-relative source files.""" + source_path = _normalized_project_path(source_file) + if source_path is None or not isinstance(module, str): + return () + + match = re.fullmatch(r"(\.*)([^\s./]+(?:\.[^\s./]+)*)?", module) + if match is None: + return () + leading_dots, dotted_module = match.groups() + module_parts = dotted_module.split(".") if dotted_module else [] + if any(not part.isidentifier() for part in module_parts): + return () + + if leading_dots: + base_parts = [ + part for part in posixpath.dirname(source_path).split("/") if part + ] + parent_levels = len(leading_dots) - 1 + if parent_levels and parent_levels >= len(base_parts): + return () + if parent_levels: + base_parts = base_parts[:-parent_levels] + path_parts = base_parts + module_parts + else: + if not module_parts: + return () + path_parts = module_parts + + module_path = "/".join(path_parts) + if not module_path: + return () + if dotted_module: + return (f"{module_path}.py", f"{module_path}/__init__.py") + return (f"{module_path}/__init__.py",) + + +def _python_imported_submodule(source: str, imported_name: str) -> str | None: + """Build the module path for `from source import imported_name` used as a package.""" + if not imported_name or not imported_name.isidentifier(): + return None + if not source: + return imported_name + if source.startswith("."): + # `from . import helper` → `.helper`; `from .pkg import helper` → `.pkg.helper` + if source.replace(".", "") == "": + return source + imported_name + return f"{source}.{imported_name}" + return f"{source}.{imported_name}" + + +def _matching_function_ids( + functions_by_file_and_name: dict[tuple[str, str], list[str]], + files: tuple[str, ...], + name: str, +) -> list[str]: + matches: list[str] = [] + for path in files: + for node_id in functions_by_file_and_name.get((path, name), []): + if node_id not in matches: + matches.append(node_id) + return matches + + +def _python_import_binds(import_record: dict[str, Any], callee: str) -> bool: + """Return whether an import record binds the called local name.""" + local_name = callee.split(".", 1)[0] + aliases = import_record.get("aliases") + if isinstance(aliases, dict) and local_name in aliases: + return True + specifiers = import_record.get("specifiers") + if not isinstance(specifiers, list): + return False + if local_name in specifiers: + return True + source = import_record.get("source") + return ( + "." in callee + and isinstance(source, str) + and source in specifiers + and source.split(".", 1)[0] == local_name + ) + + +def _active_python_import( + imports: list[Any], + callee: str, + call_line: Any, +) -> tuple[dict[str, Any] | None, bool]: + """Return the latest binding before a call and whether ordering is ambiguous.""" + matching = [ + (index, record) + for index, record in enumerate(imports) + if isinstance(record, dict) and _python_import_binds(record, callee) + ] + if not matching: + return None, False + if not isinstance(call_line, int) or isinstance(call_line, bool): + return None, True + + applicable: list[tuple[int, int, dict[str, Any]]] = [] + for index, record in matching: + import_line = record.get("lineNumber") + if not isinstance(import_line, int) or isinstance(import_line, bool): + return None, True + if import_line == call_line: + return None, True + if import_line < call_line: + applicable.append((import_line, index, record)) + if not applicable: + return None, False + return max(applicable, key=lambda item: (item[0], item[1]))[2], False + + +def _resolve_imported_python_targets( + functions_by_file_and_name: dict[tuple[str, str], list[str]], + source_file: str, + callee: str, + active_import: dict[str, Any], +) -> list[str]: + """Resolve a callee through one active import binding to function node ids.""" + source = active_import.get("source") + specifiers = active_import.get("specifiers") + aliases = active_import.get("aliases") + if not isinstance(source, str) or not isinstance(specifiers, list): + return [] + + aliases = aliases if isinstance(aliases, dict) else {} + module_files = python_module_candidates(source_file, source) + original = aliases.get(callee) + if callee in specifiers and callee not in aliases: + return _matching_function_ids( + functions_by_file_and_name, module_files, callee + ) + if ( + isinstance(original, str) + and original != source + and callee in specifiers + ): + return _matching_function_ids( + functions_by_file_and_name, module_files, original + ) + + qualifier, dot, member = callee.rpartition(".") + if not dot or not member.isidentifier(): + return [] + + aliased_module = aliases.get(qualifier) + is_module_binding = aliased_module == source and qualifier in specifiers + is_unaliased_module = qualifier == source and source in specifiers + if is_module_binding or is_unaliased_module: + return _matching_function_ids( + functions_by_file_and_name, module_files, member + ) + + # `from package import helper; helper.expand()` — helper is a submodule + if qualifier not in specifiers: + return [] + parent_symbol = aliases[qualifier] if qualifier in aliases else qualifier + if not isinstance(parent_symbol, str) or parent_symbol == source: + return [] + + # If the imported name is itself a function in the parent module, this is + # attribute access on a function — not a submodule call. Prefer skip. + if _matching_function_ids( + functions_by_file_and_name, module_files, parent_symbol + ): + return [] + + submodule = _python_imported_submodule(source, parent_symbol) + if submodule is None: + return [] + return _matching_function_ids( + functions_by_file_and_name, + python_module_candidates(source_file, submodule), + member, + ) + + +def link_python_calls( + nodes_by_id: dict[str, dict[str, Any]], + edges: list[dict[str, Any]], + extraction_results: list[dict[str, Any]], +) -> tuple[int, int]: + """Append calls edges only when Python caller and callee resolve uniquely.""" + functions_by_file_and_name: dict[tuple[str, str], list[str]] = {} + for node_id, node in nodes_by_id.items(): + if node.get("type") not in ("function", "func"): + continue + path = node.get("filePath") + name = node.get("name") + if not isinstance(path, str) or not isinstance(name, str) or not name: + continue + normalized_path = _normalized_project_path(path) + if normalized_path is None: + continue + functions_by_file_and_name.setdefault((normalized_path, name), []).append(node_id) + + existing = { + ( + edge.get("source"), + edge.get("target"), + edge.get("type"), + edge.get("direction", "forward"), + ) + for edge in edges + } + added = 0 + skipped = 0 + + for result in extraction_results: + if not isinstance(result, dict) or result.get("language") != "python": + continue + path = result.get("path") + calls = result.get("callGraph") + imports = result.get("imports") + if not isinstance(path, str) or not isinstance(calls, list): + continue + path = _normalized_project_path(path) + if path is None: + skipped += len(calls) + continue + imports = imports if isinstance(imports, list) else [] + + for call in calls: + if not isinstance(call, dict): + skipped += 1 + continue + caller = call.get("caller") + callee = call.get("callee") + if ( + not isinstance(caller, str) + or not isinstance(callee, str) + or not caller + or not callee + ): + skipped += 1 + continue + + caller_ids = functions_by_file_and_name.get((path, caller), []) + if len(caller_ids) != 1: + skipped += 1 + continue + + same_file = functions_by_file_and_name.get((path, callee), []) + if same_file: + target_ids = same_file + else: + active_import, binding_ambiguous = _active_python_import( + imports, callee, call.get("lineNumber") + ) + if binding_ambiguous: + skipped += 1 + continue + target_ids = ( + _resolve_imported_python_targets( + functions_by_file_and_name, path, callee, active_import + ) + if active_import is not None + else [] + ) + + unique_targets = list(dict.fromkeys(target_ids)) + if len(unique_targets) != 1: + skipped += 1 + continue + + key = (caller_ids[0], unique_targets[0], "calls", "forward") + if key in existing: + continue + edges.append({ + "source": caller_ids[0], + "target": unique_targets[0], + "type": "calls", + "direction": "forward", + "weight": 0.8, + "description": "Python call graph (deterministic)", + }) + existing.add(key) + added += 1 + + return added, skipped + + +def load_python_extraction_results( + ua_dir: Path, +) -> tuple[list[dict[str, Any]], list[str]]: + """Load file extraction records; malformed or missing artifacts are non-fatal.""" + artifact_paths = sorted( + (ua_dir / "tmp").glob("ua-file-extract-results-*.json") + ) + if not artifact_paths: + return [], [ + " Python extraction artifacts not found in " + f"{ua_dir / 'tmp'}" + ] + + results: list[dict[str, Any]] = [] + report: list[str] = [] + for artifact_path in artifact_paths: + try: + artifact = json.loads(artifact_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + report.append( + f" {artifact_path.name}: could not parse extraction artifact: {error}" + ) + continue + artifact_results = artifact.get("results") if isinstance(artifact, dict) else None + if not isinstance(artifact_results, list): + report.append( + f" {artifact_path.name}: ignored extraction artifact with no results array" + ) + continue + results.extend(result for result in artifact_results if isinstance(result, dict)) + return results, report + + # ── Deterministic tested_by linker ──────────────────────────────────────── # # Two-pass linker. Both passes produce canonical `production → test` edges. @@ -781,7 +1112,10 @@ def link_tests( # ── Main merge + normalize ──────────────────────────────────────────────── -def merge_and_normalize(batches: list[dict[str, Any]]) -> tuple[dict[str, Any], list[str]]: +def merge_and_normalize( + batches: list[dict[str, Any]], + extraction_results: list[dict[str, Any]] | None = None, +) -> tuple[dict[str, Any], list[str]]: """Merge batch results and normalize. Returns (assembled_graph, report_lines).""" # ── Pattern counters for "Fixed" report ────────────────────────── @@ -862,7 +1196,12 @@ def merge_and_normalize(batches: list[dict[str, Any]]) -> tuple[dict[str, Any], duplicate_count += 1 nodes_by_id[nid] = node - # ── Step 5b: Deterministic tested_by linker ────────────────────── + # ── Step 5b: Deterministic Python call linker ──────────────────── + python_calls_added, python_calls_skipped = link_python_calls( + nodes_by_id, all_edges, extraction_results or [] + ) + + # ── Step 5c: Deterministic tested_by linker ────────────────────── # See module-level "Deterministic tested_by linker" section above. tested_by_added, tested_by_dropped, tested_by_tagged, tested_by_swapped = link_tests( nodes_by_id, all_edges @@ -937,6 +1276,13 @@ def merge_and_normalize(batches: list[dict[str, Any]]) -> tuple[dict[str, Any], report.append(f" {tested_by_added:>4} × tested_by edges produced (path-convention supplement, production → test)") report.append(f" {tested_by_tagged:>4} × production nodes tagged \"tested\"") + report.append("") + report.append("Python call linker:") + report.append(f" {python_calls_added:>4} × calls edges produced") + report.append( + f" {python_calls_skipped:>4} × unresolved or ambiguous calls skipped" + ) + # Could not fix section — unknown patterns (grouped) + individual details unfixable_total = ( len(unfixable) @@ -1176,7 +1522,11 @@ def main() -> None: sys.exit(1) # Merge and normalize - assembled, report = merge_and_normalize(batches) + ua_dir = resolve_ua_dir(project_root) + extraction_results, extraction_report = load_python_extraction_results(ua_dir) + assembled, report = merge_and_normalize(batches, extraction_results) + if extraction_report: + report.extend(extraction_report) # Surface missing multi-part files to the phase report (parallel to # unrecognized-filename handling below). Stderr lines emitted during diff --git a/understand-anything-plugin/src/__tests__/extract-structure.test.mjs b/understand-anything-plugin/src/__tests__/extract-structure.test.mjs index ebcefde0d..6c6b5a412 100644 --- a/understand-anything-plugin/src/__tests__/extract-structure.test.mjs +++ b/understand-anything-plugin/src/__tests__/extract-structure.test.mjs @@ -31,6 +31,54 @@ describe("extract-structure buildResult", () => { }); }); + it("preserves deterministic import bindings", () => { + const result = buildResult( + file(), + 10, + 8, + analysis({ + imports: [{ + source: "package.service", + specifiers: ["execute"], + aliases: { execute: "run" }, + lineNumber: 2, + }], + }), + [{ caller: "main", callee: "execute", lineNumber: 5 }], + {}, + ); + + expect(result.imports).toEqual([{ + source: "package.service", + specifiers: ["execute"], + aliases: { execute: "run" }, + lineNumber: 2, + }]); + }); + + it("copies __proto__ alias keys as own properties", () => { + const aliases = Object.create(null); + aliases.__proto__ = "run"; + const result = buildResult( + file(), + 10, + 8, + analysis({ + imports: [{ + source: "package.service", + specifiers: ["__proto__"], + aliases, + lineNumber: 2, + }], + }), + null, + {}, + ); + + expect(Object.prototype.hasOwnProperty.call(result.imports[0].aliases, "__proto__")).toBe(true); + expect(result.imports[0].aliases.__proto__).toBe("run"); + }); + describe("importCount fallback", () => { // Only relative imports count toward the fallback metric — external // package imports would never produce edges so counting them would be