From d1108d13c45cadeddd84a357a612e20458affd77 Mon Sep 17 00:00:00 2001 From: Rudy Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:45:55 -0400 Subject: [PATCH] fix(providers): project array items before selecting values Signed-off-by: Rudy Celekli <47457359+rudycelekli@users.noreply.github.com> --- .../enrichment/declarative/template.py | 14 +++- tests/test_declarative_array_projection.py | 78 +++++++++++++++++++ 2 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 tests/test_declarative_array_projection.py diff --git a/apps/api/services/leadgen/enrichment/declarative/template.py b/apps/api/services/leadgen/enrichment/declarative/template.py index edb30bd9..dbca9a7d 100644 --- a/apps/api/services/leadgen/enrichment/declarative/template.py +++ b/apps/api/services/leadgen/enrichment/declarative/template.py @@ -68,7 +68,8 @@ def render_template(value: Any, ctx: Dict[str, Any], env_resolver: Callable[[str def _walk_path(data: Any, path: str) -> Any: """Walk a JSONPath-lite path (no leading `$.`). Supports `[]` fan-out and `[i]`.""" cur = data - for raw in path.split("."): + parts = path.split(".") + for position, raw in enumerate(parts): if cur is None: return None # token may be `key`, `key[]`, `key[2]`, or `[2]` @@ -82,9 +83,14 @@ def _walk_path(data: Any, path: str) -> Any: if not isinstance(cur, list): return None if idx == "" or idx is None: - # fan-out: return first non-empty element later; for scalar - # projection we take the first element - cur = cur[0] if cur else None + # Choose the first non-empty projected value, not merely + # the first array item (which may lack the remaining field). + remainder = ".".join(parts[position + 1:]) + for item in cur: + value = _walk_path(item, remainder) if remainder else item + if value is not None and value != "": + return value + return None else: i = int(idx) cur = cur[i] if 0 <= i < len(cur) else None diff --git a/tests/test_declarative_array_projection.py b/tests/test_declarative_array_projection.py new file mode 100644 index 00000000..7900274d --- /dev/null +++ b/tests/test_declarative_array_projection.py @@ -0,0 +1,78 @@ +"""Array fan-out honors the full response path before choosing a value.""" +import asyncio +import socket + +import httpx +import pytest + +from apps.api.services.leadgen.models import Lead +from apps.api.services.leadgen.enrichment.declarative.compiler import compile_manifest +from apps.api.services.leadgen.enrichment.declarative.manifest import ProviderManifest +from apps.api.services.leadgen.enrichment.declarative.template import project_value +from apps.api.services.workbook.http_column import execute_http_column + + +@pytest.fixture +def response_transport(monkeypatch): + real_init = httpx.AsyncClient.__init__ + seen = [] + + def install(payload): + def handler(request): + seen.append(request) + return httpx.Response(200, json=payload) + + def init(self, *args, **kwargs): + kwargs["transport"] = httpx.MockTransport(handler) + real_init(self, *args, **kwargs) + + monkeypatch.setattr(httpx.AsyncClient, "__init__", init) + monkeypatch.setattr(socket, "getaddrinfo", lambda *args, **kwargs: [ + (socket.AF_INET, None, None, "", ("93.184.216.34", 443)), + ]) + return seen + return install + + +@pytest.mark.parametrize("payload,path", [ + ({"results": [{}, {"email": "person@example.test"}]}, "$.results[].email"), + ({"results": [{"email": None}, {"email": "person@example.test"}]}, "$.results[].email"), + ({"results": [{"email": ""}, {"email": "person@example.test"}]}, "$.results[].email"), + ({"groups": [{"results": []}, {"results": [{}, {"email": "person@example.test"}]}]}, + "$.groups[].results[].email"), +]) +def test_compiled_provider_keeps_later_projected_result(response_transport, payload, path): + requests = response_transport(payload) + manifest = ProviderManifest( + name="array_email", capability="email", auth={"type": "none"}, + request={"method": "GET", "url": "https://api.example.test/find"}, + response={"mappings": {"email": path}}, + ) + result = asyncio.run(compile_manifest(manifest).enrich(Lead(company="Example"))) + assert result.success, result.error + assert result.fields == {"email": "person@example.test"} + assert len(requests) == 1 + + +def test_http_action_uses_later_projected_result(response_transport): + requests = response_transport({"results": [{}, {"email": "person@example.test"}]}) + result = asyncio.run(execute_http_column({ + "type": "http", "http_url": "https://api.example.test/find", + "http_extract": "$.results[].email", + }, {}, [])) + assert result == {"success": True, "value": "person@example.test", "error": None} + assert len(requests) == 1 + + +@pytest.mark.parametrize("data,path,expected", [ + ({"results": [{"email": "first"}, {"email": "second"}]}, "$.results[].email", "first"), + ({"results": [{}, {"email": "second"}]}, "$.results[0].email", None), + ({"results": [{}, {"email": "second"}]}, "$.results[1].email", "second"), + ({"results": [{"value": False}, {"value": True}]}, "$.results[].value", False), + ({"results": [{"value": 0}, {"value": 1}]}, "$.results[].value", 0), + ({"results": []}, "$.results[].email", None), + ({"results": "invalid"}, "$.results[].email", None), +]) +def test_array_projection_controls(data, path, expected): + actual = project_value(data, path) + assert actual == expected and type(actual) is type(expected)