From 8ad4b8dcec302a9f4d0111b4ff6dcbc287593b5d Mon Sep 17 00:00:00 2001 From: marndt <1682607+marndt@users.noreply.github.com> Date: Wed, 24 Jun 2026 22:32:30 +0000 Subject: [PATCH] chore(release): Release v1.4.0 --- .dockerignore | 6 + CHANGELOG.md | 14 + Dockerfile | 24 + openapi/dataplane.yaml | 16 +- pytest.ini | 1 + src/honeyhive/__init__.py | 2 +- .../_generated/models/ExperimentRunObject.py | 2 + .../models/LegacyUpdateEventRequest.py | 2 +- .../_generated/models/UpdateEventRequest.py | 2 +- src/honeyhive/adapters/__init__.py | 9 + src/honeyhive/adapters/copilot_studio.py | 375 ++++++++++ tests/integration/conftest.py | 98 +-- tests/unit/test_copilot_studio_adapter.py | 686 ++++++++++++++++++ tests/utils.py | 24 +- tests/utils/__init__.py | 16 - tests/utils/env_enforcement.py | 311 -------- 16 files changed, 1184 insertions(+), 404 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 src/honeyhive/adapters/__init__.py create mode 100644 src/honeyhive/adapters/copilot_studio.py create mode 100644 tests/unit/test_copilot_studio_adapter.py delete mode 100644 tests/utils/env_enforcement.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..1baaf8ec --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.env +.env.* +.turbo +.venv +.corepack +.ruff_cache diff --git a/CHANGELOG.md b/CHANGELOG.md index 22fea28e..ed929593 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ ## [Unreleased] +## [1.4.0] - 2026-06-24 + +### Added + +- **Copilot Studio trace forwarding adapter** + - New `honeyhive.adapters.copilot_studio.copilot_studio_records_to_spans()` converts Azure Application Insights diagnostic-export records from a Copilot Studio agent into OpenTelemetry spans you can forward to HoneyHive via OTLP. Maps user/agent messages, tool calls, and errors using GenAI semantic conventions, preserves parent/child trace correlation, and groups a conversation into a single HoneyHive session. Set `COPILOT_STUDIO_ADAPTER_KEEP_TOPIC_SPANS=1` to retain internal orchestration spans (dropped by default) and `COPILOT_STUDIO_ADAPTER_DEBUG=1` to attach the raw source record for debugging. +- **`dataset_name` on experiment runs** + - `ExperimentRunObject` now includes a `dataset_name` field alongside `dataset_id`, so the linked dataset's name is available when listing or fetching experiment runs. The field is optional and resolves to `None` for offline (`EXT-*`), deleted, or unset datasets. + +### Changed + +- **Event update: `outputs` must be an object** + - `UpdateEventRequest.outputs` and `LegacyUpdateEventRequest.outputs` are now typed as `Dict[str, Any]` instead of `Any`. Update calls accept only normalized (object-shaped) outputs; previously an array could slip through and corrupt the stored event. If you were passing a list when updating an event, wrap it as `{"value": [...]}`. Event creation and OTEL ingestion continue to normalize arrays automatically. + ## [1.3.0] - 2026-06-18 ### Added diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..6ee6a02e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +FROM ghcr.io/astral-sh/uv:python3.11-alpine +RUN apk add --no-cache nodejs npm && \ + npm install -g @anthropic-ai/claude-code@2.1.190 + +WORKDIR /app + +# Hatch resolves dynamic version/readme during uv sync, so copy metadata files +# before installing deps. Full source is copied in a later layer. +COPY pyproject.toml README.md ./ +COPY src/honeyhive/__init__.py src/honeyhive/__init__.py + +ENV UV_LINK_MODE=copy + +RUN uv sync \ + --no-install-project \ + --all-packages \ + --extra dev \ + --extra openinference-openai \ + --extra openinference-anthropic \ + --extra openinference-claude-agent-sdk + +COPY . . + +ENTRYPOINT [ "uv", "run", "--extra", "dev" ] diff --git a/openapi/dataplane.yaml b/openapi/dataplane.yaml index 0468985d..e9e2e8c3 100644 --- a/openapi/dataplane.yaml +++ b/openapi/dataplane.yaml @@ -3649,7 +3649,11 @@ components: additionalProperties: {} description: Metric values to merge into the event outputs: - description: Output data to replace on the event (accepts objects, strings, arrays, or scalars) + type: + - object + - 'null' + additionalProperties: {} + description: Output object to replace on the event. Must be an object or null; null preserves the existing outputs. Non-object values (strings, arrays, scalars) are rejected. config: type: object additionalProperties: {} @@ -3689,7 +3693,11 @@ components: additionalProperties: {} description: Metric values to merge into the event outputs: - description: Output data to replace on the event (accepts objects, strings, arrays, or scalars) + type: + - object + - 'null' + additionalProperties: {} + description: Output object to replace on the event. Must be an object or null; null preserves the existing outputs. Non-object values (strings, arrays, scalars) are rejected. config: type: object additionalProperties: {} @@ -4186,6 +4194,10 @@ components: type: - string - 'null' + dataset_name: + type: + - string + - 'null' required: - id - run_id diff --git a/pytest.ini b/pytest.ini index a5d0fe49..958f0c05 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,6 +2,7 @@ # in pyproject.toml is silently ignored — update markers and addopts here only. [pytest] testpaths = tests +pythonpath = src python_files = test_*.py python_classes = Test* python_functions = test_* diff --git a/src/honeyhive/__init__.py b/src/honeyhive/__init__.py index c5073cc4..bab7197d 100644 --- a/src/honeyhive/__init__.py +++ b/src/honeyhive/__init__.py @@ -5,7 +5,7 @@ # Version must be defined BEFORE imports to avoid circular import issues # Version must be semver or semver followed by "a" (alpha), "b" (beta), or "rc" # (release candidate) + a number -__version__ = "1.3.0" +__version__ = "1.4.0" # Main API client from .api import HoneyHive diff --git a/src/honeyhive/_generated/models/ExperimentRunObject.py b/src/honeyhive/_generated/models/ExperimentRunObject.py index eaa113c9..841d818a 100644 --- a/src/honeyhive/_generated/models/ExperimentRunObject.py +++ b/src/honeyhive/_generated/models/ExperimentRunObject.py @@ -50,3 +50,5 @@ class ExperimentRunObject(BaseModel): scope_id: str = Field(validation_alias="scope_id") dataset_id: Optional[str] = Field(validation_alias="dataset_id", default=None) + + dataset_name: Optional[str] = Field(validation_alias="dataset_name", default=None) diff --git a/src/honeyhive/_generated/models/LegacyUpdateEventRequest.py b/src/honeyhive/_generated/models/LegacyUpdateEventRequest.py index 8aeac0b6..e54f90d8 100644 --- a/src/honeyhive/_generated/models/LegacyUpdateEventRequest.py +++ b/src/honeyhive/_generated/models/LegacyUpdateEventRequest.py @@ -30,7 +30,7 @@ class LegacyUpdateEventRequest(BaseModel): metrics: Optional[Dict[str, Any]] = Field(validation_alias="metrics", default=None) - outputs: Optional[Any] = Field(validation_alias="outputs", default=None) + outputs: Optional[Dict[str, Any]] = Field(validation_alias="outputs", default=None) config: Optional[Dict[str, Any]] = Field(validation_alias="config", default=None) diff --git a/src/honeyhive/_generated/models/UpdateEventRequest.py b/src/honeyhive/_generated/models/UpdateEventRequest.py index b7c6d689..6028ab17 100644 --- a/src/honeyhive/_generated/models/UpdateEventRequest.py +++ b/src/honeyhive/_generated/models/UpdateEventRequest.py @@ -28,7 +28,7 @@ class UpdateEventRequest(BaseModel): metrics: Optional[Dict[str, Any]] = Field(validation_alias="metrics", default=None) - outputs: Optional[Any] = Field(validation_alias="outputs", default=None) + outputs: Optional[Dict[str, Any]] = Field(validation_alias="outputs", default=None) config: Optional[Dict[str, Any]] = Field(validation_alias="config", default=None) diff --git a/src/honeyhive/adapters/__init__.py b/src/honeyhive/adapters/__init__.py new file mode 100644 index 00000000..f8854a60 --- /dev/null +++ b/src/honeyhive/adapters/__init__.py @@ -0,0 +1,9 @@ +"""Adapters that convert third-party telemetry exports into OpenTelemetry spans. + +Import each adapter's submodule directly (e.g. +``from honeyhive.adapters.copilot_studio import copilot_studio_records_to_spans``) so that +pulling in one adapter does not eagerly import the OpenTelemetry SDK for users who don't need +it. This package intentionally does not re-export adapter symbols at the top level. +""" + +__all__: list[str] = [] diff --git a/src/honeyhive/adapters/copilot_studio.py b/src/honeyhive/adapters/copilot_studio.py new file mode 100644 index 00000000..39fcf5de --- /dev/null +++ b/src/honeyhive/adapters/copilot_studio.py @@ -0,0 +1,375 @@ +"""Convert Microsoft Copilot Studio diagnostic export records to OpenTelemetry spans. + +Copilot Studio agents emit telemetry to Azure Application Insights. Customers route that +telemetry through Azure Monitor diagnostic settings, which export records to an Event Hub or +to Blob storage. This module converts those exported records into :class:`ReadableSpan` +objects that can be handed directly to an OTLP exporter and shipped to HoneyHive. + +Azure Monitor diagnostic export wire format (confirmed from live records): every record is a +flat object — all Log Analytics columns (``Name``, ``Id``, ``OperationId``, ``UserId``, etc.) +appear at the top level, not nested inside a sub-object. ``Properties`` (capital P) is a dict +of Copilot Studio-specific custom key/value pairs. The type discriminant is ``category`` on +newer exports and ``Type`` on older ones; the adapter accepts both. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import re +import uuid +from datetime import datetime, timezone + +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import Event, ReadableSpan +from opentelemetry.sdk.util.instrumentation import InstrumentationScope +from opentelemetry.trace import SpanContext, SpanKind, TraceFlags +from opentelemetry.trace.status import Status, StatusCode +from opentelemetry.util.types import AttributeValue + +_LOG = logging.getLogger(__name__) + +_SPAN_TYPES: frozenset[str] = frozenset({"AppEvents", "AppRequests", "AppDependencies"}) +# TODO: handle AppExceptions — see HHAI-5791 + +# Internal Copilot Studio orchestration spans: empty inputs/outputs, no scoreable content. +# Dropped by default; set COPILOT_STUDIO_ADAPTER_KEEP_TOPIC_SPANS=1 to retain. +_TOPIC_SPAN_NAMES: frozenset[str] = frozenset( + { + "TopicStart", + "TopicEnd", + "TopicAction", + "PowerVirtualAgentRoot", + } +) + +_SCOPE = InstrumentationScope( + name="honeyhive.adapters.copilot_studio", + version="0.1.0", +) + +_GEN_AI_SYSTEM = "microsoft.copilot_studio" + +_TRUTHY: frozenset[str] = frozenset({"1", "true", "yes"}) + + +def copilot_studio_records_to_spans( + records: list[dict[str, object]], +) -> list[ReadableSpan]: + """Convert Application Insights diagnostic export records from a Copilot Studio agent to spans. + + Accepts records in the Azure Monitor diagnostic export format — the ``records`` array from + an Event Hub message already unpacked by the caller. Each record has a flat structure with + all Log Analytics columns at the top level (``Type`` or ``category``, ``Name``, ``Id``, + ``OperationId``, etc.). + + Only records with a type of ``AppEvents``, ``AppRequests``, or ``AppDependencies`` + are converted. All other types (``AppExceptions``, ``AppTraces``, etc.) are silently + skipped — they carry log-signal data, not span data. + + Each valid record is mapped to a single :class:`ReadableSpan`. See the module docstring + for the full field mapping. + + Records missing a required field (``OperationId``, ``Id``, or ``time``) are skipped with a + warning. + + Returns an empty list if no records survive filtering and validation. + + Args: + records: Diagnostic export records, each a JSON object parsed into a dict. + + Returns: + A list of converted spans, one per valid record. May be empty. + """ + resource_cache: dict[tuple[str, str], Resource] = {} + spans: list[ReadableSpan] = [] + + for record in records: + record_type = _get_str(record, "category") or _get_str(record, "Type") + if record_type not in _SPAN_TYPES: + _LOG.debug("Skipping record type %r", record_type or "(missing)") + continue + name = _get_str(record, "Name") + if name == "n/a": + _LOG.debug("Skipping unnamed dependency record") + continue + keep_topics = ( + os.environ.get("COPILOT_STUDIO_ADAPTER_KEEP_TOPIC_SPANS", "").strip() + in _TRUTHY + ) + # Topic/root orchestration spans are noise; drop them unless opted in. Error topics + # (e.g. "...topic.OnError") are exempt — they may carry error content not otherwise + # captured, so we never silently discard them. + is_topic_span = name in _TOPIC_SPAN_NAMES or ".topic." in name + if not keep_topics and is_topic_span and "error" not in name.lower(): + _LOG.debug( + "Skipping topic span %r (set COPILOT_STUDIO_ADAPTER_KEEP_TOPIC_SPANS=1 to retain)", + name, + ) + continue + try: + spans.append(_to_readable_span(record, resource_cache)) + except (ValueError, KeyError) as exc: + _LOG.warning("Skipping malformed record: %s — %.300s", exc, str(record)) + + return spans + + +def _to_readable_span( + record: dict[str, object], + resource_cache: dict[tuple[str, str], Resource], +) -> ReadableSpan: + custom_props = _get_custom_props(record) + + # ── Trace context ────────────────────────────────────────────────────────── + # Design-mode sessions in the Copilot Studio test canvas don't emit OperationId; + # fall back to conversationId from Properties, which is stable across the conversation. + operation_id = _get_str(record, "OperationId") or _get_str( + custom_props, "conversationId" + ) + if not operation_id: + raise ValueError( + "Missing required field 'OperationId' and no 'conversationId' fallback" + ) + # AppEvents has no Id column; synthesize a stable span ID from timestamp + event name. + span_raw_id = ( + _get_str(record, "Id") + or f"{_get_str(record, 'time')}:{_get_str(record, 'Name')}" + ) + hh_session_id = str(uuid.uuid5(uuid.NAMESPACE_URL, operation_id)) + + trace_id = _to_trace_id(operation_id) + context = SpanContext( + trace_id=trace_id, + span_id=_to_span_id(span_raw_id), + is_remote=True, + trace_flags=TraceFlags(TraceFlags.SAMPLED), + ) + + parent_context: SpanContext | None = None + raw_parent = _get_str(record, "ParentId") + # App Insights sometimes sets ParentId == OperationId on root spans; clear it to avoid + # making the span its own parent. + if raw_parent and raw_parent != _get_str(record, "OperationId"): + parent_context = SpanContext( + trace_id=trace_id, + span_id=_to_span_id(raw_parent), + is_remote=True, + trace_flags=TraceFlags(TraceFlags.SAMPLED), + ) + + # ── Timestamps ──────────────────────────────────────────────────────────── + start_ns = _iso_to_ns(_require(record, "time")) + end_ns = start_ns + int(_get_float(record, "DurationMs") * 1_000_000) + + # ── Name and custom properties ──────────────────────────────────────────── + event_name = _get_str(record, "Name") or "Unknown" + operation = _event_to_operation(event_name) + # Error status is keyed on the event name only. AppRequests/AppDependencies failure + # signals (Success == false, ResultCode) are out of scope here — Copilot Studio surfaces + # turn errors as OnErrorLog AppEvents, and we have no live records of request/dependency + # failure shapes to map against (see also the AppExceptions follow-up). + is_error = "error" in event_name.lower() + + # ── Attributes ──────────────────────────────────────────────────────────── + # AppRoleInstance is the actual bot name; AppRoleName is the generic runtime name. + agent_name = _get_str(record, "AppRoleInstance") or _get_str(record, "AppRoleName") + raw_attrs: dict[str, AttributeValue] = { + "gen_ai.system": _GEN_AI_SYSTEM, + "gen_ai.operation.name": operation, + "gen_ai.agent.name": agent_name, + "gen_ai.conversation.id": _get_str(custom_props, "conversationId"), + # UserId is a top-level column; fromId is the per-message sender in Properties. + "enduser.id": ( + _get_str(record, "UserId") + or _get_str(custom_props, "fromId") + or _get_str(custom_props, "userId") + ), + # SessionId is a top-level column, not a Property. + "session.id": _get_str(record, "SessionId"), + "messaging.destination": _get_str(custom_props, "channelId"), + "copilot_studio.topic_name": _normalize_topic_name( + _get_str(custom_props, "TopicName") + ), + "copilot_studio.channel_id": _get_str(custom_props, "channelId"), + "copilot_studio.design_mode": _get_str(custom_props, "DesignMode"), + "copilot_studio.user_name": _get_str(custom_props, "fromName"), + "honeyhive.session_id": hh_session_id, + "honeyhive.session_auto_create": True, + } + if is_error: + # PascalCase keys confirmed from live records; camelCase as forward-compat fallback. + raw_attrs["error.type"] = ( + _get_str(custom_props, "ErrorCode") + or _get_str(custom_props, "errorCode") + or "unknown" + ) + # Read at call time so toggling COPILOT_STUDIO_ADAPTER_DEBUG takes effect without restart. + if os.environ.get("COPILOT_STUDIO_ADAPTER_DEBUG", "").strip() in _TRUTHY: + raw_attrs["debug.raw_record"] = json.dumps(record, default=str) + + attributes: dict[str, AttributeValue] = { + k: v for k, v in raw_attrs.items() if v != "" + } + + # ── Span events (message content) ───────────────────────────────────────── + events: list[Event] = [] + if text := _get_str(custom_props, "text"): + if event_name == "BotMessageReceived": + events.append( + Event( + name="gen_ai.user.message", + attributes={"gen_ai.system": _GEN_AI_SYSTEM, "content": text}, + timestamp=start_ns, + ) + ) + else: + events.append( + Event( + name="gen_ai.choice", + attributes={ + "gen_ai.system": _GEN_AI_SYSTEM, + "content": text, + "finish_reason": "stop", + "index": 0, + }, + timestamp=start_ns, + ) + ) + + # ── Resource (deduplicated by agent name + instance) ────────────────────── + service_name = _get_str(record, "AppRoleName") or "unknown" + instance_id = _get_str(record, "AppRoleInstance") + resource_key = (service_name, instance_id) + if resource_key not in resource_cache: + resource_cache[resource_key] = Resource( + {"service.name": service_name, "service.instance.id": instance_id} + ) + + error_message = _get_str(custom_props, "ErrorMessage") or _get_str( + custom_props, "errorMessage" + ) + return ReadableSpan( + name=operation, + context=context, + parent=parent_context, + resource=resource_cache[resource_key], + attributes=attributes, + events=events, + kind=SpanKind.INTERNAL, + instrumentation_scope=_SCOPE, + status=Status( + status_code=StatusCode.ERROR if is_error else StatusCode.OK, + description=error_message if (is_error and error_message) else None, + ), + start_time=start_ns, + end_time=end_ns, + ) + + +def _to_trace_id(raw: str) -> int: + """Map a correlation id to a valid 128-bit OTel trace id (hex passthrough or hashed).""" + return _to_id(raw, byte_len=16) + + +def _to_span_id(raw: str) -> int: + """Map a correlation id to a valid 64-bit OTel span id (hex passthrough or hashed).""" + return _to_id(raw, byte_len=8) + + +def _to_id(raw: str, *, byte_len: int) -> int: + """Return a non-zero ``byte_len``-byte int derived from ``raw``. + + Strips dashes, pipes, and trailing dots before parsing (Application Insights uses + hierarchical IDs like ``|abc123.1`` that are not valid hex). If the stripped string is + exactly ``2 * byte_len`` hex chars, it is used directly. Otherwise the value is hashed + with SHA-256 and the leading ``byte_len`` bytes are used — deterministic across calls so + parent/child correlation is preserved. OTel rejects all-zero IDs, so the + (astronomically unlikely) zero result is bumped to 1. + """ + stripped = raw.replace("-", "").replace("|", "").rstrip(".") + stripped = re.sub(r"\.\d+$", "", stripped) + if re.fullmatch(rf"[0-9a-fA-F]{{{byte_len * 2}}}", stripped): + value = int(stripped, 16) + else: + digest = hashlib.sha256(raw.encode("utf-8")).digest() + value = int.from_bytes(digest[:byte_len], "big") + return value or 1 + + +def _iso_to_ns(ts: str) -> int: + ts = ts.replace("Z", "+00:00") + dt = datetime.fromisoformat(ts) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return int(dt.timestamp() * 1_000_000_000) + + +def _normalize_topic_name(raw: str) -> str: + """Strip the fully-qualified agent prefix from a Copilot Studio topic name. + + TopicStart records set TopicName to the qualified form (e.g. + ``cr44e_MyAgent.topic.Greeting``); TopicAction records use the short display + name (``Greeting``). Normalise both to the short form. + """ + if not raw: + return raw + idx = raw.find(".topic.") + return raw[idx + len(".topic.") :] if idx != -1 else raw + + +def _event_to_operation(event_name: str) -> str: + # Fully-qualified topic events emitted by the Copilot Studio runtime have the form + # "{agentPrefix}.topic.{TopicName}" (e.g. "cr44e_MyAgent.topic.Greeting"). These are + # agent-action spans; the topic name is already captured in copilot_studio.topic_name. + if ".topic." in event_name: + return "agent_action" + return { + "BotMessageReceived": "user_input", + "BotMessageSend": "agent_output", + "TopicStart": "agent_action", + "TopicEnd": "agent_action", + "TopicAction": "agent_action", + "PowerVirtualAgentRoot": "agent_action", + "OnErrorLog": "agent_error", + }.get(event_name, "agent_action") + + +def _get_custom_props(record: dict[str, object]) -> dict[str, object]: + """Return the ``Properties`` custom-dimensions dict from a record. + + The Event Hub diagnostic export emits ``Properties`` as a JSON dict object. + """ + value = record.get("Properties") + return value if isinstance(value, dict) else {} + + +def _require(record: dict[str, object], key: str) -> str: + """Return ``record[key]`` as a non-empty string or raise ``ValueError``.""" + value = _get_str(record, key) + if not value: + raise ValueError(f"Missing required field {key!r}") + return value + + +def _get_str(d: dict[str, object], key: str) -> str: + """Return ``d[key]`` as a string, or ``""`` when missing or not a string.""" + value = d.get(key) + return value if isinstance(value, str) else "" + + +def _get_float(d: dict[str, object], key: str) -> float: + """Return ``d[key]`` coerced to a float, or ``0.0`` when missing or non-numeric.""" + value = d.get(key) + if isinstance(value, bool): + return 0.0 + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + try: + return float(value) + except ValueError: + return 0.0 + return 0.0 diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 998aeac9..9529a6c9 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -24,19 +24,10 @@ # Import OTEL reset utilities from tests.utils import ( # pylint: disable=no-name-in-module - enforce_local_env_file, ensure_clean_otel_state, reset_otel_to_provider, ) -# Enforce .env file loading for local development -try: - enforce_local_env_file() -except Exception as e: - # In CI environments, this is expected to fail - environment variables - # should be set directly in CI - pass - def pytest_addoption(parser: Any) -> None: """Add command line options for integration tests.""" @@ -188,46 +179,12 @@ def integration_test_config() -> Dict[str, Any]: # Real API credentials and related fixtures @pytest.fixture(scope="session") -def real_api_credentials() -> Dict[str, Any]: - """Get real API credentials for integration tests.""" - from tests.utils import ( # pylint: disable=no-name-in-module - enforce_integration_credentials, - get_llm_credentials, - ) - - try: - # Validate environment credentials - core_credentials = enforce_integration_credentials() - llm_credentials = get_llm_credentials() - - credentials = { - "api_key": core_credentials["HH_API_KEY"], - "source": os.environ.get("HH_SOURCE", "pytest-integration"), - "server_url": os.environ.get( - "HH_API_URL", "https://api.testing-dp-1.honeyhive.ai" - ), - "project": os.environ.get("HH_PROJECT", "test-project"), - } - - # Add LLM credentials for instrumentor tests - filter out None values - filtered_llm_credentials = { - k: v for k, v in llm_credentials.items() if v is not None - } - credentials.update(filtered_llm_credentials) - - return credentials - - except Exception as e: - pytest.fail( - f"Real API credentials enforcement failed: {e}\n" - "Tests must not skip - use real credentials." - ) - - -@pytest.fixture(scope="session") -def real_api_key(real_api_credentials: Dict[str, Any]) -> str: - """Real API key for integration tests.""" - return str(real_api_credentials["api_key"]) +def real_api_key() -> str: + """Real API key for integration tests from environment.""" + api_key = os.getenv("HH_API_KEY") + if not api_key: + pytest.fail("HH_API_KEY environment variable is required for integration tests") + return api_key @pytest.fixture(scope="session") @@ -237,17 +194,36 @@ def real_project() -> str: @pytest.fixture(scope="session") -def real_source(real_api_credentials: Dict[str, Any]) -> str: - """Real source for integration tests.""" - return str(real_api_credentials["source"]) +def real_source() -> str: + """Real source for integration tests from environment.""" + return os.environ.get("HH_SOURCE", "pytest-integration") + + +@pytest.fixture(scope="session") +def real_api_credentials( + real_api_key: str, real_project: str, real_source: str +) -> Dict[str, str]: + """Combined credentials dictionary for integration tests. + + Provides a dictionary with all necessary credentials for creating + HoneyHive clients and tracers in integration tests. + + Returns: + Dict with keys: api_key, project, source + """ + return { + "api_key": real_api_key, + "project": real_project, + "source": real_source, + } @pytest.fixture -def integration_client(real_api_credentials: Dict[str, Any]) -> HoneyHive: +def integration_client(real_api_key: str) -> HoneyHive: """HoneyHive client for integration tests with real API credentials.""" return HoneyHive( - api_key=real_api_credentials["api_key"], - base_url=real_api_credentials["server_url"], + api_key=real_api_key, + base_url=os.environ.get("HH_API_URL", "https://api.testing-dp-1.honeyhive.ai"), ) @@ -337,11 +313,11 @@ def reload() -> None: @pytest.fixture -def real_honeyhive_tracer(real_api_credentials: Dict[str, Any]) -> Any: +def real_honeyhive_tracer(real_api_key: str, real_source: str) -> Any: """Create a real HoneyHive tracer with NO MOCKING.""" tracer = HoneyHiveTracer( - api_key=real_api_credentials["api_key"], - source=real_api_credentials["source"], + api_key=real_api_key, + source=real_source, test_mode=False, # Real API mode disable_http_tracing=True, # Avoid HTTP conflicts in tests ) @@ -357,7 +333,7 @@ def real_honeyhive_tracer(real_api_credentials: Dict[str, Any]) -> Any: @pytest.fixture -def fresh_tracer_environment(real_api_credentials: Dict[str, Any]) -> Any: +def fresh_tracer_environment(real_api_key: str, real_source: str) -> Any: """Create a completely fresh tracer environment for each test.""" # Reset OpenTelemetry global state try: @@ -370,8 +346,8 @@ def fresh_tracer_environment(real_api_credentials: Dict[str, Any]) -> Any: # Create fresh tracer tracer = HoneyHiveTracer( - api_key=real_api_credentials["api_key"], - source=f"{real_api_credentials['source']}-fresh", + api_key=real_api_key, + source=f"{real_source}-fresh", test_mode=False, disable_http_tracing=True, ) diff --git a/tests/unit/test_copilot_studio_adapter.py b/tests/unit/test_copilot_studio_adapter.py new file mode 100644 index 00000000..397907ff --- /dev/null +++ b/tests/unit/test_copilot_studio_adapter.py @@ -0,0 +1,686 @@ +"""Unit tests for honeyhive.adapters.copilot_studio.""" + +from __future__ import annotations + +import hashlib +import json +import uuid + +import pytest +from opentelemetry.trace.status import StatusCode + +from honeyhive.adapters.copilot_studio import copilot_studio_records_to_spans + + +def _make_record( + *, + type_: str = "AppEvents", + operation_id: str = "a" * 32, + id_: str = "b" * 16, + parent_id: str | None = None, + time: str = "2024-01-15T10:30:00.0000000Z", + duration_ms: float = 150.0, + name: str = "BotMessageSend", + app_role_name: str = "MyBot", + app_role_instance: str = "instance-1", + user_id: str | None = None, + session_id: str | None = None, + properties: dict[str, object] | None = None, +) -> dict[str, object]: + # All Log Analytics columns are at the top level in the Azure Monitor diagnostic + # export format. "Properties" (capital P) is a JSON dict object in the Event Hub export. + record: dict[str, object] = { + "Type": type_, + "OperationId": operation_id, + "Id": id_, + "time": time, + "DurationMs": duration_ms, + "Name": name, + "AppRoleName": app_role_name, + "AppRoleInstance": app_role_instance, + "Properties": properties or {}, + } + if parent_id is not None: + record["ParentId"] = parent_id + if user_id is not None: + record["UserId"] = user_id + if session_id is not None: + record["SessionId"] = session_id + return record + + +class TestTypeFiltering: + def test_app_events_passes(self) -> None: + spans = copilot_studio_records_to_spans([_make_record(type_="AppEvents")]) + assert len(spans) == 1 + + def test_app_requests_passes(self) -> None: + spans = copilot_studio_records_to_spans([_make_record(type_="AppRequests")]) + assert len(spans) == 1 + + def test_app_dependencies_passes(self) -> None: + spans = copilot_studio_records_to_spans([_make_record(type_="AppDependencies")]) + assert len(spans) == 1 + + def test_app_exceptions_skipped(self) -> None: + spans = copilot_studio_records_to_spans([_make_record(type_="AppExceptions")]) + assert spans == [] + + def test_app_traces_skipped(self) -> None: + spans = copilot_studio_records_to_spans([_make_record(type_="AppTraces")]) + assert spans == [] + + def test_missing_type_skipped(self) -> None: + record = _make_record() + del record["Type"] + spans = copilot_studio_records_to_spans([record]) + assert spans == [] + + def test_empty_input(self) -> None: + spans = copilot_studio_records_to_spans([]) + assert spans == [] + + +class TestBasicSpanShape: + def test_span_name_from_event(self) -> None: + spans = copilot_studio_records_to_spans([_make_record(name="BotMessageSend")]) + assert spans[0].name == "agent_output" + + def test_span_kind_internal(self) -> None: + from opentelemetry.trace import SpanKind + + spans = copilot_studio_records_to_spans([_make_record()]) + assert spans[0].kind == SpanKind.INTERNAL + + def test_status_ok_by_default(self) -> None: + spans = copilot_studio_records_to_spans([_make_record(name="BotMessageSend")]) + assert spans[0].status.status_code == StatusCode.OK + + def test_status_error_when_name_contains_error(self) -> None: + spans = copilot_studio_records_to_spans([_make_record(name="OnErrorLog")]) + assert spans[0].status.status_code == StatusCode.ERROR + + def test_error_type_attribute_set_on_error_pascalcase(self) -> None: + # Copilot Studio emits ErrorCode/ErrorMessage in PascalCase. + spans = copilot_studio_records_to_spans( + [ + _make_record( + name="OnErrorLog", + properties={ + "ErrorCode": "HttpRequestNetworkError", + "ErrorMessage": "A network error occurred reaching the target.", + }, + ) + ] + ) + assert spans[0].attributes is not None + assert spans[0].attributes.get("error.type") == "HttpRequestNetworkError" + assert ( + spans[0].status.description + == "A network error occurred reaching the target." + ) + + def test_error_type_attribute_set_on_error_camelcase_fallback(self) -> None: + spans = copilot_studio_records_to_spans( + [_make_record(name="OnErrorLog", properties={"errorCode": "BOT_ERR_42"})] + ) + assert spans[0].attributes is not None + assert spans[0].attributes.get("error.type") == "BOT_ERR_42" + + def test_error_type_fallback_unknown(self) -> None: + spans = copilot_studio_records_to_spans([_make_record(name="OnErrorLog")]) + assert spans[0].attributes is not None + assert spans[0].attributes.get("error.type") == "unknown" + + def test_enduser_id_from_top_level_user_id(self) -> None: + spans = copilot_studio_records_to_spans([_make_record(user_id="msteams29:abc")]) + assert spans[0].attributes is not None + assert spans[0].attributes.get("enduser.id") == "msteams29:abc" + + def test_enduser_id_falls_back_to_from_id(self) -> None: + spans = copilot_studio_records_to_spans( + [ + _make_record( + name="BotMessageReceived", properties={"fromId": "29:sender"} + ) + ] + ) + assert spans[0].attributes is not None + assert spans[0].attributes.get("enduser.id") == "29:sender" + + def test_session_id_from_top_level_session_id(self) -> None: + spans = copilot_studio_records_to_spans( + [_make_record(session_id="sess-hash==")] + ) + assert spans[0].attributes is not None + assert spans[0].attributes.get("session.id") == "sess-hash==" + + def test_user_name_from_from_name(self) -> None: + spans = copilot_studio_records_to_spans( + [ + _make_record( + name="BotMessageReceived", properties={"fromName": "Mike Arndt"} + ) + ] + ) + assert spans[0].attributes is not None + assert spans[0].attributes.get("copilot_studio.user_name") == "Mike Arndt" + + def test_gen_ai_system_attribute(self) -> None: + spans = copilot_studio_records_to_spans([_make_record()]) + assert spans[0].attributes is not None + assert spans[0].attributes["gen_ai.system"] == "microsoft.copilot_studio" + + def test_gen_ai_agent_name_uses_instance_over_role(self) -> None: + # AppRoleInstance is the bot's name; AppRoleName is the generic runtime ("Microsoft Copilot Studio") + spans = copilot_studio_records_to_spans( + [ + _make_record( + app_role_name="Microsoft Copilot Studio", app_role_instance="My Bot" + ) + ] + ) + assert spans[0].attributes is not None + assert spans[0].attributes["gen_ai.agent.name"] == "My Bot" + + def test_gen_ai_agent_name_falls_back_to_role_when_no_instance(self) -> None: + record = _make_record(app_role_name="FallbackBot", app_role_instance="") + spans = copilot_studio_records_to_spans([record]) + assert spans[0].attributes is not None + assert spans[0].attributes["gen_ai.agent.name"] == "FallbackBot" + + def test_empty_properties_not_included(self) -> None: + spans = copilot_studio_records_to_spans([_make_record(properties={})]) + assert spans[0].attributes is not None + for val in spans[0].attributes.values(): + assert val != "" + + def test_copilot_properties_mapped(self) -> None: + spans = copilot_studio_records_to_spans( + [ + _make_record( + properties={ + "conversationId": "conv-123", + "TopicName": "Welcome Topic", + "channelId": "msteams", + "DesignMode": "False", + } + ) + ] + ) + attrs = spans[0].attributes + assert attrs is not None + assert attrs["gen_ai.conversation.id"] == "conv-123" + assert attrs["copilot_studio.topic_name"] == "Welcome Topic" + assert attrs["copilot_studio.channel_id"] == "msteams" + assert attrs["copilot_studio.design_mode"] == "False" + + +class TestIdHandling: + def test_valid_hex_trace_id_passthrough(self) -> None: + hex_id = "deadbeef" * 4 # 32 hex chars + spans = copilot_studio_records_to_spans([_make_record(operation_id=hex_id)]) + assert spans[0].context is not None + assert spans[0].context.trace_id == int(hex_id, 16) + + def test_valid_hex_span_id_passthrough(self) -> None: + hex_id = "cafebabe" * 2 # 16 hex chars + spans = copilot_studio_records_to_spans([_make_record(id_=hex_id)]) + assert spans[0].context is not None + assert spans[0].context.span_id == int(hex_id, 16) + + def test_non_hex_trace_id_derived_deterministically(self) -> None: + raw = "fCOhCdCnZ9I=" # base64-style, not hex + spans1 = copilot_studio_records_to_spans([_make_record(operation_id=raw)]) + spans2 = copilot_studio_records_to_spans([_make_record(operation_id=raw)]) + assert spans1[0].context is not None + assert spans2[0].context is not None + assert spans1[0].context.trace_id == spans2[0].context.trace_id + + def test_non_hex_span_id_derived_from_sha256(self) -> None: + raw = "fCOhCdCnZ9I=" + spans = copilot_studio_records_to_spans([_make_record(id_=raw)]) + expected = int.from_bytes( + hashlib.sha256(raw.encode("utf-8")).digest()[:8], "big" + ) + assert spans[0].context is not None + assert spans[0].context.span_id == expected + + def test_parent_child_share_trace_id(self) -> None: + op_id = "00112233445566778899aabbccddeeff" + parent_span_id = "aabbccdd11223344" + child_span_id = "1234567890abcdef" + records = [ + _make_record(operation_id=op_id, id_=parent_span_id), + _make_record( + operation_id=op_id, id_=child_span_id, parent_id=parent_span_id + ), + ] + spans = copilot_studio_records_to_spans(records) + assert len(spans) == 2 + assert spans[0].context is not None + assert spans[1].context is not None + assert spans[0].context.trace_id == spans[1].context.trace_id + + def test_child_parent_id_matches_parent_span_id(self) -> None: + op_id = "00112233445566778899aabbccddeeff" + parent_span_id = "aabbccdd11223344" + records = [ + _make_record(operation_id=op_id, id_=parent_span_id), + _make_record( + operation_id=op_id, id_="1234567890abcdef", parent_id=parent_span_id + ), + ] + spans = copilot_studio_records_to_spans(records) + child = spans[1] + assert child.parent is not None + assert child.parent.span_id == int(parent_span_id, 16) + + def test_non_hex_parent_id_derives_to_same_value_as_non_hex_span_id(self) -> None: + """Non-hex IDs must hash consistently so parent refs resolve.""" + raw_id = "legacyId|abc123" + records = [ + _make_record(operation_id="a" * 32, id_=raw_id), + _make_record(operation_id="a" * 32, id_="b" * 16, parent_id=raw_id), + ] + spans = copilot_studio_records_to_spans(records) + parent_span = spans[0] + child_span = spans[1] + assert parent_span.context is not None + assert child_span.parent is not None + assert child_span.parent.span_id == parent_span.context.span_id + + def test_id_is_never_zero(self) -> None: + spans = copilot_studio_records_to_spans([_make_record(id_="nonhex!!")]) + assert spans[0].context is not None + assert spans[0].context.span_id != 0 + + def test_no_parent_id_produces_root_span(self) -> None: + spans = copilot_studio_records_to_spans([_make_record()]) + assert spans[0].parent is None + + def test_empty_parent_id_produces_root_span(self) -> None: + record = _make_record() + record["ParentId"] = "" + spans = copilot_studio_records_to_spans([record]) + assert spans[0].parent is None + + def test_parent_id_equal_to_operation_id_produces_root_span(self) -> None: + # App Insights sets ParentId == OperationId on root spans; clear it to avoid + # a self-referencing span in OTLP. + op_id = "a" * 32 + record = _make_record(operation_id=op_id) + record["ParentId"] = op_id + spans = copilot_studio_records_to_spans([record]) + assert spans[0].parent is None + + +class TestTimestamps: + def test_start_time_from_iso_z(self) -> None: + from datetime import datetime, timezone + + ts = "2024-01-15T10:30:00.0000000Z" + spans = copilot_studio_records_to_spans([_make_record(time=ts, duration_ms=0)]) + dt = datetime(2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc) + expected_ns = int(dt.timestamp() * 1_000_000_000) + assert spans[0].start_time == expected_ns + + def test_end_time_is_start_plus_duration(self) -> None: + spans = copilot_studio_records_to_spans( + [_make_record(time="2024-01-15T10:30:00.000Z", duration_ms=200.0)] + ) + assert spans[0].end_time is not None + assert spans[0].start_time is not None + assert spans[0].end_time - spans[0].start_time == 200_000_000 + + def test_duration_ms_string_coerced(self) -> None: + record = _make_record(duration_ms=0) + record["DurationMs"] = "250.5" + spans = copilot_studio_records_to_spans([record]) + assert spans[0].end_time is not None + assert spans[0].start_time is not None + assert spans[0].end_time - spans[0].start_time == 250_500_000 + + def test_missing_duration_ms_defaults_zero(self) -> None: + record = _make_record() + del record["DurationMs"] + spans = copilot_studio_records_to_spans([record]) + assert spans[0].end_time == spans[0].start_time + + def test_seven_fractional_digits(self) -> None: + spans = copilot_studio_records_to_spans( + [_make_record(time="2024-01-15T10:30:00.1234567Z", duration_ms=0)] + ) + assert spans[0].start_time is not None + assert spans[0].start_time > 0 + + +class TestSpanEvents: + def test_bot_message_received_adds_user_event(self) -> None: + spans = copilot_studio_records_to_spans( + [_make_record(name="BotMessageReceived", properties={"text": "Hello bot"})] + ) + events = spans[0].events + assert len(events) == 1 + assert events[0].name == "gen_ai.user.message" + assert events[0].attributes is not None + assert events[0].attributes["content"] == "Hello bot" + + def test_bot_message_send_adds_choice_event(self) -> None: + # gen_ai.choice → outputs.content in HoneyHive's StandardGenAI normalizer + spans = copilot_studio_records_to_spans( + [_make_record(name="BotMessageSend", properties={"text": "Hi there!"})] + ) + events = spans[0].events + assert len(events) == 1 + assert events[0].name == "gen_ai.choice" + assert events[0].attributes is not None + assert events[0].attributes["content"] == "Hi there!" + assert events[0].attributes["finish_reason"] == "stop" + + def test_no_text_no_event(self) -> None: + spans = copilot_studio_records_to_spans([_make_record(name="BotMessageSend")]) + assert spans[0].events == () + + +class TestEventNameMapping: + """Verify operation names map correctly to the event names actually emitted by Copilot Studio. + + Event names confirmed from a live test session (June 2025). + """ + + def test_bot_message_received_maps_to_user_input(self) -> None: + spans = copilot_studio_records_to_spans( + [_make_record(name="BotMessageReceived")] + ) + assert spans[0].name == "user_input" + + def test_bot_message_send_maps_to_agent_output(self) -> None: + spans = copilot_studio_records_to_spans([_make_record(name="BotMessageSend")]) + assert spans[0].name == "agent_output" + + def test_topic_start_maps_to_agent_action( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("COPILOT_STUDIO_ADAPTER_KEEP_TOPIC_SPANS", "1") + spans = copilot_studio_records_to_spans([_make_record(name="TopicStart")]) + assert spans[0].name == "agent_action" + + def test_topic_end_maps_to_agent_action( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("COPILOT_STUDIO_ADAPTER_KEEP_TOPIC_SPANS", "1") + spans = copilot_studio_records_to_spans([_make_record(name="TopicEnd")]) + assert spans[0].name == "agent_action" + + def test_topic_action_maps_to_agent_action( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("COPILOT_STUDIO_ADAPTER_KEEP_TOPIC_SPANS", "1") + spans = copilot_studio_records_to_spans([_make_record(name="TopicAction")]) + assert spans[0].name == "agent_action" + + def test_on_error_log_maps_to_agent_error(self) -> None: + spans = copilot_studio_records_to_spans([_make_record(name="OnErrorLog")]) + assert spans[0].name == "agent_error" + + def test_power_virtual_agent_root_maps_to_agent_action( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("COPILOT_STUDIO_ADAPTER_KEEP_TOPIC_SPANS", "1") + spans = copilot_studio_records_to_spans( + [_make_record(name="PowerVirtualAgentRoot")] + ) + assert spans[0].name == "agent_action" + + def test_qualified_topic_event_maps_to_agent_action( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("COPILOT_STUDIO_ADAPTER_KEEP_TOPIC_SPANS", "1") + # e.g. "cr44e_HoneyHiveTestAgent.topic.Greeting" + spans = copilot_studio_records_to_spans( + [_make_record(name="cr44e_MyAgent.topic.Greeting")] + ) + assert spans[0].name == "agent_action" + + def test_qualified_on_error_topic_is_error_status( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("COPILOT_STUDIO_ADAPTER_KEEP_TOPIC_SPANS", "1") + # "cr44e_MyAgent.topic.OnError" contains "error" so it should be StatusCode.ERROR + spans = copilot_studio_records_to_spans( + [_make_record(name="cr44e_MyAgent.topic.OnError")] + ) + assert spans[0].status.status_code == StatusCode.ERROR + + def test_unknown_event_name_defaults_to_agent_action(self) -> None: + spans = copilot_studio_records_to_spans([_make_record(name="SomeFutureEvent")]) + assert spans[0].name == "agent_action" + + +class TestResourceDedup: + def test_same_role_and_instance_share_resource(self) -> None: + records = [ + _make_record(app_role_name="Bot", app_role_instance="i1"), + _make_record(app_role_name="Bot", app_role_instance="i1", id_="c" * 16), + ] + spans = copilot_studio_records_to_spans(records) + assert spans[0].resource is spans[1].resource + + def test_different_instance_different_resource(self) -> None: + records = [ + _make_record(app_role_name="Bot", app_role_instance="i1"), + _make_record(app_role_name="Bot", app_role_instance="i2", id_="c" * 16), + ] + spans = copilot_studio_records_to_spans(records) + assert spans[0].resource is not spans[1].resource + + def test_resource_service_name(self) -> None: + spans = copilot_studio_records_to_spans([_make_record(app_role_name="MyAgent")]) + assert spans[0].resource.attributes["service.name"] == "MyAgent" + + +class TestPropertiesFormat: + """Topic-name normalization and span-ID synthesis for the AppEvents schema.""" + + def test_qualified_topic_name_is_normalized_to_short_form(self) -> None: + # TopicStart records emit TopicName as the FQ form; TopicAction uses the short form. + # Both should resolve to the short name in HoneyHive. + spans = copilot_studio_records_to_spans( + [ + _make_record( + properties={"TopicName": "cr44e_HoneyHiveTestAgent.topic.Greeting"} + ) + ] + ) + assert spans[0].attributes is not None + assert spans[0].attributes["copilot_studio.topic_name"] == "Greeting" + + def test_short_topic_name_unchanged(self) -> None: + spans = copilot_studio_records_to_spans( + [_make_record(properties={"TopicName": "Greeting"})] + ) + assert spans[0].attributes is not None + assert spans[0].attributes["copilot_studio.topic_name"] == "Greeting" + + def test_two_app_events_same_time_name_share_span_id( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("COPILOT_STUDIO_ADAPTER_KEEP_TOPIC_SPANS", "1") + # Deterministic: same time+name → same synthesized span ID + r1 = _make_record(time="2024-01-15T10:30:00.000Z", name="TopicStart") + r2 = _make_record(time="2024-01-15T10:30:00.000Z", name="TopicStart") + del r1["Id"] + del r2["Id"] + spans = copilot_studio_records_to_spans([r1, r2]) + assert spans[0].context is not None + assert spans[1].context is not None + assert spans[0].context.span_id == spans[1].context.span_id + + def test_different_times_produce_different_span_ids( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("COPILOT_STUDIO_ADAPTER_KEEP_TOPIC_SPANS", "1") + r1 = _make_record(time="2024-01-15T10:30:00.000Z", name="TopicStart") + r2 = _make_record(time="2024-01-15T10:30:01.000Z", name="TopicStart") + del r1["Id"] + del r2["Id"] + spans = copilot_studio_records_to_spans([r1, r2]) + assert spans[0].context is not None + assert spans[1].context is not None + assert spans[0].context.span_id != spans[1].context.span_id + + +class TestDesignModeFallback: + """Design-mode test canvas sessions have no OperationId; conversationId is the fallback.""" + + def test_missing_operation_id_with_conversation_id_produces_span(self) -> None: + record = _make_record(properties={"conversationId": "conv-abc-123"}) + record["OperationId"] = "" + spans = copilot_studio_records_to_spans([record]) + assert len(spans) == 1 + + def test_conversation_id_seeds_deterministic_session_id(self) -> None: + conv_id = "54453b3c-d001-4997-b70e-0c7aa9f0ba29" + record = _make_record(properties={"conversationId": conv_id}) + record["OperationId"] = "" + spans = copilot_studio_records_to_spans([record]) + expected_session_id = str(uuid.uuid5(uuid.NAMESPACE_URL, conv_id)) + assert spans[0].attributes is not None + assert spans[0].attributes["honeyhive.session_id"] == expected_session_id + + def test_two_design_mode_spans_with_same_conversation_share_trace(self) -> None: + conv_id = "54453b3c-d001-4997-b70e-0c7aa9f0ba29" + r1 = _make_record(id_="a" * 16, properties={"conversationId": conv_id}) + r2 = _make_record(id_="b" * 16, properties={"conversationId": conv_id}) + r1["OperationId"] = "" + r2["OperationId"] = "" + spans = copilot_studio_records_to_spans([r1, r2]) + assert len(spans) == 2 + assert spans[0].context is not None + assert spans[1].context is not None + assert spans[0].context.trace_id == spans[1].context.trace_id + + +class TestTopicFiltering: + def test_topic_spans_dropped_by_default(self) -> None: + records = [ + _make_record(name="TopicStart"), + _make_record(name="TopicEnd"), + _make_record(name="TopicAction"), + _make_record(name="PowerVirtualAgentRoot"), + _make_record(name="BotMessageSend"), + ] + spans = copilot_studio_records_to_spans(records) + assert len(spans) == 1 + assert spans[0].name == "agent_output" + + def test_qualified_topic_name_dropped_by_default(self) -> None: + spans = copilot_studio_records_to_spans( + [_make_record(name="cr44e_MyAgent.topic.Greeting")] + ) + assert spans == [] + + def test_error_topic_span_not_dropped_by_default(self) -> None: + # Error topics may carry error content not otherwise captured, so they are + # exempt from the default topic drop. + spans = copilot_studio_records_to_spans( + [_make_record(name="cr44e_MyAgent.topic.OnError")] + ) + assert len(spans) == 1 + assert spans[0].status.status_code == StatusCode.ERROR + + def test_topic_spans_retained_when_env_set( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("COPILOT_STUDIO_ADAPTER_KEEP_TOPIC_SPANS", "1") + records = [ + _make_record(name="TopicStart"), + _make_record(name="BotMessageSend"), + ] + spans = copilot_studio_records_to_spans(records) + assert len(spans) == 2 + + def test_unnamed_dependency_always_dropped( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("COPILOT_STUDIO_ADAPTER_KEEP_TOPIC_SPANS", "1") + records = [_make_record(type_="AppDependencies", name="n/a")] + spans = copilot_studio_records_to_spans(records) + assert spans == [] + + def test_message_spans_never_dropped(self) -> None: + records = [ + _make_record(name="BotMessageReceived"), + _make_record(name="BotMessageSend"), + _make_record(name="OnErrorLog"), + ] + spans = copilot_studio_records_to_spans(records) + assert len(spans) == 3 + + +class TestDebugMode: + def test_debug_attribute_absent_by_default(self) -> None: + spans = copilot_studio_records_to_spans([_make_record()]) + assert spans[0].attributes is not None + assert "debug.raw_record" not in spans[0].attributes + + def test_debug_attribute_present_when_env_set( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("COPILOT_STUDIO_ADAPTER_DEBUG", "1") + record = _make_record(properties={"conversationId": "conv-debug"}) + spans = copilot_studio_records_to_spans([record]) + assert spans[0].attributes is not None + raw = spans[0].attributes.get("debug.raw_record") + assert raw is not None + parsed = json.loads(str(raw)) + assert parsed["Properties"]["conversationId"] == "conv-debug" + + def test_debug_attribute_is_valid_json( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("COPILOT_STUDIO_ADAPTER_DEBUG", "1") + spans = copilot_studio_records_to_spans([_make_record()]) + raw = spans[0].attributes.get("debug.raw_record") # type: ignore[union-attr] + assert raw is not None + parsed = json.loads(str(raw)) + assert "Type" in parsed + + +class TestMalformedRecords: + def test_missing_operation_id_and_no_conversation_id_skipped(self) -> None: + record = _make_record() # Properties is {} (no conversationId) + record["OperationId"] = "" + spans = copilot_studio_records_to_spans([record]) + assert spans == [] + + def test_missing_id_synthesizes_from_time_and_name(self) -> None: + # AppEvents has no Id column; the adapter synthesizes a span ID from time+Name. + record = _make_record() + del record["Id"] + spans = copilot_studio_records_to_spans([record]) + assert len(spans) == 1 + assert spans[0].context is not None + assert spans[0].context.span_id != 0 + + def test_missing_time_skipped(self) -> None: + record = _make_record() + del record["time"] + spans = copilot_studio_records_to_spans([record]) + assert spans == [] + + def test_malformed_mixed_with_good_yields_good(self) -> None: + bad = _make_record() + bad["OperationId"] = "" # no conversationId fallback either + good = _make_record(id_="d" * 16) + spans = copilot_studio_records_to_spans([bad, good]) + assert len(spans) == 1 + + def test_all_malformed_returns_empty_list(self) -> None: + bad1 = _make_record() + bad1["OperationId"] = "" + bad2 = _make_record() + del bad2["time"] + spans = copilot_studio_records_to_spans([bad1, bad2]) + assert spans == [] diff --git a/tests/utils.py b/tests/utils.py index fac40d1a..f70e1922 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,9 +1,11 @@ """Test utilities for HoneyHive SDK.""" import os +from pathlib import Path from unittest.mock import Mock, patch import pytest +from dotenv import load_dotenv def create_openai_config_request(project="test-project", name="test-config"): @@ -68,17 +70,17 @@ def setup_test_environment(): os.environ["HH_DISABLE_HTTP_TRACING"] = "true" os.environ["HH_OTLP_ENABLED"] = "false" - # Patch the config module to use test values - try: - from honeyhive.utils.config import ( # pylint: disable=import-outside-toplevel - config, - ) - - # Reset the config to use default values - config.api_url = "https://api.dp1.us.honeyhive.ai" - except ImportError: - # Config module doesn't exist or has changed - this is expected - pass + # load .env files if they exist + project_root = Path(__file__).parent.parent + + # Try to load .env files in priority order + for env_file in [ + project_root / ".env.integration", # Integration-specific + project_root / ".env", # General project_root + ]: + if env_file.exists(): + load_dotenv(env_file, override=True) + print(f"✅ Loaded environment from: {env_file}") def cleanup_test_environment(): diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index 1d780fad..b3435470 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -15,15 +15,6 @@ from .backend_verification import ( # pylint: disable=wrong-import-position verify_backend_event, ) -from .env_enforcement import ( # pylint: disable=wrong-import-position - EnvFileNotFoundError, - EnvironmentEnforcer, - MissingCredentialsError, - enforce_integration_credentials, - enforce_local_env_file, - get_llm_credentials, - print_env_status, -) from .otel_reset import ( # pylint: disable=wrong-import-position debug_otel_state, ensure_clean_otel_state, @@ -115,13 +106,6 @@ def test_error_handling_common( __all__ = [ - "EnvironmentEnforcer", - "EnvFileNotFoundError", - "MissingCredentialsError", - "enforce_local_env_file", - "enforce_integration_credentials", - "get_llm_credentials", - "print_env_status", "setup_test_environment", "cleanup_test_environment", "create_openai_config_request", diff --git a/tests/utils/env_enforcement.py b/tests/utils/env_enforcement.py deleted file mode 100644 index 32b84e43..00000000 --- a/tests/utils/env_enforcement.py +++ /dev/null @@ -1,311 +0,0 @@ -""" -Environment Variable Enforcement for Local Development - -This module provides programmatic enforcement for detecting and sourcing -.env files in local development environments. -""" - -import os -import sys -from pathlib import Path -from typing import Dict, List, Optional - -from dotenv import load_dotenv - - -class EnvFileNotFoundError(Exception): - """Raised when required .env file is not found in local development.""" - - -class MissingCredentialsError(Exception): - """Raised when required credentials are missing from environment.""" - - -class EnvironmentEnforcer: - """Enforces .env file loading and credential validation for local development.""" - - def __init__(self, project_root: Optional[Path] = None): - """Initialize the environment enforcer. - - Args: - project_root: Path to project root. If None, auto-detects from current file. - """ - if project_root is None: - # Auto-detect project root (look for pyproject.toml) - current_path = Path(__file__).resolve() - for parent in current_path.parents: - if (parent / "pyproject.toml").exists(): - project_root = parent - break - else: - raise RuntimeError( - "Could not find project root (no pyproject.toml found)" - ) - - self.project_root = project_root - self.env_files = [ - self.project_root / ".env.integration", # Integration-specific - self.project_root / ".env", # General project - ] - self.loaded_env_file: Optional[Path] = None - - def is_local_development(self) -> bool: - """Detect if we're running in local development environment. - - Returns: - True if running locally, False if in CI/production. - """ - # CI environment indicators - ci_indicators = [ - "CI", - "GITHUB_ACTIONS", - "GITLAB_CI", - "JENKINS_URL", - "TRAVIS", - "CIRCLECI", - "BUILDKITE", - "AZURE_PIPELINES", - ] - - # Check if any CI indicator is present - if any(os.getenv(indicator) for indicator in ci_indicators): - return False - - # Check if HH_SOURCE indicates CI environment - hh_source = os.getenv("HH_SOURCE", "") - if hh_source.startswith(("github-actions", "ci-", "pipeline-")): - return False - - # Check if we're in a tox environment (but still local) - if os.getenv("TOX_ENV_NAME"): - # Tox is local development, but check if it's CI-triggered - return not any(os.getenv(indicator) for indicator in ci_indicators) - - return True - - def detect_and_load_env_file(self) -> bool: - """Detect and load .env file for local development. - - Returns: - True if .env file was found and loaded, False otherwise. - - Raises: - EnvFileNotFoundError: If no .env file found in local development. - """ - if not self.is_local_development(): - # In CI/production, don't require .env files - return False - - # Try to load .env files in priority order - for env_file in self.env_files: - if env_file.exists(): - load_dotenv(env_file, override=True) - self.loaded_env_file = env_file - print(f"✅ Loaded environment from: {env_file}") - return True - - # No .env file found in local development - this is an error - env_file_paths = "\n".join(f" - {path}" for path in self.env_files) - example_file = self.project_root / "env.integration.example" - - error_msg = f""" -🚨 LOCAL DEVELOPMENT ERROR: No .env file found! - -Local development MUST use .env files for credentials. - -Expected .env file locations: -{env_file_paths} - -To fix this: -1. Copy the example file: - cp {example_file} {self.env_files[0]} - -2. Edit {self.env_files[0]} with your real credentials: - HH_API_KEY=your_honeyhive_api_key_here - HH_PROJECT=your_project_name_here - OPENAI_API_KEY=your_openai_key_here # (optional, for LLM tests) - -3. Never commit .env files to git (they're in .gitignore) - -For CI/production environments, set environment variables directly. -""" - raise EnvFileNotFoundError(error_msg.strip()) - - def validate_required_credentials(self, required_vars: List[str]) -> Dict[str, str]: - """Validate that required environment variables are present. - - Args: - required_vars: List of required environment variable names. - - Returns: - Dictionary of variable names to values. - - Raises: - MissingCredentialsError: If required variables are missing. - """ - missing_vars = [] - credentials = {} - - for var_name in required_vars: - value = os.getenv(var_name) - if not value: - missing_vars.append(var_name) - else: - credentials[var_name] = value - - if missing_vars: - env_file_info = "" - if self.loaded_env_file: - env_file_info = f"\nLoaded from: {self.loaded_env_file}" - elif self.is_local_development(): - env_file_info = ( - "\nNo .env file was loaded (see detect_and_load_env_file())" - ) - - missing_list = "\n".join(f" - {var}" for var in missing_vars) - error_msg = f""" -🚨 MISSING REQUIRED CREDENTIALS: - -The following environment variables are required: -{missing_list} -{env_file_info} - -For local development, add these to your .env file: -{chr(10).join(f"{var}=your_{var.lower()}_here" for var in missing_vars)} - -For CI/production, set these environment variables directly. -""" - raise MissingCredentialsError(error_msg.strip()) - - return credentials - - def enforce_integration_test_credentials(self) -> Dict[str, str]: - """Enforce credentials required for integration tests. - - Returns: - Dictionary of validated credentials. - - Raises: - EnvFileNotFoundError: If .env file missing in local development. - MissingCredentialsError: If required credentials are missing. - """ - # Always try to load .env file in local development - self.detect_and_load_env_file() - - # Core required credentials for integration tests - required_vars = ["HH_API_KEY"] - - # Validate and return credentials - return self.validate_required_credentials(required_vars) - - def get_optional_llm_credentials(self) -> Dict[str, Optional[str]]: - """Get optional LLM provider credentials for instrumentor tests. - - Returns: - Dictionary of LLM provider credentials (may contain None values). - """ - llm_vars = [ - "OPENAI_API_KEY", - "ANTHROPIC_API_KEY", - "GOOGLE_API_KEY", - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - "AZURE_OPENAI_API_KEY", - ] - - return {var: os.getenv(var) for var in llm_vars} - - def print_environment_status(self) -> None: - """Print current environment status for debugging.""" - print("\n" + "=" * 60) - print("🔍 ENVIRONMENT STATUS") - print("=" * 60) - - print(f"Local Development: {self.is_local_development()}") - print(f"Project Root: {self.project_root}") - - if self.loaded_env_file: - print(f"Loaded .env file: {self.loaded_env_file}") - else: - print("No .env file loaded") - - # Show key environment variables (without exposing secrets) - key_vars = ["HH_API_KEY", "HH_PROJECT", "HH_SOURCE", "OPENAI_API_KEY"] - print("\nKey Environment Variables:") - for var in key_vars: - value = os.getenv(var) - if value: - # Show first 8 chars + "..." for security - masked_value = f"{value[:8]}..." if len(value) > 8 else "***" - print(f" {var}: {masked_value}") - else: - print(f" {var}: (not set)") - - print("=" * 60 + "\n") - - -# Global instance for easy access -_enforcer = EnvironmentEnforcer() - - -def enforce_local_env_file() -> bool: - """Convenience function to enforce .env file loading in local development. - - Returns: - True if .env file was loaded, False if not needed (CI/production). - - Raises: - EnvFileNotFoundError: If .env file missing in local development. - """ - return _enforcer.detect_and_load_env_file() - - -def enforce_integration_credentials() -> Dict[str, str]: - """Convenience function to enforce integration test credentials. - - Returns: - Dictionary of validated credentials. - - Raises: - EnvFileNotFoundError: If .env file missing in local development. - MissingCredentialsError: If required credentials are missing. - """ - return _enforcer.enforce_integration_test_credentials() - - -def get_llm_credentials() -> Dict[str, Optional[str]]: - """Convenience function to get optional LLM credentials. - - Returns: - Dictionary of LLM provider credentials (may contain None values). - """ - return _enforcer.get_optional_llm_credentials() - - -def print_env_status() -> None: - """Convenience function to print environment status.""" - _enforcer.print_environment_status() - - -if __name__ == "__main__": - try: - print("Testing Environment Enforcement...") - print_env_status() - - print("Enforcing .env file loading...") - enforce_local_env_file() - - print("Enforcing integration credentials...") - creds = enforce_integration_credentials() - print(f"✅ Found {len(creds)} required credentials") - - print("Checking optional LLM credentials...") - llm_creds = get_llm_credentials() - available_llm = [k for k, v in llm_creds.items() if v] - print( - f"✅ Found {len(available_llm)} LLM provider credentials: {available_llm}" - ) - - except (EnvFileNotFoundError, MissingCredentialsError) as e: - print(f"❌ {e}") - sys.exit(1)