Skip to content

Move patched evaluation TypedDicts to the azure.ai.projects.types namespace #48159

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dargilco wants to merge 3 commits into
base: feature/azure-ai-projects/vnext
Choose a base branch
from
Open

Move patched evaluation TypedDicts to the azure.ai.projects.types namespace #48159

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5ad5894
first
dargilco Jul 21, 2026
ffc544a
Remove duplicate TypedDicts
dargilco Jul 21, 2026
2d6126e
Fix broken import in _patch.pyi
dargilco Jul 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions sdk/ai/azure-ai-projects/azure/ai/projects/_patch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import os
import re
import logging
import importlib
from typing import List, Any, Optional
import httpx # pylint: disable=networking-import-outside-azure-core-transport
from openai import OpenAI
Expand Down Expand Up @@ -409,3 +410,8 @@ def patch_sdk():
you can't accomplish using the techniques described in
https://aka.ms/azsdk/python/dpcodegen/python/customize
"""
types_module = importlib.import_module(f"{__package__}.types")
patch_types_module = importlib.import_module(f"{__package__}._patch_types")

for name in getattr(patch_types_module, "__all__", []):
setattr(types_module, name, getattr(patch_types_module, name))

@dargilco dargilco Jul 21, 2026 •
edited
Loading

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The above is the run-time hack to export the hand-written TypedDicts into the "azure.ai.projects.typed" namespace. But since it's run-time, there won't be any typing hints...

2 changes: 1 addition & 1 deletion sdk/ai/azure-ai-projects/azure/ai/projects/_patch.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ from openai.types.eval_create_response import EvalCreateResponse
from openai.types.shared_params.metadata import Metadata
from ._client import AIProjectClient as AIProjectClientGenerated
from .operations import TelemetryOperations
from .models import (
from ._patch_types import (
AzureAIBenchmarkPreviewEvalRunDataSource,
AzureAIDataSourceConfig,
AzureAIResponsesEvalRunDataSource,
Expand Down
96 changes: 25 additions & 71 deletions ...ts/models/_patch_evaluation_typeddicts.py → ...rojects/azure/ai/projects/_patch_types.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,81 +5,21 @@
# ------------------------------------

import datetime
from typing import Dict, Any, List, Union
from typing import Any, Dict, List, Union
from typing_extensions import Literal, Required, TypedDict
from openai.types.evals.create_eval_completions_run_data_source_param import (
InputMessagesItemReference,
SourceFileContent,
SourceFileID,
)

# **************************************************************************************************
# BEGIN - These are duplicates of full classes implementation in _models.py. Redefined here as TypedDicts
# with "Param" suffix, so they can be used in type annotations for "openai_client.evals" operations
# **************************************************************************************************


# Note all properties on this class where required. Sample code suggest they are all optional. Update it here.
class ModelSamplingConfigParam(TypedDict, total=False):
"""Represents a set of parameters used to control the sampling behavior of a language model
during text generation.
"""

temperature: float
"""The temperature parameter for sampling. Required."""
top_p: float
"""The top-p parameter for nucleus sampling. Required."""
seed: int
"""The random seed for reproducibility. Required."""
max_completion_tokens: int
"""The maximum number of tokens allowed in the completion. Required."""


class ToolDescriptionParam(TypedDict, total=False):
"""Description of a tool that can be used by an agent."""

name: str
"""The name of the tool."""
description: str
"""A brief description of the tool's purpose."""


class AzureAIAgentTargetParam(TypedDict, total=False):
"""Represents a target specifying an Azure AI agent."""

type: Required[Literal["azure_ai_agent"]]
"""The type of target, always ``azure_ai_agent``. Required. Default value is \"azure_ai_agent\"."""
name: Required[str]
"""The unique identifier of the Azure AI agent. Required."""
version: str
"""The version of the Azure AI agent."""
tool_descriptions: List[ToolDescriptionParam]
"""The parameters used to control the sampling behavior of the agent during text generation."""


class AzureAIModelTargetParam(TypedDict, total=False):
"""Represents a target specifying an Azure AI model for operations requiring model selection."""

type: Required[Literal["azure_ai_model"]]
"""The type of target, always ``azure_ai_model``. Required. Default value is \"azure_ai_model\"."""
model: str
"""The unique identifier of the Azure AI model."""
sampling_params: ModelSamplingConfigParam
"""The parameters used to control the sampling behavior of the model during text generation."""


# *************************************************************************************************
# END - Typed re-definitions
# *************************************************************************************************


class ResponseRetrievalItemGenerationParams(TypedDict, total=False):
"""Represents the parameters for response retrieval item generation."""

type: Required[Literal["response_retrieval"]]
"""The type of item generation parameters, always ``response_retrieval``. Required. The
ResponseRetrieval item generation parameters."""
max_num_turns: int # Required[int] # TODO: In TypeSpec this is required, but sample code does not set it
max_num_turns: int
"""The maximum number of turns of chat history to evaluate. Required."""
data_mapping: Required[Dict[str, str]]
"""Mapping from source fields to response_id field, required for retrieving chat history.
Expand All @@ -99,9 +39,9 @@ class AzureAIResponsesEvalRunDataSource(TypedDict, total=False):
\"azure_ai_responses\"."""
item_generation_params: Required[ResponseRetrievalItemGenerationParams]
"""The parameters for item generation. Required."""
max_runs_hourly: int # Required[int] # TODO: In TypeSpec this is required, but sample code does not set it
max_runs_hourly: int
"""Maximum number of evaluation runs allowed per hour. Required."""
event_configuration_id: str # Required[str] # TODO: In TypeSpec this is required, but sample code does not set it
event_configuration_id: str
"""The event configuration name associated with this evaluation run. Required."""


Expand All @@ -111,7 +51,7 @@ class AzureAIDataSourceConfig(TypedDict, total=False):
type: Required[Literal["azure_ai_source"]]
"""The object type, which is always ``azure_ai_source``. Required. Default value is
\"azure_ai_source\"."""
scenario: Required[str] # TODO: Update typespec to define the below strings as enum
scenario: Required[str]
"""Data schema scenario. Required. Is one of the following types: Literal[\"red_team\"],
Literal[\"responses\"], Literal[\"traces_preview\"], Literal[\"synthetic_data_gen_preview\"],
Literal[\"benchmark_preview\"]"""
Expand All @@ -126,7 +66,7 @@ class TargetCompletionEvalRunDataSource(TypedDict, total=False):
source: Required[Union[SourceFileContent, SourceFileID]]
"""The source configuration for inline or file data. Required. Is either a
SourceFileContent type or a SourceFileID type."""
target: Required[Union[AzureAIAgentTargetParam, AzureAIModelTargetParam, dict[str, Any]]]
target: Required[Union[AzureAIAgentTarget, AzureAIModelTarget, dict[str, Any]]]
"""The target configuration for the evaluation. Required."""
input_messages: Required[InputMessagesItemReference]
"""Input messages configuration."""
Expand Down Expand Up @@ -156,11 +96,11 @@ class AzureAIBenchmarkPreviewEvalRunDataSource(TypedDict, total=False):
type: Required[Literal["azure_ai_benchmark_preview"]]
"""The type of data source, always ``azure_ai_benchmark_preview``. Required. Default value is
\"azure_ai_benchmark_preview\"."""
target: Required[Union[AzureAIModelTargetParam, AzureAIAgentTargetParam, dict[str, Any]]]
target: Required[Union[AzureAIModelTarget, AzureAIAgentTarget, dict[str, Any]]]
"""The target model or agent to evaluate against the benchmark. When using ``azure_ai_model``
target, ``sampling_params`` must not be provided; inference parameters are auto-filled from the
benchmark specification stored in eval group properties. Required. Is either a
AzureAIModelTargetParam type or a AzureAIAgentTargetParam type."""
AzureAIModelTarget type or a AzureAIAgentTarget type."""
input_messages: InputMessagesItemReference
"""Input messages configuration."""

Expand All @@ -179,7 +119,7 @@ class EvalCsvRunDataSource(TypedDict, total=False):

type: Required[Literal["csv"]]
"""The type of data source, always ``csv``. Required. Default value is \"csv\"."""
source: Required[EvalCsvFileIdSource] # EvalCsvFileIdSource
source: Required[EvalCsvFileIdSource]
"""The source of the CSV data, either inline content or a file reference. Required."""


Expand All @@ -189,9 +129,9 @@ class RedTeamEvalRunDataSource(TypedDict, total=False):
type: Required[Literal["azure_ai_red_team"]]
"""The type of data source. Always ``azure_ai_red_team``. Required. Default value is
\"azure_ai_red_team\"."""
item_generation_params: Required[Any] # ItemGenerationParams
item_generation_params: Required[Any]
"""The parameters for item generation. Required."""
target: Required[Union[AzureAIModelTargetParam, AzureAIAgentTargetParam, dict[str, Any]]]
target: Required[Union[AzureAIModelTarget, AzureAIAgentTarget, dict[str, Any]]]
"""The target configuration for the evaluation. Required."""


Expand Down Expand Up @@ -219,3 +159,17 @@ class TracesPreviewEvalRunDataSource(TypedDict, total=False):
"""Sampling limit applied to traces retrieved for evaluation."""
ingestion_delay_seconds: int
"""The delay to apply for ingestion when querying traces."""


__all__ = [
"AzureAIBenchmarkPreviewEvalRunDataSource",
"AzureAIDataSourceConfig",
"AzureAIResponsesEvalRunDataSource",
"EvalCsvFileIdSource",
"EvalCsvRunDataSource",
"TestingCriterionAzureAIEvaluator",
"RedTeamEvalRunDataSource",
"ResponseRetrievalItemGenerationParams",
"TargetCompletionEvalRunDataSource",
"TracesPreviewEvalRunDataSource",
]
2 changes: 1 addition & 1 deletion sdk/ai/azure-ai-projects/azure/ai/projects/aio/_patch.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ from openai.types.eval_create_response import EvalCreateResponse
from openai.types.shared_params.metadata import Metadata
from ._client import AIProjectClient as AIProjectClientGenerated
from .operations import TelemetryOperations
from ..models import (
from .._patch_types import (
AzureAIBenchmarkPreviewEvalRunDataSource,
AzureAIDataSourceConfig,
AzureAIResponsesEvalRunDataSource,
Expand Down
30 changes: 0 additions & 30 deletions sdk/ai/azure-ai-projects/azure/ai/projects/models/_patch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,22 +16,6 @@
_raise_if_bad_http_status_and_method,
)
from azure.core.polling.async_base_polling import AsyncLROBasePolling
from ._patch_evaluation_typeddicts import (
AzureAIAgentTargetParam,
AzureAIBenchmarkPreviewEvalRunDataSource,
AzureAIDataSourceConfig,
AzureAIModelTargetParam,
AzureAIResponsesEvalRunDataSource,
EvalCsvFileIdSource,
EvalCsvRunDataSource,
TestingCriterionAzureAIEvaluator,
ModelSamplingConfigParam,
RedTeamEvalRunDataSource,
ResponseRetrievalItemGenerationParams,
TargetCompletionEvalRunDataSource,
ToolDescriptionParam,
TracesPreviewEvalRunDataSource,
)
from ._models import CustomCredential as CustomCredentialGenerated
from ..models import MemoryStoreUpdateCompletedResult, MemoryStoreUpdateResult
from ._enums import _FoundryFeaturesOptInKeys, _AgentDefinitionOptInKeys
Expand Down Expand Up @@ -382,21 +366,7 @@ def from_continuation_token(

__all__: List[str] = [
"AsyncUpdateMemoriesLROPoller",
"AzureAIAgentTargetParam",
"AzureAIBenchmarkPreviewEvalRunDataSource",
"AzureAIDataSourceConfig",
"AzureAIModelTargetParam",
"AzureAIResponsesEvalRunDataSource",
"CustomCredential",
"EvalCsvFileIdSource",
"EvalCsvRunDataSource",
"TestingCriterionAzureAIEvaluator",
"ModelSamplingConfigParam",
"RedTeamEvalRunDataSource",
"ResponseRetrievalItemGenerationParams",
"TargetCompletionEvalRunDataSource",
"ToolDescriptionParam",
"TracesPreviewEvalRunDataSource",
"UpdateMemoriesLROPoller",
] # Add all objects you want publicly available to users at this package level

Expand Down
2 changes: 1 addition & 1 deletion ...i-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,8 @@
JobStatus,
PromptDataGenerationJobSource,
SimpleQnADataGenerationJobOptions,
TestingCriterionAzureAIEvaluator,
)
from azure.ai.projects.types import TestingCriterionAzureAIEvaluator

load_dotenv()

Expand Down
2 changes: 1 addition & 1 deletion sdk/ai/azure-ai-projects/samples/evaluations/agentic_evaluators/sample_coherence.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
from openai.types.eval_create_params import DataSourceConfigCustom
from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import TestingCriterionAzureAIEvaluator
from azure.ai.projects.types import TestingCriterionAzureAIEvaluator

load_dotenv()

Expand Down
2 changes: 1 addition & 1 deletion sdk/ai/azure-ai-projects/samples/evaluations/agentic_evaluators/sample_fluency.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
from openai.types.eval_create_params import DataSourceConfigCustom
from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import TestingCriterionAzureAIEvaluator
from azure.ai.projects.types import TestingCriterionAzureAIEvaluator

load_dotenv()

Expand Down
2 changes: 1 addition & 1 deletion ...ts/samples/evaluations/agentic_evaluators/sample_generic_agentic_evaluator/agent_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from openai.types.eval_create_params import DataSourceConfigCustom
from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import TestingCriterionAzureAIEvaluator
from azure.ai.projects.types import TestingCriterionAzureAIEvaluator

load_dotenv()

Expand Down
2 changes: 1 addition & 1 deletion sdk/ai/azure-ai-projects/samples/evaluations/agentic_evaluators/sample_groundedness.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
from openai.types.eval_create_params import DataSourceConfigCustom
from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import TestingCriterionAzureAIEvaluator
from azure.ai.projects.types import TestingCriterionAzureAIEvaluator

load_dotenv()

Expand Down
2 changes: 1 addition & 1 deletion sdk/ai/azure-ai-projects/samples/evaluations/agentic_evaluators/sample_intent_resolution.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
from openai.types.eval_create_params import DataSourceConfigCustom
from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import TestingCriterionAzureAIEvaluator
from azure.ai.projects.types import TestingCriterionAzureAIEvaluator

load_dotenv()

Expand Down
2 changes: 1 addition & 1 deletion sdk/ai/azure-ai-projects/samples/evaluations/agentic_evaluators/sample_quality_grader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
from openai.types.eval_create_params import DataSourceConfigCustom
from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import TestingCriterionAzureAIEvaluator
from azure.ai.projects.types import TestingCriterionAzureAIEvaluator

load_dotenv()

Expand Down
2 changes: 1 addition & 1 deletion sdk/ai/azure-ai-projects/samples/evaluations/agentic_evaluators/sample_relevance.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
from openai.types.eval_create_params import DataSourceConfigCustom
from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import TestingCriterionAzureAIEvaluator
from azure.ai.projects.types import TestingCriterionAzureAIEvaluator

load_dotenv()

Expand Down
2 changes: 1 addition & 1 deletion .../azure-ai-projects/samples/evaluations/agentic_evaluators/sample_response_completeness.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
from openai.types.eval_create_params import DataSourceConfigCustom
from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import TestingCriterionAzureAIEvaluator
from azure.ai.projects.types import TestingCriterionAzureAIEvaluator

load_dotenv()

Expand Down
2 changes: 1 addition & 1 deletion sdk/ai/azure-ai-projects/samples/evaluations/agentic_evaluators/sample_task_adherence.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
from openai.types.eval_create_params import DataSourceConfigCustom
from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import TestingCriterionAzureAIEvaluator
from azure.ai.projects.types import TestingCriterionAzureAIEvaluator

load_dotenv()

Expand Down
2 changes: 1 addition & 1 deletion sdk/ai/azure-ai-projects/samples/evaluations/agentic_evaluators/sample_task_completion.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
from openai.types.eval_create_params import DataSourceConfigCustom
from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import TestingCriterionAzureAIEvaluator
from azure.ai.projects.types import TestingCriterionAzureAIEvaluator

load_dotenv()

Expand Down
2 changes: 1 addition & 1 deletion ...e-ai-projects/samples/evaluations/agentic_evaluators/sample_task_navigation_efficiency.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
from openai.types.eval_create_params import DataSourceConfigCustom
from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import TestingCriterionAzureAIEvaluator
from azure.ai.projects.types import TestingCriterionAzureAIEvaluator

load_dotenv()

Expand Down
2 changes: 1 addition & 1 deletion sdk/ai/azure-ai-projects/samples/evaluations/agentic_evaluators/sample_tool_call_accuracy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
from openai.types.eval_create_params import DataSourceConfigCustom
from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import TestingCriterionAzureAIEvaluator
from azure.ai.projects.types import TestingCriterionAzureAIEvaluator

load_dotenv()

Expand Down
2 changes: 1 addition & 1 deletion sdk/ai/azure-ai-projects/samples/evaluations/agentic_evaluators/sample_tool_call_success.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
)
from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import TestingCriterionAzureAIEvaluator
from azure.ai.projects.types import TestingCriterionAzureAIEvaluator

load_dotenv()

Expand Down
2 changes: 1 addition & 1 deletion ...ai/azure-ai-projects/samples/evaluations/agentic_evaluators/sample_tool_input_accuracy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
from openai.types.eval_create_params import DataSourceConfigCustom
from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import TestingCriterionAzureAIEvaluator
from azure.ai.projects.types import TestingCriterionAzureAIEvaluator

load_dotenv()

Expand Down
2 changes: 1 addition & 1 deletion ...zure-ai-projects/samples/evaluations/agentic_evaluators/sample_tool_output_utilization.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
from openai.types.eval_create_params import DataSourceConfigCustom
from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import TestingCriterionAzureAIEvaluator
from azure.ai.projects.types import TestingCriterionAzureAIEvaluator

load_dotenv()

Expand Down
Loading
Loading