fix: GenAI Client(evals) - Reformat codebase 1. Remove duplicated code in _evals_utils and _evals_metric_loader 2. Keep metric utils in _evals_metric_loader and data util in _evals_utils

PiperOrigin-RevId: 833893675

Author: vertex-sdk-bot

tests/unit/vertexai/genai/test_evals.py

@@ -287,7 +287,7 @@ def setup_method(self):
         self.client = vertexai.Client(project=_TEST_PROJECT, location=_TEST_LOCATION)
 
     @mock.patch.object(_evals_common, "Models")
-    @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader")
+    @mock.patch.object(_evals_utils, "EvalDatasetLoader")
     def test_inference_with_string_model_success(
         self, mock_eval_dataset_loader, mock_models
     ):
@@ -330,7 +330,7 @@ def test_inference_with_string_model_success(
         assert inference_result.candidate_name == "gemini-pro"
         assert inference_result.gcs_source is None
 
-    @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader")
+    @mock.patch.object(_evals_utils, "EvalDatasetLoader")
     def test_inference_with_callable_model_sets_candidate_name(
         self, mock_eval_dataset_loader
     ):
@@ -349,7 +349,7 @@ def my_model_fn(contents):
         assert inference_result.candidate_name == "my_model_fn"
         assert inference_result.gcs_source is None
 
-    @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader")
+    @mock.patch.object(_evals_utils, "EvalDatasetLoader")
     def test_inference_with_lambda_model_candidate_name_is_none(
         self, mock_eval_dataset_loader
     ):
@@ -371,7 +371,7 @@ def test_inference_with_lambda_model_candidate_name_is_none(
         )
         assert inference_result.gcs_source is None
 
-    @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader")
+    @mock.patch.object(_evals_utils, "EvalDatasetLoader")
     def test_inference_with_callable_model_success(self, mock_eval_dataset_loader):
         mock_df = pd.DataFrame({"prompt": ["test prompt"]})
         mock_eval_dataset_loader.return_value.load.return_value = mock_df.to_dict(
@@ -399,7 +399,7 @@ def mock_model_fn(contents):
         assert inference_result.gcs_source is None
 
     @mock.patch.object(_evals_common, "Models")
-    @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader")
+    @mock.patch.object(_evals_utils, "EvalDatasetLoader")
     def test_inference_with_prompt_template(
         self, mock_eval_dataset_loader, mock_models
     ):
@@ -446,7 +446,7 @@ def test_inference_with_prompt_template(
         assert inference_result.gcs_source is None
 
     @mock.patch.object(_evals_common, "Models")
-    @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader")
+    @mock.patch.object(_evals_utils, "EvalDatasetLoader")
     @mock.patch.object(_gcs_utils, "GcsUtils")
     def test_inference_with_gcs_destination(
         self, mock_gcs_utils, mock_eval_dataset_loader, mock_models
@@ -500,7 +500,7 @@ def test_inference_with_gcs_destination(
         )
 
     @mock.patch.object(_evals_common, "Models")
-    @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader")
+    @mock.patch.object(_evals_utils, "EvalDatasetLoader")
     @mock.patch("pandas.DataFrame.to_json")
     @mock.patch("os.makedirs")
     def test_inference_with_local_destination(
@@ -552,7 +552,7 @@ def test_inference_with_local_destination(
         assert inference_result.gcs_source is None
 
     @mock.patch.object(_evals_common, "Models")
-    @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader")
+    @mock.patch.object(_evals_utils, "EvalDatasetLoader")
     def test_inference_from_request_column_save_to_local_dir(
         self, mock_eval_dataset_loader, mock_models
     ):
@@ -787,7 +787,7 @@ def test_inference_from_local_csv_file(self, mock_models):
         assert inference_result.gcs_source is None
 
     @mock.patch.object(_evals_common, "Models")
-    @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader")
+    @mock.patch.object(_evals_utils, "EvalDatasetLoader")
     def test_inference_with_row_level_config_overrides(
         self, mock_eval_dataset_loader, mock_models
     ):
@@ -972,7 +972,7 @@ def mock_generate_content_logic(*args, **kwargs):
         assert inference_result.gcs_source is None
 
     @mock.patch.object(_evals_common, "Models")
-    @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader")
+    @mock.patch.object(_evals_utils, "EvalDatasetLoader")
     def test_inference_with_multimodal_content(
         self, mock_eval_dataset_loader, mock_models
     ):
@@ -1048,7 +1048,7 @@ def test_inference_with_multimodal_content(
         assert inference_result.candidate_name == "gemini-pro"
         assert inference_result.gcs_source is None
 
-    @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader")
+    @mock.patch.object(_evals_utils, "EvalDatasetLoader")
     @mock.patch("vertexai._genai._evals_common.vertexai.Client")
     def test_run_inference_with_agent_engine_and_session_inputs_dict(
         self,
@@ -1136,7 +1136,7 @@ def test_run_inference_with_agent_engine_and_session_inputs_dict(
         assert inference_result.candidate_name is None
         assert inference_result.gcs_source is None
 
-    @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader")
+    @mock.patch.object(_evals_utils, "EvalDatasetLoader")
     @mock.patch("vertexai._genai._evals_common.vertexai.Client")
     def test_run_inference_with_agent_engine_and_session_inputs_literal_string(
         self,
@@ -1411,7 +1411,7 @@ def test_run_inference_with_litellm_import_error(self, mock_api_client_fixture):
     @mock.patch.object(_evals_common, "_is_gemini_model")
     @mock.patch.object(_evals_common, "_is_litellm_model")
     @mock.patch.object(_evals_common, "_is_litellm_vertex_maas_model")
-    @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader")
+    @mock.patch.object(_evals_utils, "EvalDatasetLoader")
     def test_run_inference_with_litellm_parsing(
         self,
         mock_eval_dataset_loader,
@@ -4536,9 +4536,7 @@ def test_execute_evaluation_with_openai_schema(
             name="test_metric", prompt_template="Evaluate: {response}"
         )
 
-        with mock.patch.object(
-            _evals_metric_loaders, "EvalDatasetLoader"
-        ) as mock_loader_class:
+        with mock.patch.object(_evals_utils, "EvalDatasetLoader") as mock_loader_class:
             mock_loader_instance = mock_loader_class.return_value
             mock_loader_instance.load.return_value = mock_openai_raw_data
 

vertexai/_genai/_evals_common.py

@@ -37,6 +37,7 @@
 from . import _evals_data_converters
 from . import _evals_metric_handlers
 from . import _evals_metric_loaders
+from . import _evals_utils
 from . import _gcs_utils
 
 from . import evals
@@ -695,7 +696,7 @@ def _load_dataframe(
     """Loads and prepares the prompt dataset for inference."""
     logger.info("Loading prompt dataset from: %s", src)
     try:
-        loader = _evals_metric_loaders.EvalDatasetLoader(api_client=api_client)
+        loader = _evals_utils.EvalDatasetLoader(api_client=api_client)
         dataset_list_of_dicts = loader.load(src)
         if not dataset_list_of_dicts:
             raise ValueError("Prompt dataset 'prompt_dataset' must not be empty.")
@@ -859,7 +860,7 @@ def _get_dataset_source(
 def _resolve_dataset_inputs(
     dataset: list[types.EvaluationDataset],
     dataset_schema: Optional[Literal["GEMINI", "FLATTEN", "OPENAI"]],
-    loader: "_evals_metric_loaders.EvalDatasetLoader",
+    loader: "_evals_utils.EvalDatasetLoader",
     agent_info: Optional[types.evals.AgentInfo] = None,
 ) -> tuple[types.EvaluationDataset, int]:
     """Loads and processes single or multiple datasets for evaluation.
@@ -1103,7 +1104,7 @@ def _execute_evaluation(  # type: ignore[no-untyped-def]
         else:
             deduped_candidate_names.append(name)
 
-    loader = _evals_metric_loaders.EvalDatasetLoader(api_client=api_client)
+    loader = _evals_utils.EvalDatasetLoader(api_client=api_client)
 
     agent_info = kwargs.get("agent_info", None)
     validated_agent_info = None

vertexai/_genai/_evals_metric_loaders.py

@@ -20,13 +20,10 @@
 import re
 from typing import Any, Optional, Union, TYPE_CHECKING
 
-from google.genai._api_client import BaseApiClient
-import pandas as pd
 import yaml
 
 from . import _evals_constant
 from . import _gcs_utils
-from . import _bigquery_utils
 
 if TYPE_CHECKING:
     from . import types
@@ -35,72 +32,6 @@
 logger = logging.getLogger(__name__)
 
 
-GCS_PREFIX = "gs://"
-BQ_PREFIX = "bq://"
-
-
-class EvalDatasetLoader:
-    """A loader for datasets from various sources, using a shared client."""
-
-    def __init__(self, api_client: BaseApiClient):
-        self.api_client = api_client
-        self.gcs_utils = _gcs_utils.GcsUtils(self.api_client)
-        self.bigquery_utils = _bigquery_utils.BigQueryUtils(self.api_client)
-
-    def _load_file(
-        self, filepath: str, file_type: str
-    ) -> Union[list[dict[str, Any]], Any]:
-        """Loads data from a file into a list of dictionaries."""
-        if filepath.startswith(GCS_PREFIX):
-            df = self.gcs_utils.read_gcs_file_to_dataframe(filepath, file_type)
-            return df.to_dict(orient="records")
-        else:
-            if file_type == "jsonl":
-                df = pd.read_json(filepath, lines=True)
-                return df.to_dict(orient="records")
-            elif file_type == "csv":
-                df = pd.read_csv(filepath, encoding="utf-8")
-                return df.to_dict(orient="records")
-            else:
-                raise ValueError(
-                    f"Unsupported file type: '{file_type}'. Please provide 'jsonl' or"
-                    " 'csv'."
-                )
-
-    def load(
-        self, source: Union[str, "pd.DataFrame"]
-    ) -> Union[list[dict[str, Any]], Any]:
-        """Loads dataset from various sources into a list of dictionaries."""
-        if isinstance(source, pd.DataFrame):
-            return source.to_dict(orient="records")
-        elif isinstance(source, str):
-            if source.startswith(BQ_PREFIX):
-                df = self.bigquery_utils.load_bigquery_to_dataframe(
-                    source[len(BQ_PREFIX) :]
-                )
-                return df.to_dict(orient="records")
-
-            _, extension = os.path.splitext(source)
-            file_type = extension.lower()[1:]
-
-            if file_type == "jsonl":
-                return self._load_file(source, "jsonl")
-            elif file_type == "csv":
-                return self._load_file(source, "csv")
-            else:
-                raise TypeError(
-                    f"Unsupported file type: {file_type} from {source}. Please"
-                    " provide a valid GCS path with `jsonl` or `csv` suffix, "
-                    "a local file path, or a valid BigQuery table URI."
-                )
-        else:
-            raise TypeError(
-                "Unsupported dataset type. Must be a `pd.DataFrame`, Python"
-                " a valid GCS path with `jsonl` or `csv` suffix, a local"
-                " file path, or a valid BigQuery table URI."
-            )
-
-
 class LazyLoadedPrebuiltMetric:
     """A proxy object representing a prebuilt metric to be loaded on demand.
 
@@ -408,6 +339,22 @@ def MULTI_TURN_SAFETY(self) -> LazyLoadedPrebuiltMetric:
     def FINAL_RESPONSE_QUALITY(self) -> LazyLoadedPrebuiltMetric:
         return self.__getattr__("FINAL_RESPONSE_QUALITY")
 
+    @property
+    def HALLUCINATION(self) -> LazyLoadedPrebuiltMetric:
+        return self.__getattr__("HALLUCINATION")
+
+    @property
+    def TOOL_USE_QUALITY(self) -> LazyLoadedPrebuiltMetric:
+        return self.__getattr__("TOOL_USE_QUALITY")
+
+    @property
+    def GECKO_TEXT2IMAGE(self) -> LazyLoadedPrebuiltMetric:
+        return self.__getattr__("GECKO_TEXT2IMAGE")
+
+    @property
+    def GECKO_TEXT2VIDEO(self) -> LazyLoadedPrebuiltMetric:
+        return self.__getattr__("GECKO_TEXT2VIDEO")
+
 
 PrebuiltMetric = PrebuiltMetricLoader()
 RubricMetric = PrebuiltMetric