From c69dcf87795c4fa2ad280b804c9b0bd3fa9bf06f Mon Sep 17 00:00:00 2001 From: Ankur Sharma Date: Fri, 25 Jul 2025 16:24:30 -0700 Subject: [PATCH] feat: Added an Fast API new endpoint to serve eval metric info This endpoint could be used by ADK Web to dynamically know: - What are the available eval metrics in an App - A description of those metrics - A value range supported by those metrics We also update the metric registry to make it mandatory to supply these details. The goal is to improve usability and interpretability of the eval metrics. PiperOrigin-RevId: 787277695 --- src/google/adk/cli/adk_web_server.py | 19 +++ src/google/adk/evaluation/eval_metrics.py | 109 +++++++++++++++--- .../adk/evaluation/final_response_match_v1.py | 23 +++- .../adk/evaluation/final_response_match_v2.py | 21 +++- .../evaluation/metric_evaluator_registry.py | 40 +++++-- .../adk/evaluation/response_evaluator.py | 34 +++++- src/google/adk/evaluation/safety_evaluator.py | 18 +++ .../adk/evaluation/trajectory_evaluator.py | 20 ++++ tests/unittests/cli/test_fast_api.py | 17 +++ .../test_final_response_match_v1.py | 9 ++ .../test_final_response_match_v2.py | 11 ++ .../evaluation/test_local_eval_service.py | 15 ++- .../test_metric_evaluator_registry.py | 66 ++++++++--- .../evaluation/test_response_evaluator.py | 27 +++++ .../evaluation/test_safety_evaluator.py | 8 ++ .../evaluation/test_trajectory_evaluator.py | 11 ++ 16 files changed, 394 insertions(+), 54 deletions(-) diff --git a/src/google/adk/cli/adk_web_server.py b/src/google/adk/cli/adk_web_server.py index e2215288..724a1298 100644 --- a/src/google/adk/cli/adk_web_server.py +++ b/src/google/adk/cli/adk_web_server.py @@ -64,6 +64,7 @@ from ..evaluation.eval_case import SessionInput from ..evaluation.eval_metrics import EvalMetric from ..evaluation.eval_metrics import EvalMetricResult from ..evaluation.eval_metrics import EvalMetricResultPerInvocation +from ..evaluation.eval_metrics import MetricInfo from ..evaluation.eval_result import EvalSetResult from ..evaluation.eval_set_results_manager import EvalSetResultsManager from ..evaluation.eval_sets_manager import EvalSetsManager @@ -697,6 +698,24 @@ class AdkWebServer: """Lists all eval results for the given app.""" return self.eval_set_results_manager.list_eval_set_results(app_name) + @app.get( + "/apps/{app_name}/eval_metrics", + response_model_exclude_none=True, + ) + def list_eval_metrics(app_name: str) -> list[MetricInfo]: + """Lists all eval metrics for the given app.""" + try: + from ..evaluation.metric_evaluator_registry import DEFAULT_METRIC_EVALUATOR_REGISTRY + + # Right now we ignore the app_name as eval metrics are not tied to the + # app_name, but they could be moving forward. + return DEFAULT_METRIC_EVALUATOR_REGISTRY.get_registered_metrics() + except ModuleNotFoundError as e: + logger.exception("%s\n%s", MISSING_EVAL_DEPENDENCIES_MESSAGE, e) + raise HTTPException( + status_code=400, detail=MISSING_EVAL_DEPENDENCIES_MESSAGE + ) from e + @app.delete("/apps/{app_name}/users/{user_id}/sessions/{session_id}") async def delete_session(app_name: str, user_id: str, session_id: str): await self.session_service.delete_session( diff --git a/src/google/adk/evaluation/eval_metrics.py b/src/google/adk/evaluation/eval_metrics.py index 1f6acf26..d73ce1e6 100644 --- a/src/google/adk/evaluation/eval_metrics.py +++ b/src/google/adk/evaluation/eval_metrics.py @@ -49,16 +49,22 @@ class JudgeModelOptions(BaseModel): judge_model: str = Field( default="gemini-2.5-flash", - description="""The judge model to use for evaluation. It can be a model name.""", + description=( + "The judge model to use for evaluation. It can be a model name." + ), ) judge_model_config: Optional[genai_types.GenerateContentConfig] = Field( - default=None, description="""The configuration for the judge model.""" + default=None, + description="The configuration for the judge model.", ) num_samples: Optional[int] = Field( default=None, - description="""The number of times to sample the model for each invocation evaluation.""", + description=( + "The number of times to sample the model for each invocation" + " evaluation." + ), ) @@ -70,15 +76,20 @@ class EvalMetric(BaseModel): populate_by_name=True, ) - metric_name: str - """The name of the metric.""" + metric_name: str = Field( + description="The name of the metric.", + ) - threshold: float - """A threshold value. Each metric decides how to interpret this threshold.""" + threshold: float = Field( + description=( + "A threshold value. Each metric decides how to interpret this" + " threshold." + ), + ) judge_model_options: Optional[JudgeModelOptions] = Field( default=None, - description="""Options for the judge model.""", + description="Options for the judge model.", ) @@ -90,8 +101,14 @@ class EvalMetricResult(EvalMetric): populate_by_name=True, ) - score: Optional[float] = None - eval_status: EvalStatus + score: Optional[float] = Field( + default=None, + description=( + "Score obtained after evaluating the metric. Optional, as evaluation" + " might not have happened." + ), + ) + eval_status: EvalStatus = Field(description="The status of this evaluation.") class EvalMetricResultPerInvocation(BaseModel): @@ -102,11 +119,71 @@ class EvalMetricResultPerInvocation(BaseModel): populate_by_name=True, ) - actual_invocation: Invocation - """The actual invocation, usually obtained by inferencing the agent.""" + actual_invocation: Invocation = Field( + description=( + "The actual invocation, usually obtained by inferencing the agent." + ) + ) - expected_invocation: Invocation - """The expected invocation, usually the reference or golden invocation.""" + expected_invocation: Invocation = Field( + description=( + "The expected invocation, usually the reference or golden invocation." + ) + ) - eval_metric_results: list[EvalMetricResult] = [] - """Eval resutls for each applicable metric.""" + eval_metric_results: list[EvalMetricResult] = Field( + default=[], + description="Eval resutls for each applicable metric.", + ) + + +class Interval(BaseModel): + """Represents a range of numeric values, e.g. [0 ,1] or (2,3) or [-1, 6).""" + + min_value: float = Field(description="The smaller end of the interval.") + + open_at_min: bool = Field( + default=False, + description=( + "The interval is Open on the min end. The default value is False," + " which means that we assume that the interval is Closed." + ), + ) + + max_value: float = Field(description="The larger end of the interval.") + + open_at_max: bool = Field( + default=False, + description=( + "The interval is Open on the max end. The default value is False," + " which means that we assume that the interval is Closed." + ), + ) + + +class MetricValueInfo(BaseModel): + """Information about the type of metric value.""" + + interval: Optional[Interval] = Field( + default=None, + description="The values represented by the metric are of type interval.", + ) + + +class MetricInfo(BaseModel): + """Information about the metric that are used for Evals.""" + + model_config = ConfigDict( + alias_generator=alias_generators.to_camel, + populate_by_name=True, + ) + + metric_name: str = Field(description="The name of the metric.") + + description: str = Field( + default=None, description="A 2 to 3 line description of the metric." + ) + + metric_value_info: MetricValueInfo = Field( + description="Information on the nature of values supported by the metric." + ) diff --git a/src/google/adk/evaluation/final_response_match_v1.py b/src/google/adk/evaluation/final_response_match_v1.py index a034b470..4d94d03a 100644 --- a/src/google/adk/evaluation/final_response_match_v1.py +++ b/src/google/adk/evaluation/final_response_match_v1.py @@ -22,6 +22,10 @@ from typing_extensions import override from .eval_case import Invocation from .eval_metrics import EvalMetric +from .eval_metrics import Interval +from .eval_metrics import MetricInfo +from .eval_metrics import MetricValueInfo +from .eval_metrics import PrebuiltMetrics from .evaluator import EvalStatus from .evaluator import EvaluationResult from .evaluator import Evaluator @@ -29,11 +33,28 @@ from .evaluator import PerInvocationResult class RougeEvaluator(Evaluator): - """Calculates the ROUGE-1 metric to compare responses.""" + """Evaluates if agent's final response matches a golden/expected final response using Rouge_1 metric. + + Value range for this metric is [0,1], with values closer to 1 more desirable. + """ def __init__(self, eval_metric: EvalMetric): self._eval_metric = eval_metric + @staticmethod + def get_metric_info() -> MetricInfo: + return MetricInfo( + metric_name=PrebuiltMetrics.RESPONSE_MATCH_SCORE.value, + description=( + "This metric evaluates if the agent's final response matches a" + " golden/expected final response using Rouge_1 metric. Value range" + " for this metric is [0,1], with values closer to 1 more desirable." + ), + metric_value_info=MetricValueInfo( + interval=Interval(min_value=0.0, max_value=1.0) + ), + ) + @override def evaluate_invocations( self, diff --git a/src/google/adk/evaluation/final_response_match_v2.py b/src/google/adk/evaluation/final_response_match_v2.py index cd13a073..177e719a 100644 --- a/src/google/adk/evaluation/final_response_match_v2.py +++ b/src/google/adk/evaluation/final_response_match_v2.py @@ -24,6 +24,10 @@ from ..models.llm_response import LlmResponse from ..utils.feature_decorator import experimental from .eval_case import Invocation from .eval_metrics import EvalMetric +from .eval_metrics import Interval +from .eval_metrics import MetricInfo +from .eval_metrics import MetricValueInfo +from .eval_metrics import PrebuiltMetrics from .evaluator import EvalStatus from .evaluator import EvaluationResult from .evaluator import PerInvocationResult @@ -146,6 +150,20 @@ class FinalResponseMatchV2Evaluator(LlmAsJudge): if self._eval_metric.judge_model_options.num_samples is None: self._eval_metric.judge_model_options.num_samples = _DEFAULT_NUM_SAMPLES + @staticmethod + def get_metric_info() -> MetricInfo: + return MetricInfo( + metric_name=PrebuiltMetrics.FINAL_RESPONSE_MATCH_V2.value, + description=( + "This metric evaluates if the agent's final response matches a" + " golden/expected final response using LLM as a judge. Value range" + " for this metric is [0,1], with values closer to 1 more desirable." + ), + metric_value_info=MetricValueInfo( + interval=Interval(min_value=0.0, max_value=1.0) + ), + ) + @override def format_auto_rater_prompt( self, actual_invocation: Invocation, expected_invocation: Invocation @@ -185,8 +203,7 @@ class FinalResponseMatchV2Evaluator(LlmAsJudge): tie, consider the result to be invalid. Args: - per_invocation_samples: Samples of per-invocation results to - aggregate. + per_invocation_samples: Samples of per-invocation results to aggregate. Returns: If there is a majority of valid results, return the first valid result. diff --git a/src/google/adk/evaluation/metric_evaluator_registry.py b/src/google/adk/evaluation/metric_evaluator_registry.py index c3af0656..e5fd33f4 100644 --- a/src/google/adk/evaluation/metric_evaluator_registry.py +++ b/src/google/adk/evaluation/metric_evaluator_registry.py @@ -17,7 +17,9 @@ from __future__ import annotations import logging from ..errors.not_found_error import NotFoundError +from ..utils.feature_decorator import experimental from .eval_metrics import EvalMetric +from .eval_metrics import MetricInfo from .eval_metrics import MetricName from .eval_metrics import PrebuiltMetrics from .evaluator import Evaluator @@ -29,10 +31,11 @@ from .trajectory_evaluator import TrajectoryEvaluator logger = logging.getLogger("google_adk." + __name__) +@experimental class MetricEvaluatorRegistry: """A registry for metric Evaluators.""" - _registry: dict[str, type[Evaluator]] = {} + _registry: dict[str, tuple[type[Evaluator], MetricInfo]] = {} def get_evaluator(self, eval_metric: EvalMetric) -> Evaluator: """Returns an Evaluator for the given metric. @@ -48,15 +51,18 @@ class MetricEvaluatorRegistry: if eval_metric.metric_name not in self._registry: raise NotFoundError(f"{eval_metric.metric_name} not found in registry.") - return self._registry[eval_metric.metric_name](eval_metric=eval_metric) + return self._registry[eval_metric.metric_name][0](eval_metric=eval_metric) def register_evaluator( - self, metric_name: MetricName, evaluator: type[Evaluator] + self, + metric_info: MetricInfo, + evaluator: type[Evaluator], ): - """Registers an evaluator given the metric name. + """Registers an evaluator given the metric info. If a mapping already exist, then it is updated. """ + metric_name = metric_info.metric_name if metric_name in self._registry: logger.info( "Updating Evaluator class for %s from %s to %s", @@ -65,7 +71,16 @@ class MetricEvaluatorRegistry: evaluator, ) - self._registry[str(metric_name)] = evaluator + self._registry[str(metric_name)] = (evaluator, metric_info) + + def get_registered_metrics( + self, + ) -> list[MetricInfo]: + """Returns a list of MetricInfo about the metrics registered so far.""" + return [ + evaluator_and_metric_info[1].model_copy(deep=True) + for _, evaluator_and_metric_info in self._registry.items() + ] def _get_default_metric_evaluator_registry() -> MetricEvaluatorRegistry: @@ -73,23 +88,28 @@ def _get_default_metric_evaluator_registry() -> MetricEvaluatorRegistry: metric_evaluator_registry = MetricEvaluatorRegistry() metric_evaluator_registry.register_evaluator( - metric_name=PrebuiltMetrics.TOOL_TRAJECTORY_AVG_SCORE.value, + metric_info=TrajectoryEvaluator.get_metric_info(), evaluator=TrajectoryEvaluator, ) + metric_evaluator_registry.register_evaluator( - metric_name=PrebuiltMetrics.RESPONSE_EVALUATION_SCORE.value, + metric_info=ResponseEvaluator.get_metric_info( + PrebuiltMetrics.RESPONSE_EVALUATION_SCORE.value + ), evaluator=ResponseEvaluator, ) metric_evaluator_registry.register_evaluator( - metric_name=PrebuiltMetrics.RESPONSE_MATCH_SCORE.value, + metric_info=ResponseEvaluator.get_metric_info( + PrebuiltMetrics.RESPONSE_MATCH_SCORE.value + ), evaluator=ResponseEvaluator, ) metric_evaluator_registry.register_evaluator( - metric_name=PrebuiltMetrics.SAFETY_V1.value, + metric_info=SafetyEvaluatorV1.get_metric_info(), evaluator=SafetyEvaluatorV1, ) metric_evaluator_registry.register_evaluator( - metric_name=PrebuiltMetrics.FINAL_RESPONSE_MATCH_V2.value, + metric_info=FinalResponseMatchV2Evaluator.get_metric_info(), evaluator=FinalResponseMatchV2Evaluator, ) diff --git a/src/google/adk/evaluation/response_evaluator.py b/src/google/adk/evaluation/response_evaluator.py index b38d5553..fa6be8bf 100644 --- a/src/google/adk/evaluation/response_evaluator.py +++ b/src/google/adk/evaluation/response_evaluator.py @@ -21,6 +21,10 @@ from vertexai import types as vertexai_types from .eval_case import Invocation from .eval_metrics import EvalMetric +from .eval_metrics import Interval +from .eval_metrics import MetricInfo +from .eval_metrics import MetricValueInfo +from .eval_metrics import PrebuiltMetrics from .evaluator import EvaluationResult from .evaluator import Evaluator from .final_response_match_v1 import RougeEvaluator @@ -38,7 +42,7 @@ class ResponseEvaluator(Evaluator): 2) response_match_score: This metric evaluates if agent's final response matches a golden/expected - final response. + final response using Rouge_1 metric. Value range for this metric is [0,1], with values closer to 1 more desirable. """ @@ -61,15 +65,35 @@ class ResponseEvaluator(Evaluator): threshold = eval_metric.threshold metric_name = eval_metric.metric_name - if "response_evaluation_score" == metric_name: + if PrebuiltMetrics.RESPONSE_EVALUATION_SCORE.value == metric_name: self._metric_name = vertexai_types.PrebuiltMetric.COHERENCE - elif "response_match_score" == metric_name: - self._metric_name = "response_match_score" + elif PrebuiltMetrics.RESPONSE_MATCH_SCORE.value == metric_name: + self._metric_name = metric_name else: raise ValueError(f"`{metric_name}` is not supported.") self._threshold = threshold + @staticmethod + def get_metric_info(metric_name: str) -> MetricInfo: + """Returns MetricInfo for the given metric name.""" + if PrebuiltMetrics.RESPONSE_EVALUATION_SCORE.value == metric_name: + return MetricInfo( + metric_name=PrebuiltMetrics.RESPONSE_EVALUATION_SCORE.value, + description=( + "This metric evaluates how coherent agent's resposne was. Value" + " range of this metric is [1,5], with values closer to 5 more" + " desirable." + ), + metric_value_info=MetricValueInfo( + interval=Interval(min_value=1.0, max_value=5.0) + ), + ) + elif PrebuiltMetrics.RESPONSE_MATCH_SCORE.value == metric_name: + return RougeEvaluator.get_metric_info() + else: + raise ValueError(f"`{metric_name}` is not supported.") + @override def evaluate_invocations( self, @@ -77,7 +101,7 @@ class ResponseEvaluator(Evaluator): expected_invocations: list[Invocation], ) -> EvaluationResult: # If the metric is response_match_score, just use the RougeEvaluator. - if self._metric_name == "response_match_score": + if self._metric_name == PrebuiltMetrics.RESPONSE_MATCH_SCORE.value: rouge_evaluator = RougeEvaluator( EvalMetric(metric_name=self._metric_name, threshold=self._threshold) ) diff --git a/src/google/adk/evaluation/safety_evaluator.py b/src/google/adk/evaluation/safety_evaluator.py index 6b9ad242..f24931a2 100644 --- a/src/google/adk/evaluation/safety_evaluator.py +++ b/src/google/adk/evaluation/safety_evaluator.py @@ -19,6 +19,10 @@ from vertexai import types as vertexai_types from .eval_case import Invocation from .eval_metrics import EvalMetric +from .eval_metrics import Interval +from .eval_metrics import MetricInfo +from .eval_metrics import MetricValueInfo +from .eval_metrics import PrebuiltMetrics from .evaluator import EvaluationResult from .evaluator import Evaluator from .vertex_ai_eval_facade import _VertexAiEvalFacade @@ -42,6 +46,20 @@ class SafetyEvaluatorV1(Evaluator): def __init__(self, eval_metric: EvalMetric): self._eval_metric = eval_metric + @staticmethod + def get_metric_info() -> MetricInfo: + return MetricInfo( + metric_name=PrebuiltMetrics.SAFETY_V1.value, + description=( + "This metric evaluates the safety (harmlessness) of an Agent's" + " Response. Value range of the metric is [0, 1], with values closer" + " to 1 to be more desirable (safe)." + ), + metric_value_info=MetricValueInfo( + interval=Interval(min_value=0.0, max_value=1.0) + ), + ) + @override def evaluate_invocations( self, diff --git a/src/google/adk/evaluation/trajectory_evaluator.py b/src/google/adk/evaluation/trajectory_evaluator.py index 81566eb2..8f7508d4 100644 --- a/src/google/adk/evaluation/trajectory_evaluator.py +++ b/src/google/adk/evaluation/trajectory_evaluator.py @@ -25,6 +25,10 @@ from typing_extensions import override from .eval_case import Invocation from .eval_metrics import EvalMetric +from .eval_metrics import Interval +from .eval_metrics import MetricInfo +from .eval_metrics import MetricValueInfo +from .eval_metrics import PrebuiltMetrics from .evaluation_constants import EvalConstants from .evaluator import EvalStatus from .evaluator import EvaluationResult @@ -51,6 +55,22 @@ class TrajectoryEvaluator(Evaluator): self._threshold = threshold + @staticmethod + def get_metric_info() -> MetricInfo: + return MetricInfo( + metric_name=PrebuiltMetrics.TOOL_TRAJECTORY_AVG_SCORE.value, + description=( + "This metric compares two tool call trajectories (expected vs." + " actual) for the same user interaction. It performs an exact match" + " on the tool name and arguments for each step in the trajectory." + " A score of 1.0 indicates a perfect match, while 0.0 indicates a" + " mismatch. Higher values are better." + ), + metric_value_info=MetricValueInfo( + interval=Interval(min_value=0.0, max_value=1.0) + ), + ) + @override def evaluate_invocations( self, diff --git a/tests/unittests/cli/test_fast_api.py b/tests/unittests/cli/test_fast_api.py index 70d53034..f1c9e9d6 100755 --- a/tests/unittests/cli/test_fast_api.py +++ b/tests/unittests/cli/test_fast_api.py @@ -845,6 +845,23 @@ def test_run_eval(test_app, create_test_eval_set): assert data == [f"{info['app_name']}_test_eval_set_id_eval_result"] +def test_list_eval_metrics(test_app): + """Test listing eval metrics.""" + url = "/apps/test_app/eval_metrics" + response = test_app.get(url) + + # Verify the response + assert response.status_code == 200 + data = response.json() + assert isinstance(data, list) + # Add more assertions based on the expected metrics + assert len(data) > 0 + for metric in data: + assert "metricName" in metric + assert "description" in metric + assert "metricValueInfo" in metric + + def test_debug_trace(test_app): """Test the debug trace endpoint.""" # This test will likely return 404 since we haven't set up trace data, diff --git a/tests/unittests/evaluation/test_final_response_match_v1.py b/tests/unittests/evaluation/test_final_response_match_v1.py index d5544a5a..d5fe0464 100644 --- a/tests/unittests/evaluation/test_final_response_match_v1.py +++ b/tests/unittests/evaluation/test_final_response_match_v1.py @@ -16,6 +16,7 @@ from __future__ import annotations from google.adk.evaluation.eval_case import Invocation from google.adk.evaluation.eval_metrics import EvalMetric +from google.adk.evaluation.eval_metrics import PrebuiltMetrics from google.adk.evaluation.evaluator import EvalStatus from google.adk.evaluation.final_response_match_v1 import _calculate_rouge_1_scores from google.adk.evaluation.final_response_match_v1 import RougeEvaluator @@ -138,3 +139,11 @@ def test_rouge_evaluator_multiple_invocations( expected_score, rel=1e-3 ) assert evaluation_result.overall_eval_status == expected_status + + +def test_get_metric_info(): + """Test get_metric_info function for response match metric.""" + metric_info = RougeEvaluator.get_metric_info() + assert metric_info.metric_name == PrebuiltMetrics.RESPONSE_MATCH_SCORE.value + assert metric_info.metric_value_info.interval.min_value == 0.0 + assert metric_info.metric_value_info.interval.max_value == 1.0 diff --git a/tests/unittests/evaluation/test_final_response_match_v2.py b/tests/unittests/evaluation/test_final_response_match_v2.py index 859e6d20..911c5e22 100644 --- a/tests/unittests/evaluation/test_final_response_match_v2.py +++ b/tests/unittests/evaluation/test_final_response_match_v2.py @@ -17,6 +17,7 @@ from __future__ import annotations from google.adk.evaluation.eval_case import Invocation from google.adk.evaluation.eval_metrics import EvalMetric from google.adk.evaluation.eval_metrics import JudgeModelOptions +from google.adk.evaluation.eval_metrics import PrebuiltMetrics from google.adk.evaluation.evaluator import EvalStatus from google.adk.evaluation.evaluator import PerInvocationResult from google.adk.evaluation.final_response_match_v2 import _parse_critique @@ -476,3 +477,13 @@ def test_aggregate_invocation_results(): # Only 4 / 8 invocations are evaluated, and 2 / 4 are valid. assert aggregated_result.overall_score == 0.5 assert aggregated_result.overall_eval_status == EvalStatus.PASSED + + +def test_get_metric_info(): + """Test get_metric_info function for Final Response Match V2 metric.""" + metric_info = FinalResponseMatchV2Evaluator.get_metric_info() + assert ( + metric_info.metric_name == PrebuiltMetrics.FINAL_RESPONSE_MATCH_V2.value + ) + assert metric_info.metric_value_info.interval.min_value == 0.0 + assert metric_info.metric_value_info.interval.max_value == 1.0 diff --git a/tests/unittests/evaluation/test_local_eval_service.py b/tests/unittests/evaluation/test_local_eval_service.py index 5353f1f1..49ebead2 100644 --- a/tests/unittests/evaluation/test_local_eval_service.py +++ b/tests/unittests/evaluation/test_local_eval_service.py @@ -24,6 +24,9 @@ from google.adk.evaluation.base_eval_service import InferenceResult from google.adk.evaluation.eval_case import Invocation from google.adk.evaluation.eval_metrics import EvalMetric from google.adk.evaluation.eval_metrics import EvalMetricResult +from google.adk.evaluation.eval_metrics import Interval +from google.adk.evaluation.eval_metrics import MetricInfo +from google.adk.evaluation.eval_metrics import MetricValueInfo from google.adk.evaluation.eval_result import EvalCaseResult from google.adk.evaluation.eval_set import EvalCase from google.adk.evaluation.eval_set import EvalSet @@ -61,7 +64,7 @@ def eval_service( dummy_agent, mock_eval_sets_manager, mock_eval_set_results_manager ): DEFAULT_METRIC_EVALUATOR_REGISTRY.register_evaluator( - metric_name="fake_metric", evaluator=FakeEvaluator + metric_info=FakeEvaluator.get_metric_info(), evaluator=FakeEvaluator ) return LocalEvalService( root_agent=dummy_agent, @@ -75,6 +78,16 @@ class FakeEvaluator(Evaluator): def __init__(self, eval_metric: EvalMetric): self._eval_metric = eval_metric + @staticmethod + def get_metric_info() -> MetricInfo: + return MetricInfo( + metric_name="fake_metric", + description="Fake metric description", + metric_value_info=MetricValueInfo( + interval=Interval(min_value=0.0, max_value=1.0) + ), + ) + def evaluate_invocations( self, actual_invocations: list[Invocation], diff --git a/tests/unittests/evaluation/test_metric_evaluator_registry.py b/tests/unittests/evaluation/test_metric_evaluator_registry.py index f36acc41..60b39d54 100644 --- a/tests/unittests/evaluation/test_metric_evaluator_registry.py +++ b/tests/unittests/evaluation/test_metric_evaluator_registry.py @@ -16,10 +16,15 @@ from __future__ import annotations from google.adk.errors.not_found_error import NotFoundError from google.adk.evaluation.eval_metrics import EvalMetric +from google.adk.evaluation.eval_metrics import Interval +from google.adk.evaluation.eval_metrics import MetricInfo +from google.adk.evaluation.eval_metrics import MetricValueInfo from google.adk.evaluation.evaluator import Evaluator from google.adk.evaluation.metric_evaluator_registry import MetricEvaluatorRegistry import pytest +_DUMMY_METRIC_NAME = "dummy_metric_name" + class TestMetricEvaluatorRegistry: """Test cases for MetricEvaluatorRegistry.""" @@ -36,6 +41,16 @@ class TestMetricEvaluatorRegistry: def evaluate_invocations(self, actual_invocations, expected_invocations): return "dummy_result" + @staticmethod + def get_metric_info() -> MetricInfo: + return MetricInfo( + metric_name=_DUMMY_METRIC_NAME, + description="Dummy metric description", + metric_value_info=MetricValueInfo( + interval=Interval(min_value=0.0, max_value=1.0) + ), + ) + class AnotherDummyEvaluator(Evaluator): def __init__(self, eval_metric: EvalMetric): @@ -44,45 +59,58 @@ class TestMetricEvaluatorRegistry: def evaluate_invocations(self, actual_invocations, expected_invocations): return "another_dummy_result" + @staticmethod + def get_metric_info() -> MetricInfo: + return MetricInfo( + metric_name=_DUMMY_METRIC_NAME, + description="Another dummy metric description", + metric_value_info=MetricValueInfo( + interval=Interval(min_value=0.0, max_value=1.0) + ), + ) + def test_register_evaluator(self, registry): - dummy_metric_name = "dummy_metric_name" + metric_info = TestMetricEvaluatorRegistry.DummyEvaluator.get_metric_info() registry.register_evaluator( - dummy_metric_name, + metric_info, TestMetricEvaluatorRegistry.DummyEvaluator, ) - assert dummy_metric_name in registry._registry - assert ( - registry._registry[dummy_metric_name] - == TestMetricEvaluatorRegistry.DummyEvaluator + assert _DUMMY_METRIC_NAME in registry._registry + assert registry._registry[_DUMMY_METRIC_NAME] == ( + TestMetricEvaluatorRegistry.DummyEvaluator, + metric_info, ) def test_register_evaluator_updates_existing(self, registry): - dummy_metric_name = "dummy_metric_name" + metric_info = TestMetricEvaluatorRegistry.DummyEvaluator.get_metric_info() registry.register_evaluator( - dummy_metric_name, + metric_info, TestMetricEvaluatorRegistry.DummyEvaluator, ) - assert ( - registry._registry[dummy_metric_name] - == TestMetricEvaluatorRegistry.DummyEvaluator + assert registry._registry[_DUMMY_METRIC_NAME] == ( + TestMetricEvaluatorRegistry.DummyEvaluator, + metric_info, ) - registry.register_evaluator( - dummy_metric_name, TestMetricEvaluatorRegistry.AnotherDummyEvaluator + metric_info = ( + TestMetricEvaluatorRegistry.AnotherDummyEvaluator.get_metric_info() ) - assert ( - registry._registry[dummy_metric_name] - == TestMetricEvaluatorRegistry.AnotherDummyEvaluator + registry.register_evaluator( + metric_info, TestMetricEvaluatorRegistry.AnotherDummyEvaluator + ) + assert registry._registry[_DUMMY_METRIC_NAME] == ( + TestMetricEvaluatorRegistry.AnotherDummyEvaluator, + metric_info, ) def test_get_evaluator(self, registry): - dummy_metric_name = "dummy_metric_name" + metric_info = TestMetricEvaluatorRegistry.DummyEvaluator.get_metric_info() registry.register_evaluator( - dummy_metric_name, + metric_info, TestMetricEvaluatorRegistry.DummyEvaluator, ) - eval_metric = EvalMetric(metric_name=dummy_metric_name, threshold=0.5) + eval_metric = EvalMetric(metric_name=_DUMMY_METRIC_NAME, threshold=0.5) evaluator = registry.get_evaluator(eval_metric) assert isinstance(evaluator, TestMetricEvaluatorRegistry.DummyEvaluator) diff --git a/tests/unittests/evaluation/test_response_evaluator.py b/tests/unittests/evaluation/test_response_evaluator.py index 09946772..bace9c6a 100644 --- a/tests/unittests/evaluation/test_response_evaluator.py +++ b/tests/unittests/evaluation/test_response_evaluator.py @@ -16,6 +16,7 @@ from unittest.mock import patch from google.adk.evaluation.eval_case import Invocation +from google.adk.evaluation.eval_metrics import PrebuiltMetrics from google.adk.evaluation.evaluator import EvalStatus from google.adk.evaluation.response_evaluator import ResponseEvaluator from google.genai import types as genai_types @@ -113,3 +114,29 @@ class TestResponseEvaluator: assert [m.name for m in mock_kwargs["metrics"]] == [ vertexai_types.PrebuiltMetric.COHERENCE.name ] + + def test_get_metric_info_response_evaluation_score(self, mock_perform_eval): + """Test get_metric_info function for response evaluation metric.""" + metric_info = ResponseEvaluator.get_metric_info( + PrebuiltMetrics.RESPONSE_EVALUATION_SCORE.value + ) + assert ( + metric_info.metric_name + == PrebuiltMetrics.RESPONSE_EVALUATION_SCORE.value + ) + assert metric_info.metric_value_info.interval.min_value == 1.0 + assert metric_info.metric_value_info.interval.max_value == 5.0 + + def test_get_metric_info_response_match_score(self, mock_perform_eval): + """Test get_metric_info function for response match metric.""" + metric_info = ResponseEvaluator.get_metric_info( + PrebuiltMetrics.RESPONSE_MATCH_SCORE.value + ) + assert metric_info.metric_name == PrebuiltMetrics.RESPONSE_MATCH_SCORE.value + assert metric_info.metric_value_info.interval.min_value == 0.0 + assert metric_info.metric_value_info.interval.max_value == 1.0 + + def test_get_metric_info_invalid(self, mock_perform_eval): + """Test get_metric_info function for invalid metric.""" + with pytest.raises(ValueError): + ResponseEvaluator.get_metric_info("invalid_metric") diff --git a/tests/unittests/evaluation/test_safety_evaluator.py b/tests/unittests/evaluation/test_safety_evaluator.py index 077e3143..5cc95b1d 100644 --- a/tests/unittests/evaluation/test_safety_evaluator.py +++ b/tests/unittests/evaluation/test_safety_evaluator.py @@ -17,6 +17,7 @@ from unittest.mock import patch from google.adk.evaluation.eval_case import Invocation from google.adk.evaluation.eval_metrics import EvalMetric +from google.adk.evaluation.eval_metrics import PrebuiltMetrics from google.adk.evaluation.evaluator import EvalStatus from google.adk.evaluation.safety_evaluator import SafetyEvaluatorV1 from google.genai import types as genai_types @@ -76,3 +77,10 @@ class TestSafetyEvaluatorV1: assert [m.name for m in mock_kwargs["metrics"]] == [ vertexai_types.PrebuiltMetric.SAFETY.name ] + + def test_get_metric_info(self, mock_perform_eval): + """Test get_metric_info function for Safety metric.""" + metric_info = SafetyEvaluatorV1.get_metric_info() + assert metric_info.metric_name == PrebuiltMetrics.SAFETY_V1.value + assert metric_info.metric_value_info.interval.min_value == 0.0 + assert metric_info.metric_value_info.interval.max_value == 1.0 diff --git a/tests/unittests/evaluation/test_trajectory_evaluator.py b/tests/unittests/evaluation/test_trajectory_evaluator.py index f3622a53..a8053dd1 100644 --- a/tests/unittests/evaluation/test_trajectory_evaluator.py +++ b/tests/unittests/evaluation/test_trajectory_evaluator.py @@ -16,6 +16,7 @@ import math +from google.adk.evaluation.eval_metrics import PrebuiltMetrics from google.adk.evaluation.trajectory_evaluator import TrajectoryEvaluator import pytest @@ -270,3 +271,13 @@ def test_are_tools_equal_one_empty_one_not(): list_a = [] list_b = [TOOL_GET_WEATHER] assert not TrajectoryEvaluator.are_tools_equal(list_a, list_b) + + +def test_get_metric_info(): + """Test get_metric_info function for tool trajectory avg metric.""" + metric_info = TrajectoryEvaluator.get_metric_info() + assert ( + metric_info.metric_name == PrebuiltMetrics.TOOL_TRAJECTORY_AVG_SCORE.value + ) + assert metric_info.metric_value_info.interval.min_value == 0.0 + assert metric_info.metric_value_info.interval.max_value == 1.0