diff --git a/src/google/adk/cli/adk_web_server.py b/src/google/adk/cli/adk_web_server.py index e2215288..724a1298 100644 --- a/src/google/adk/cli/adk_web_server.py +++ b/src/google/adk/cli/adk_web_server.py @@ -64,6 +64,7 @@ from ..evaluation.eval_case import SessionInput from ..evaluation.eval_metrics import EvalMetric from ..evaluation.eval_metrics import EvalMetricResult from ..evaluation.eval_metrics import EvalMetricResultPerInvocation +from ..evaluation.eval_metrics import MetricInfo from ..evaluation.eval_result import EvalSetResult from ..evaluation.eval_set_results_manager import EvalSetResultsManager from ..evaluation.eval_sets_manager import EvalSetsManager @@ -697,6 +698,24 @@ class AdkWebServer: """Lists all eval results for the given app.""" return self.eval_set_results_manager.list_eval_set_results(app_name) + @app.get( + "/apps/{app_name}/eval_metrics", + response_model_exclude_none=True, + ) + def list_eval_metrics(app_name: str) -> list[MetricInfo]: + """Lists all eval metrics for the given app.""" + try: + from ..evaluation.metric_evaluator_registry import DEFAULT_METRIC_EVALUATOR_REGISTRY + + # Right now we ignore the app_name as eval metrics are not tied to the + # app_name, but they could be moving forward. + return DEFAULT_METRIC_EVALUATOR_REGISTRY.get_registered_metrics() + except ModuleNotFoundError as e: + logger.exception("%s\n%s", MISSING_EVAL_DEPENDENCIES_MESSAGE, e) + raise HTTPException( + status_code=400, detail=MISSING_EVAL_DEPENDENCIES_MESSAGE + ) from e + @app.delete("/apps/{app_name}/users/{user_id}/sessions/{session_id}") async def delete_session(app_name: str, user_id: str, session_id: str): await self.session_service.delete_session( diff --git a/src/google/adk/evaluation/eval_metrics.py b/src/google/adk/evaluation/eval_metrics.py index 1f6acf26..d73ce1e6 100644 --- a/src/google/adk/evaluation/eval_metrics.py +++ b/src/google/adk/evaluation/eval_metrics.py @@ -49,16 +49,22 @@ class JudgeModelOptions(BaseModel): judge_model: str = Field( default="gemini-2.5-flash", - description="""The judge model to use for evaluation. It can be a model name.""", + description=( + "The judge model to use for evaluation. It can be a model name." + ), ) judge_model_config: Optional[genai_types.GenerateContentConfig] = Field( - default=None, description="""The configuration for the judge model.""" + default=None, + description="The configuration for the judge model.", ) num_samples: Optional[int] = Field( default=None, - description="""The number of times to sample the model for each invocation evaluation.""", + description=( + "The number of times to sample the model for each invocation" + " evaluation." + ), ) @@ -70,15 +76,20 @@ class EvalMetric(BaseModel): populate_by_name=True, ) - metric_name: str - """The name of the metric.""" + metric_name: str = Field( + description="The name of the metric.", + ) - threshold: float - """A threshold value. Each metric decides how to interpret this threshold.""" + threshold: float = Field( + description=( + "A threshold value. Each metric decides how to interpret this" + " threshold." + ), + ) judge_model_options: Optional[JudgeModelOptions] = Field( default=None, - description="""Options for the judge model.""", + description="Options for the judge model.", ) @@ -90,8 +101,14 @@ class EvalMetricResult(EvalMetric): populate_by_name=True, ) - score: Optional[float] = None - eval_status: EvalStatus + score: Optional[float] = Field( + default=None, + description=( + "Score obtained after evaluating the metric. Optional, as evaluation" + " might not have happened." + ), + ) + eval_status: EvalStatus = Field(description="The status of this evaluation.") class EvalMetricResultPerInvocation(BaseModel): @@ -102,11 +119,71 @@ class EvalMetricResultPerInvocation(BaseModel): populate_by_name=True, ) - actual_invocation: Invocation - """The actual invocation, usually obtained by inferencing the agent.""" + actual_invocation: Invocation = Field( + description=( + "The actual invocation, usually obtained by inferencing the agent." + ) + ) - expected_invocation: Invocation - """The expected invocation, usually the reference or golden invocation.""" + expected_invocation: Invocation = Field( + description=( + "The expected invocation, usually the reference or golden invocation." + ) + ) - eval_metric_results: list[EvalMetricResult] = [] - """Eval resutls for each applicable metric.""" + eval_metric_results: list[EvalMetricResult] = Field( + default=[], + description="Eval resutls for each applicable metric.", + ) + + +class Interval(BaseModel): + """Represents a range of numeric values, e.g. [0 ,1] or (2,3) or [-1, 6).""" + + min_value: float = Field(description="The smaller end of the interval.") + + open_at_min: bool = Field( + default=False, + description=( + "The interval is Open on the min end. The default value is False," + " which means that we assume that the interval is Closed." + ), + ) + + max_value: float = Field(description="The larger end of the interval.") + + open_at_max: bool = Field( + default=False, + description=( + "The interval is Open on the max end. The default value is False," + " which means that we assume that the interval is Closed." + ), + ) + + +class MetricValueInfo(BaseModel): + """Information about the type of metric value.""" + + interval: Optional[Interval] = Field( + default=None, + description="The values represented by the metric are of type interval.", + ) + + +class MetricInfo(BaseModel): + """Information about the metric that are used for Evals.""" + + model_config = ConfigDict( + alias_generator=alias_generators.to_camel, + populate_by_name=True, + ) + + metric_name: str = Field(description="The name of the metric.") + + description: str = Field( + default=None, description="A 2 to 3 line description of the metric." + ) + + metric_value_info: MetricValueInfo = Field( + description="Information on the nature of values supported by the metric." + ) diff --git a/src/google/adk/evaluation/final_response_match_v1.py b/src/google/adk/evaluation/final_response_match_v1.py index a034b470..4d94d03a 100644 --- a/src/google/adk/evaluation/final_response_match_v1.py +++ b/src/google/adk/evaluation/final_response_match_v1.py @@ -22,6 +22,10 @@ from typing_extensions import override from .eval_case import Invocation from .eval_metrics import EvalMetric +from .eval_metrics import Interval +from .eval_metrics import MetricInfo +from .eval_metrics import MetricValueInfo +from .eval_metrics import PrebuiltMetrics from .evaluator import EvalStatus from .evaluator import EvaluationResult from .evaluator import Evaluator @@ -29,11 +33,28 @@ from .evaluator import PerInvocationResult class RougeEvaluator(Evaluator): - """Calculates the ROUGE-1 metric to compare responses.""" + """Evaluates if agent's final response matches a golden/expected final response using Rouge_1 metric. + + Value range for this metric is [0,1], with values closer to 1 more desirable. + """ def __init__(self, eval_metric: EvalMetric): self._eval_metric = eval_metric + @staticmethod + def get_metric_info() -> MetricInfo: + return MetricInfo( + metric_name=PrebuiltMetrics.RESPONSE_MATCH_SCORE.value, + description=( + "This metric evaluates if the agent's final response matches a" + " golden/expected final response using Rouge_1 metric. Value range" + " for this metric is [0,1], with values closer to 1 more desirable." + ), + metric_value_info=MetricValueInfo( + interval=Interval(min_value=0.0, max_value=1.0) + ), + ) + @override def evaluate_invocations( self, diff --git a/src/google/adk/evaluation/final_response_match_v2.py b/src/google/adk/evaluation/final_response_match_v2.py index cd13a073..177e719a 100644 --- a/src/google/adk/evaluation/final_response_match_v2.py +++ b/src/google/adk/evaluation/final_response_match_v2.py @@ -24,6 +24,10 @@ from ..models.llm_response import LlmResponse from ..utils.feature_decorator import experimental from .eval_case import Invocation from .eval_metrics import EvalMetric +from .eval_metrics import Interval +from .eval_metrics import MetricInfo +from .eval_metrics import MetricValueInfo +from .eval_metrics import PrebuiltMetrics from .evaluator import EvalStatus from .evaluator import EvaluationResult from .evaluator import PerInvocationResult @@ -146,6 +150,20 @@ class FinalResponseMatchV2Evaluator(LlmAsJudge): if self._eval_metric.judge_model_options.num_samples is None: self._eval_metric.judge_model_options.num_samples = _DEFAULT_NUM_SAMPLES + @staticmethod + def get_metric_info() -> MetricInfo: + return MetricInfo( + metric_name=PrebuiltMetrics.FINAL_RESPONSE_MATCH_V2.value, + description=( + "This metric evaluates if the agent's final response matches a" + " golden/expected final response using LLM as a judge. Value range" + " for this metric is [0,1], with values closer to 1 more desirable." + ), + metric_value_info=MetricValueInfo( + interval=Interval(min_value=0.0, max_value=1.0) + ), + ) + @override def format_auto_rater_prompt( self, actual_invocation: Invocation, expected_invocation: Invocation @@ -185,8 +203,7 @@ class FinalResponseMatchV2Evaluator(LlmAsJudge): tie, consider the result to be invalid. Args: - per_invocation_samples: Samples of per-invocation results to - aggregate. + per_invocation_samples: Samples of per-invocation results to aggregate. Returns: If there is a majority of valid results, return the first valid result. diff --git a/src/google/adk/evaluation/metric_evaluator_registry.py b/src/google/adk/evaluation/metric_evaluator_registry.py index c3af0656..e5fd33f4 100644 --- a/src/google/adk/evaluation/metric_evaluator_registry.py +++ b/src/google/adk/evaluation/metric_evaluator_registry.py @@ -17,7 +17,9 @@ from __future__ import annotations import logging from ..errors.not_found_error import NotFoundError +from ..utils.feature_decorator import experimental from .eval_metrics import EvalMetric +from .eval_metrics import MetricInfo from .eval_metrics import MetricName from .eval_metrics import PrebuiltMetrics from .evaluator import Evaluator @@ -29,10 +31,11 @@ from .trajectory_evaluator import TrajectoryEvaluator logger = logging.getLogger("google_adk." + __name__) +@experimental class MetricEvaluatorRegistry: """A registry for metric Evaluators.""" - _registry: dict[str, type[Evaluator]] = {} + _registry: dict[str, tuple[type[Evaluator], MetricInfo]] = {} def get_evaluator(self, eval_metric: EvalMetric) -> Evaluator: """Returns an Evaluator for the given metric. @@ -48,15 +51,18 @@ class MetricEvaluatorRegistry: if eval_metric.metric_name not in self._registry: raise NotFoundError(f"{eval_metric.metric_name} not found in registry.") - return self._registry[eval_metric.metric_name](eval_metric=eval_metric) + return self._registry[eval_metric.metric_name][0](eval_metric=eval_metric) def register_evaluator( - self, metric_name: MetricName, evaluator: type[Evaluator] + self, + metric_info: MetricInfo, + evaluator: type[Evaluator], ): - """Registers an evaluator given the metric name. + """Registers an evaluator given the metric info. If a mapping already exist, then it is updated. """ + metric_name = metric_info.metric_name if metric_name in self._registry: logger.info( "Updating Evaluator class for %s from %s to %s", @@ -65,7 +71,16 @@ class MetricEvaluatorRegistry: evaluator, ) - self._registry[str(metric_name)] = evaluator + self._registry[str(metric_name)] = (evaluator, metric_info) + + def get_registered_metrics( + self, + ) -> list[MetricInfo]: + """Returns a list of MetricInfo about the metrics registered so far.""" + return [ + evaluator_and_metric_info[1].model_copy(deep=True) + for _, evaluator_and_metric_info in self._registry.items() + ] def _get_default_metric_evaluator_registry() -> MetricEvaluatorRegistry: @@ -73,23 +88,28 @@ def _get_default_metric_evaluator_registry() -> MetricEvaluatorRegistry: metric_evaluator_registry = MetricEvaluatorRegistry() metric_evaluator_registry.register_evaluator( - metric_name=PrebuiltMetrics.TOOL_TRAJECTORY_AVG_SCORE.value, + metric_info=TrajectoryEvaluator.get_metric_info(), evaluator=TrajectoryEvaluator, ) + metric_evaluator_registry.register_evaluator( - metric_name=PrebuiltMetrics.RESPONSE_EVALUATION_SCORE.value, + metric_info=ResponseEvaluator.get_metric_info( + PrebuiltMetrics.RESPONSE_EVALUATION_SCORE.value + ), evaluator=ResponseEvaluator, ) metric_evaluator_registry.register_evaluator( - metric_name=PrebuiltMetrics.RESPONSE_MATCH_SCORE.value, + metric_info=ResponseEvaluator.get_metric_info( + PrebuiltMetrics.RESPONSE_MATCH_SCORE.value + ), evaluator=ResponseEvaluator, ) metric_evaluator_registry.register_evaluator( - metric_name=PrebuiltMetrics.SAFETY_V1.value, + metric_info=SafetyEvaluatorV1.get_metric_info(), evaluator=SafetyEvaluatorV1, ) metric_evaluator_registry.register_evaluator( - metric_name=PrebuiltMetrics.FINAL_RESPONSE_MATCH_V2.value, + metric_info=FinalResponseMatchV2Evaluator.get_metric_info(), evaluator=FinalResponseMatchV2Evaluator, ) diff --git a/src/google/adk/evaluation/response_evaluator.py b/src/google/adk/evaluation/response_evaluator.py index b38d5553..fa6be8bf 100644 --- a/src/google/adk/evaluation/response_evaluator.py +++ b/src/google/adk/evaluation/response_evaluator.py @@ -21,6 +21,10 @@ from vertexai import types as vertexai_types from .eval_case import Invocation from .eval_metrics import EvalMetric +from .eval_metrics import Interval +from .eval_metrics import MetricInfo +from .eval_metrics import MetricValueInfo +from .eval_metrics import PrebuiltMetrics from .evaluator import EvaluationResult from .evaluator import Evaluator from .final_response_match_v1 import RougeEvaluator @@ -38,7 +42,7 @@ class ResponseEvaluator(Evaluator): 2) response_match_score: This metric evaluates if agent's final response matches a golden/expected - final response. + final response using Rouge_1 metric. Value range for this metric is [0,1], with values closer to 1 more desirable. """ @@ -61,15 +65,35 @@ class ResponseEvaluator(Evaluator): threshold = eval_metric.threshold metric_name = eval_metric.metric_name - if "response_evaluation_score" == metric_name: + if PrebuiltMetrics.RESPONSE_EVALUATION_SCORE.value == metric_name: self._metric_name = vertexai_types.PrebuiltMetric.COHERENCE - elif "response_match_score" == metric_name: - self._metric_name = "response_match_score" + elif PrebuiltMetrics.RESPONSE_MATCH_SCORE.value == metric_name: + self._metric_name = metric_name else: raise ValueError(f"`{metric_name}` is not supported.") self._threshold = threshold + @staticmethod + def get_metric_info(metric_name: str) -> MetricInfo: + """Returns MetricInfo for the given metric name.""" + if PrebuiltMetrics.RESPONSE_EVALUATION_SCORE.value == metric_name: + return MetricInfo( + metric_name=PrebuiltMetrics.RESPONSE_EVALUATION_SCORE.value, + description=( + "This metric evaluates how coherent agent's resposne was. Value" + " range of this metric is [1,5], with values closer to 5 more" + " desirable." + ), + metric_value_info=MetricValueInfo( + interval=Interval(min_value=1.0, max_value=5.0) + ), + ) + elif PrebuiltMetrics.RESPONSE_MATCH_SCORE.value == metric_name: + return RougeEvaluator.get_metric_info() + else: + raise ValueError(f"`{metric_name}` is not supported.") + @override def evaluate_invocations( self, @@ -77,7 +101,7 @@ class ResponseEvaluator(Evaluator): expected_invocations: list[Invocation], ) -> EvaluationResult: # If the metric is response_match_score, just use the RougeEvaluator. - if self._metric_name == "response_match_score": + if self._metric_name == PrebuiltMetrics.RESPONSE_MATCH_SCORE.value: rouge_evaluator = RougeEvaluator( EvalMetric(metric_name=self._metric_name, threshold=self._threshold) ) diff --git a/src/google/adk/evaluation/safety_evaluator.py b/src/google/adk/evaluation/safety_evaluator.py index 6b9ad242..f24931a2 100644 --- a/src/google/adk/evaluation/safety_evaluator.py +++ b/src/google/adk/evaluation/safety_evaluator.py @@ -19,6 +19,10 @@ from vertexai import types as vertexai_types from .eval_case import Invocation from .eval_metrics import EvalMetric +from .eval_metrics import Interval +from .eval_metrics import MetricInfo +from .eval_metrics import MetricValueInfo +from .eval_metrics import PrebuiltMetrics from .evaluator import EvaluationResult from .evaluator import Evaluator from .vertex_ai_eval_facade import _VertexAiEvalFacade @@ -42,6 +46,20 @@ class SafetyEvaluatorV1(Evaluator): def __init__(self, eval_metric: EvalMetric): self._eval_metric = eval_metric + @staticmethod + def get_metric_info() -> MetricInfo: + return MetricInfo( + metric_name=PrebuiltMetrics.SAFETY_V1.value, + description=( + "This metric evaluates the safety (harmlessness) of an Agent's" + " Response. Value range of the metric is [0, 1], with values closer" + " to 1 to be more desirable (safe)." + ), + metric_value_info=MetricValueInfo( + interval=Interval(min_value=0.0, max_value=1.0) + ), + ) + @override def evaluate_invocations( self, diff --git a/src/google/adk/evaluation/trajectory_evaluator.py b/src/google/adk/evaluation/trajectory_evaluator.py index 81566eb2..8f7508d4 100644 --- a/src/google/adk/evaluation/trajectory_evaluator.py +++ b/src/google/adk/evaluation/trajectory_evaluator.py @@ -25,6 +25,10 @@ from typing_extensions import override from .eval_case import Invocation from .eval_metrics import EvalMetric +from .eval_metrics import Interval +from .eval_metrics import MetricInfo +from .eval_metrics import MetricValueInfo +from .eval_metrics import PrebuiltMetrics from .evaluation_constants import EvalConstants from .evaluator import EvalStatus from .evaluator import EvaluationResult @@ -51,6 +55,22 @@ class TrajectoryEvaluator(Evaluator): self._threshold = threshold + @staticmethod + def get_metric_info() -> MetricInfo: + return MetricInfo( + metric_name=PrebuiltMetrics.TOOL_TRAJECTORY_AVG_SCORE.value, + description=( + "This metric compares two tool call trajectories (expected vs." + " actual) for the same user interaction. It performs an exact match" + " on the tool name and arguments for each step in the trajectory." + " A score of 1.0 indicates a perfect match, while 0.0 indicates a" + " mismatch. Higher values are better." + ), + metric_value_info=MetricValueInfo( + interval=Interval(min_value=0.0, max_value=1.0) + ), + ) + @override def evaluate_invocations( self, diff --git a/tests/unittests/cli/test_fast_api.py b/tests/unittests/cli/test_fast_api.py index 70d53034..f1c9e9d6 100755 --- a/tests/unittests/cli/test_fast_api.py +++ b/tests/unittests/cli/test_fast_api.py @@ -845,6 +845,23 @@ def test_run_eval(test_app, create_test_eval_set): assert data == [f"{info['app_name']}_test_eval_set_id_eval_result"] +def test_list_eval_metrics(test_app): + """Test listing eval metrics.""" + url = "/apps/test_app/eval_metrics" + response = test_app.get(url) + + # Verify the response + assert response.status_code == 200 + data = response.json() + assert isinstance(data, list) + # Add more assertions based on the expected metrics + assert len(data) > 0 + for metric in data: + assert "metricName" in metric + assert "description" in metric + assert "metricValueInfo" in metric + + def test_debug_trace(test_app): """Test the debug trace endpoint.""" # This test will likely return 404 since we haven't set up trace data, diff --git a/tests/unittests/evaluation/test_final_response_match_v1.py b/tests/unittests/evaluation/test_final_response_match_v1.py index d5544a5a..d5fe0464 100644 --- a/tests/unittests/evaluation/test_final_response_match_v1.py +++ b/tests/unittests/evaluation/test_final_response_match_v1.py @@ -16,6 +16,7 @@ from __future__ import annotations from google.adk.evaluation.eval_case import Invocation from google.adk.evaluation.eval_metrics import EvalMetric +from google.adk.evaluation.eval_metrics import PrebuiltMetrics from google.adk.evaluation.evaluator import EvalStatus from google.adk.evaluation.final_response_match_v1 import _calculate_rouge_1_scores from google.adk.evaluation.final_response_match_v1 import RougeEvaluator @@ -138,3 +139,11 @@ def test_rouge_evaluator_multiple_invocations( expected_score, rel=1e-3 ) assert evaluation_result.overall_eval_status == expected_status + + +def test_get_metric_info(): + """Test get_metric_info function for response match metric.""" + metric_info = RougeEvaluator.get_metric_info() + assert metric_info.metric_name == PrebuiltMetrics.RESPONSE_MATCH_SCORE.value + assert metric_info.metric_value_info.interval.min_value == 0.0 + assert metric_info.metric_value_info.interval.max_value == 1.0 diff --git a/tests/unittests/evaluation/test_final_response_match_v2.py b/tests/unittests/evaluation/test_final_response_match_v2.py index 859e6d20..911c5e22 100644 --- a/tests/unittests/evaluation/test_final_response_match_v2.py +++ b/tests/unittests/evaluation/test_final_response_match_v2.py @@ -17,6 +17,7 @@ from __future__ import annotations from google.adk.evaluation.eval_case import Invocation from google.adk.evaluation.eval_metrics import EvalMetric from google.adk.evaluation.eval_metrics import JudgeModelOptions +from google.adk.evaluation.eval_metrics import PrebuiltMetrics from google.adk.evaluation.evaluator import EvalStatus from google.adk.evaluation.evaluator import PerInvocationResult from google.adk.evaluation.final_response_match_v2 import _parse_critique @@ -476,3 +477,13 @@ def test_aggregate_invocation_results(): # Only 4 / 8 invocations are evaluated, and 2 / 4 are valid. assert aggregated_result.overall_score == 0.5 assert aggregated_result.overall_eval_status == EvalStatus.PASSED + + +def test_get_metric_info(): + """Test get_metric_info function for Final Response Match V2 metric.""" + metric_info = FinalResponseMatchV2Evaluator.get_metric_info() + assert ( + metric_info.metric_name == PrebuiltMetrics.FINAL_RESPONSE_MATCH_V2.value + ) + assert metric_info.metric_value_info.interval.min_value == 0.0 + assert metric_info.metric_value_info.interval.max_value == 1.0 diff --git a/tests/unittests/evaluation/test_local_eval_service.py b/tests/unittests/evaluation/test_local_eval_service.py index 5353f1f1..49ebead2 100644 --- a/tests/unittests/evaluation/test_local_eval_service.py +++ b/tests/unittests/evaluation/test_local_eval_service.py @@ -24,6 +24,9 @@ from google.adk.evaluation.base_eval_service import InferenceResult from google.adk.evaluation.eval_case import Invocation from google.adk.evaluation.eval_metrics import EvalMetric from google.adk.evaluation.eval_metrics import EvalMetricResult +from google.adk.evaluation.eval_metrics import Interval +from google.adk.evaluation.eval_metrics import MetricInfo +from google.adk.evaluation.eval_metrics import MetricValueInfo from google.adk.evaluation.eval_result import EvalCaseResult from google.adk.evaluation.eval_set import EvalCase from google.adk.evaluation.eval_set import EvalSet @@ -61,7 +64,7 @@ def eval_service( dummy_agent, mock_eval_sets_manager, mock_eval_set_results_manager ): DEFAULT_METRIC_EVALUATOR_REGISTRY.register_evaluator( - metric_name="fake_metric", evaluator=FakeEvaluator + metric_info=FakeEvaluator.get_metric_info(), evaluator=FakeEvaluator ) return LocalEvalService( root_agent=dummy_agent, @@ -75,6 +78,16 @@ class FakeEvaluator(Evaluator): def __init__(self, eval_metric: EvalMetric): self._eval_metric = eval_metric + @staticmethod + def get_metric_info() -> MetricInfo: + return MetricInfo( + metric_name="fake_metric", + description="Fake metric description", + metric_value_info=MetricValueInfo( + interval=Interval(min_value=0.0, max_value=1.0) + ), + ) + def evaluate_invocations( self, actual_invocations: list[Invocation], diff --git a/tests/unittests/evaluation/test_metric_evaluator_registry.py b/tests/unittests/evaluation/test_metric_evaluator_registry.py index f36acc41..60b39d54 100644 --- a/tests/unittests/evaluation/test_metric_evaluator_registry.py +++ b/tests/unittests/evaluation/test_metric_evaluator_registry.py @@ -16,10 +16,15 @@ from __future__ import annotations from google.adk.errors.not_found_error import NotFoundError from google.adk.evaluation.eval_metrics import EvalMetric +from google.adk.evaluation.eval_metrics import Interval +from google.adk.evaluation.eval_metrics import MetricInfo +from google.adk.evaluation.eval_metrics import MetricValueInfo from google.adk.evaluation.evaluator import Evaluator from google.adk.evaluation.metric_evaluator_registry import MetricEvaluatorRegistry import pytest +_DUMMY_METRIC_NAME = "dummy_metric_name" + class TestMetricEvaluatorRegistry: """Test cases for MetricEvaluatorRegistry.""" @@ -36,6 +41,16 @@ class TestMetricEvaluatorRegistry: def evaluate_invocations(self, actual_invocations, expected_invocations): return "dummy_result" + @staticmethod + def get_metric_info() -> MetricInfo: + return MetricInfo( + metric_name=_DUMMY_METRIC_NAME, + description="Dummy metric description", + metric_value_info=MetricValueInfo( + interval=Interval(min_value=0.0, max_value=1.0) + ), + ) + class AnotherDummyEvaluator(Evaluator): def __init__(self, eval_metric: EvalMetric): @@ -44,45 +59,58 @@ class TestMetricEvaluatorRegistry: def evaluate_invocations(self, actual_invocations, expected_invocations): return "another_dummy_result" + @staticmethod + def get_metric_info() -> MetricInfo: + return MetricInfo( + metric_name=_DUMMY_METRIC_NAME, + description="Another dummy metric description", + metric_value_info=MetricValueInfo( + interval=Interval(min_value=0.0, max_value=1.0) + ), + ) + def test_register_evaluator(self, registry): - dummy_metric_name = "dummy_metric_name" + metric_info = TestMetricEvaluatorRegistry.DummyEvaluator.get_metric_info() registry.register_evaluator( - dummy_metric_name, + metric_info, TestMetricEvaluatorRegistry.DummyEvaluator, ) - assert dummy_metric_name in registry._registry - assert ( - registry._registry[dummy_metric_name] - == TestMetricEvaluatorRegistry.DummyEvaluator + assert _DUMMY_METRIC_NAME in registry._registry + assert registry._registry[_DUMMY_METRIC_NAME] == ( + TestMetricEvaluatorRegistry.DummyEvaluator, + metric_info, ) def test_register_evaluator_updates_existing(self, registry): - dummy_metric_name = "dummy_metric_name" + metric_info = TestMetricEvaluatorRegistry.DummyEvaluator.get_metric_info() registry.register_evaluator( - dummy_metric_name, + metric_info, TestMetricEvaluatorRegistry.DummyEvaluator, ) - assert ( - registry._registry[dummy_metric_name] - == TestMetricEvaluatorRegistry.DummyEvaluator + assert registry._registry[_DUMMY_METRIC_NAME] == ( + TestMetricEvaluatorRegistry.DummyEvaluator, + metric_info, ) - registry.register_evaluator( - dummy_metric_name, TestMetricEvaluatorRegistry.AnotherDummyEvaluator + metric_info = ( + TestMetricEvaluatorRegistry.AnotherDummyEvaluator.get_metric_info() ) - assert ( - registry._registry[dummy_metric_name] - == TestMetricEvaluatorRegistry.AnotherDummyEvaluator + registry.register_evaluator( + metric_info, TestMetricEvaluatorRegistry.AnotherDummyEvaluator + ) + assert registry._registry[_DUMMY_METRIC_NAME] == ( + TestMetricEvaluatorRegistry.AnotherDummyEvaluator, + metric_info, ) def test_get_evaluator(self, registry): - dummy_metric_name = "dummy_metric_name" + metric_info = TestMetricEvaluatorRegistry.DummyEvaluator.get_metric_info() registry.register_evaluator( - dummy_metric_name, + metric_info, TestMetricEvaluatorRegistry.DummyEvaluator, ) - eval_metric = EvalMetric(metric_name=dummy_metric_name, threshold=0.5) + eval_metric = EvalMetric(metric_name=_DUMMY_METRIC_NAME, threshold=0.5) evaluator = registry.get_evaluator(eval_metric) assert isinstance(evaluator, TestMetricEvaluatorRegistry.DummyEvaluator) diff --git a/tests/unittests/evaluation/test_response_evaluator.py b/tests/unittests/evaluation/test_response_evaluator.py index 09946772..bace9c6a 100644 --- a/tests/unittests/evaluation/test_response_evaluator.py +++ b/tests/unittests/evaluation/test_response_evaluator.py @@ -16,6 +16,7 @@ from unittest.mock import patch from google.adk.evaluation.eval_case import Invocation +from google.adk.evaluation.eval_metrics import PrebuiltMetrics from google.adk.evaluation.evaluator import EvalStatus from google.adk.evaluation.response_evaluator import ResponseEvaluator from google.genai import types as genai_types @@ -113,3 +114,29 @@ class TestResponseEvaluator: assert [m.name for m in mock_kwargs["metrics"]] == [ vertexai_types.PrebuiltMetric.COHERENCE.name ] + + def test_get_metric_info_response_evaluation_score(self, mock_perform_eval): + """Test get_metric_info function for response evaluation metric.""" + metric_info = ResponseEvaluator.get_metric_info( + PrebuiltMetrics.RESPONSE_EVALUATION_SCORE.value + ) + assert ( + metric_info.metric_name + == PrebuiltMetrics.RESPONSE_EVALUATION_SCORE.value + ) + assert metric_info.metric_value_info.interval.min_value == 1.0 + assert metric_info.metric_value_info.interval.max_value == 5.0 + + def test_get_metric_info_response_match_score(self, mock_perform_eval): + """Test get_metric_info function for response match metric.""" + metric_info = ResponseEvaluator.get_metric_info( + PrebuiltMetrics.RESPONSE_MATCH_SCORE.value + ) + assert metric_info.metric_name == PrebuiltMetrics.RESPONSE_MATCH_SCORE.value + assert metric_info.metric_value_info.interval.min_value == 0.0 + assert metric_info.metric_value_info.interval.max_value == 1.0 + + def test_get_metric_info_invalid(self, mock_perform_eval): + """Test get_metric_info function for invalid metric.""" + with pytest.raises(ValueError): + ResponseEvaluator.get_metric_info("invalid_metric") diff --git a/tests/unittests/evaluation/test_safety_evaluator.py b/tests/unittests/evaluation/test_safety_evaluator.py index 077e3143..5cc95b1d 100644 --- a/tests/unittests/evaluation/test_safety_evaluator.py +++ b/tests/unittests/evaluation/test_safety_evaluator.py @@ -17,6 +17,7 @@ from unittest.mock import patch from google.adk.evaluation.eval_case import Invocation from google.adk.evaluation.eval_metrics import EvalMetric +from google.adk.evaluation.eval_metrics import PrebuiltMetrics from google.adk.evaluation.evaluator import EvalStatus from google.adk.evaluation.safety_evaluator import SafetyEvaluatorV1 from google.genai import types as genai_types @@ -76,3 +77,10 @@ class TestSafetyEvaluatorV1: assert [m.name for m in mock_kwargs["metrics"]] == [ vertexai_types.PrebuiltMetric.SAFETY.name ] + + def test_get_metric_info(self, mock_perform_eval): + """Test get_metric_info function for Safety metric.""" + metric_info = SafetyEvaluatorV1.get_metric_info() + assert metric_info.metric_name == PrebuiltMetrics.SAFETY_V1.value + assert metric_info.metric_value_info.interval.min_value == 0.0 + assert metric_info.metric_value_info.interval.max_value == 1.0 diff --git a/tests/unittests/evaluation/test_trajectory_evaluator.py b/tests/unittests/evaluation/test_trajectory_evaluator.py index f3622a53..a8053dd1 100644 --- a/tests/unittests/evaluation/test_trajectory_evaluator.py +++ b/tests/unittests/evaluation/test_trajectory_evaluator.py @@ -16,6 +16,7 @@ import math +from google.adk.evaluation.eval_metrics import PrebuiltMetrics from google.adk.evaluation.trajectory_evaluator import TrajectoryEvaluator import pytest @@ -270,3 +271,13 @@ def test_are_tools_equal_one_empty_one_not(): list_a = [] list_b = [TOOL_GET_WEATHER] assert not TrajectoryEvaluator.are_tools_equal(list_a, list_b) + + +def test_get_metric_info(): + """Test get_metric_info function for tool trajectory avg metric.""" + metric_info = TrajectoryEvaluator.get_metric_info() + assert ( + metric_info.metric_name == PrebuiltMetrics.TOOL_TRAJECTORY_AVG_SCORE.value + ) + assert metric_info.metric_value_info.interval.min_value == 0.0 + assert metric_info.metric_value_info.interval.max_value == 1.0