diff --git a/src/google/adk/evaluation/vertex_ai_eval_facade.py b/src/google/adk/evaluation/vertex_ai_eval_facade.py index 7adab34e..492fe448 100644 --- a/src/google/adk/evaluation/vertex_ai_eval_facade.py +++ b/src/google/adk/evaluation/vertex_ai_eval_facade.py @@ -14,6 +14,7 @@ from __future__ import annotations +import math import os from typing import Optional @@ -112,7 +113,12 @@ class _VertexAiEvalFacade(Evaluator): return "" def _get_score(self, eval_result) -> Optional[float]: - if eval_result and eval_result.summary_metrics: + if ( + eval_result + and eval_result.summary_metrics + and isinstance(eval_result.summary_metrics[0].mean_score, float) + and not math.isnan(eval_result.summary_metrics[0].mean_score) + ): return eval_result.summary_metrics[0].mean_score return None diff --git a/tests/unittests/evaluation/test_vertex_ai_eval_facade.py b/tests/unittests/evaluation/test_vertex_ai_eval_facade.py index 63468be0..8fd1705c 100644 --- a/tests/unittests/evaluation/test_vertex_ai_eval_facade.py +++ b/tests/unittests/evaluation/test_vertex_ai_eval_facade.py @@ -13,6 +13,7 @@ # limitations under the License. """Tests for the Response Evaluator.""" +import math import random from unittest.mock import patch @@ -122,7 +123,18 @@ class TestVertexAiEvalFacade: vertexai_types.PrebuiltMetric.COHERENCE.name ] - def test_evaluate_invocations_metric_no_score(self, mock_perform_eval): + @pytest.mark.parametrize( + "summary_metric_with_no_score", + [ + ([]), + ([vertexai_types.AggregatedMetricResult(mean_score=float("nan"))]), + ([vertexai_types.AggregatedMetricResult(mean_score=None)]), + ([vertexai_types.AggregatedMetricResult(mean_score=math.nan)]), + ], + ) + def test_evaluate_invocations_metric_no_score( + self, mock_perform_eval, summary_metric_with_no_score + ): """Test evaluate_invocations function for a metric.""" actual_invocations = [ Invocation( @@ -151,7 +163,7 @@ class TestVertexAiEvalFacade: ) # Mock the return value of _perform_eval mock_perform_eval.return_value = vertexai_types.EvaluationResult( - summary_metrics=[], + summary_metrics=summary_metric_with_no_score, eval_case_results=[], )