diff --git a/pyproject.toml b/pyproject.toml index 48105974..3b4ef0be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,7 @@ a2a = [ eval = [ # go/keep-sorted start - "google-cloud-aiplatform[evaluation]>=1.87.0", + "google-cloud-aiplatform[evaluation]>=1.100.0", "pandas>=2.2.3", "tabulate>=0.9.0", "rouge-score>=0.1.2", diff --git a/src/google/adk/evaluation/response_evaluator.py b/src/google/adk/evaluation/response_evaluator.py index 2f383e4b..1a73df87 100644 --- a/src/google/adk/evaluation/response_evaluator.py +++ b/src/google/adk/evaluation/response_evaluator.py @@ -14,18 +14,15 @@ from __future__ import annotations -from typing import Any +import os from typing import Optional from google.genai import types as genai_types import pandas as pd -from tabulate import tabulate -from typing_extensions import deprecated from typing_extensions import override -from vertexai.preview.evaluation import EvalTask -from vertexai.preview.evaluation import MetricPromptTemplateExamples +from vertexai import Client as VertexAiClient +from vertexai import types as vertexai_types -from .eval_case import IntermediateData from .eval_case import Invocation from .eval_metrics import EvalMetric from .evaluator import EvalStatus @@ -57,7 +54,7 @@ class ResponseEvaluator(Evaluator): metric_name = eval_metric.metric_name if "response_evaluation_score" == metric_name: - self._metric_name = MetricPromptTemplateExamples.Pointwise.COHERENCE + self._metric_name = vertexai_types.PrebuiltMetric.COHERENCE elif "response_match_score" == metric_name: self._metric_name = "response_match_score" else: @@ -87,17 +84,11 @@ class ResponseEvaluator(Evaluator): prompt = self._get_text(expected.user_content) reference = self._get_text(expected.final_response) response = self._get_text(actual.final_response) - actual_tool_use = self._get_tool_use_trajectory(actual.intermediate_data) - reference_trajectory = self._get_tool_use_trajectory( - expected.intermediate_data - ) eval_case = { "prompt": prompt, "reference": reference, "response": response, - "actual_tool_user": actual_tool_use, - "reference_trajectory": reference_trajectory, } eval_case_result = ResponseEvaluator._perform_eval( @@ -112,11 +103,15 @@ class ResponseEvaluator(Evaluator): eval_status=self._get_eval_status(score), ) ) - total_score += score - num_invocations += 1 + + if score: + total_score += score + num_invocations += 1 if per_invocation_results: - overall_score = total_score / num_invocations + overall_score = ( + total_score / num_invocations if num_invocations > 0 else None + ) return EvaluationResult( overall_score=overall_score, overall_eval_status=self._get_eval_status(overall_score), @@ -131,138 +126,19 @@ class ResponseEvaluator(Evaluator): return "" - def _get_tool_use_trajectory( - self, intermediate_data: Optional[IntermediateData] - ) -> list[dict[str, Any]]: - tool_use_trajectory = [] - if not intermediate_data: - return tool_use_trajectory + def _get_score(self, eval_result) -> Optional[float]: + if eval_result and eval_result.summary_metrics: + return eval_result.summary_metrics[0].mean_score - for function_call in intermediate_data.tool_uses: - tool_use_trajectory.append({ - "tool_name": function_call.name, - "tool_input": function_call.args or {}, - }) + return None - return tool_use_trajectory + def _get_eval_status(self, score: Optional[float]): + if score: + return ( + EvalStatus.PASSED if score >= self._threshold else EvalStatus.FAILED + ) - def _get_score(self, eval_result) -> float: - return eval_result.summary_metrics[f"{self._metric_name}/mean"].item() - - def _get_eval_status(self, score: float): - return EvalStatus.PASSED if score >= self._threshold else EvalStatus.FAILED - - @staticmethod - @deprecated( - "This method has been deprecated and will be removed soon. Please use" - " evaluate_invocations instead." - ) - def evaluate( - raw_eval_dataset: list[list[dict[str, Any]]], - evaluation_criteria: list[str], - *, - print_detailed_results: bool = False, - ): - r"""Returns the value of requested evaluation metrics. - - Args: - raw_eval_dataset: The dataset that will be evaluated. - evaluation_criteria: The evaluation criteria to be used. This method - support two criteria, `response_evaluation_score` and - `response_match_score`. - print_detailed_results: Prints detailed results on the console. This is - usually helpful during debugging. - - A note on evaluation_criteria: - `response_match_score`: This metric compares the agents final natural - language response with the expected final response, stored in the - "reference" field in test/eval files. We use Rouge metric to compare the - two responses. - - Value Range: [0, 1]. A score closer to 0 means poor similarity between - response and reference. A score closer to 1 means strong similarity - between response and reference. - - `response_evaluation_score`: Uses LLM to evalaute coherence of the - response, including tool use. This is pointwise metric. - - Value range: [0, 5], where 0 means that the agent's response is not - coherent, while 5 means it is . High values are good. - A note on raw_eval_dataset: - The dataset should be a list session, where each session is represented - as a list of interaction that need evaluation. Each evaluation is - represented as a dictionary that is expected to have values for the - following keys: - - 1) query - 2) response - 3) acutal_tool_use - 4) expected_tool_use - 5) reference - - Here is a sample eval_dataset value with one entry: - [ - [ - { - "query": "roll a die for me", - "response": "I rolled a 16 sided die and got 13.\n", - "expected_tool_use": [ - { - "tool_name": "roll_die", - "tool_input": { - "sides": 16 - } - } - ], - "acutal_tool_use": [ - { - "tool_name": "roll_die", - "tool_input": { - "sides": 16 - } - } - ], - "reference": "I rolled a 16 sided die and got 13.\n" - } - ] - ] - """ - if not raw_eval_dataset: - raise ValueError("The evaluation dataset is empty.") - - metrics = ResponseEvaluator._get_metrics( - raw_eval_dataset, evaluation_criteria - ) - flattened_queries = [ - item for sublist in raw_eval_dataset for item in sublist - ] - eval_dataset = pd.DataFrame(flattened_queries).rename( - columns={"query": "prompt", "expected_tool_use": "reference_trajectory"} - ) - - eval_result = ResponseEvaluator._perform_eval( - dataset=eval_dataset, metrics=metrics - ) - - if print_detailed_results: - ResponseEvaluator._print_results(eval_result) - return eval_result.summary_metrics - - @staticmethod - def _get_metrics(raw_eval_dataset, criteria): - metrics = [] - if ( - "response_evaluation_score" in criteria - and "query" in raw_eval_dataset[0][0] - and "expected_tool_use" in raw_eval_dataset[0][0] - ): - metrics.append(MetricPromptTemplateExamples.Pointwise.COHERENCE) - if ( - "response_match_score" in criteria - and "reference" in raw_eval_dataset[0][0] - ): - metrics.append("rouge_1") - return metrics + return EvalStatus.NOT_EVALUATED @staticmethod def _perform_eval(dataset, metrics): @@ -270,11 +146,11 @@ class ResponseEvaluator(Evaluator): Primarily helps with unit testing. """ - eval_task = EvalTask(dataset=dataset, metrics=metrics) + project_id = str(os.environ.get("GOOGLE_CLOUD_PROJECT")) + location = os.environ.get("GOOGLE_CLOUD_REGION") + client = VertexAiClient(project=project_id, location=location) - return eval_task.evaluate() - - @staticmethod - def _print_results(eval_result): - print("Evaluation Summary Metrics:", eval_result.summary_metrics) - print(tabulate(eval_result.metrics_table, headers="keys", tablefmt="grid")) + return client.evals.evaluate( + dataset=vertexai_types.EvaluationDataset(eval_dataset_df=dataset), + metrics=metrics, + ) diff --git a/tests/unittests/evaluation/test_response_evaluator.py b/tests/unittests/evaluation/test_response_evaluator.py index 839b7188..a1dc3aac 100644 --- a/tests/unittests/evaluation/test_response_evaluator.py +++ b/tests/unittests/evaluation/test_response_evaluator.py @@ -13,53 +13,15 @@ # limitations under the License. """Tests for the Response Evaluator.""" -from unittest.mock import MagicMock +import random from unittest.mock import patch from google.adk.evaluation.eval_case import Invocation from google.adk.evaluation.evaluator import EvalStatus from google.adk.evaluation.response_evaluator import ResponseEvaluator from google.genai import types as genai_types -import pandas as pd import pytest -from vertexai.preview.evaluation import MetricPromptTemplateExamples - -# Mock object for the result normally returned by _perform_eval -MOCK_EVAL_RESULT = MagicMock() -MOCK_EVAL_RESULT.summary_metrics = {"mock_metric": 0.75, "another_mock": 3.5} -# Add a metrics_table for testing _print_results interaction -MOCK_EVAL_RESULT.metrics_table = pd.DataFrame({ - "prompt": ["mock_query1"], - "response": ["mock_resp1"], - "mock_metric": [0.75], -}) - -SAMPLE_TURN_1_ALL_KEYS = { - "query": "query1", - "response": "response1", - "actual_tool_use": [{"tool_name": "tool_a", "tool_input": {}}], - "expected_tool_use": [{"tool_name": "tool_a", "tool_input": {}}], - "reference": "reference1", -} -SAMPLE_TURN_2_MISSING_REF = { - "query": "query2", - "response": "response2", - "actual_tool_use": [], - "expected_tool_use": [], - # "reference": "reference2" # Missing -} -SAMPLE_TURN_3_MISSING_EXP_TOOLS = { - "query": "query3", - "response": "response3", - "actual_tool_use": [{"tool_name": "tool_b", "tool_input": {}}], - # "expected_tool_use": [], # Missing - "reference": "reference3", -} -SAMPLE_TURN_4_MINIMAL = { - "query": "query4", - "response": "response4", - # Minimal keys, others missing -} +from vertexai import types as vertexai_types @patch( @@ -68,18 +30,6 @@ SAMPLE_TURN_4_MINIMAL = { class TestResponseEvaluator: """A class to help organize "patch" that are applicable to all tests.""" - def test_evaluate_none_dataset_raises_value_error(self, mock_perform_eval): - """Test evaluate function raises ValueError for an empty list.""" - with pytest.raises(ValueError, match="The evaluation dataset is empty."): - ResponseEvaluator.evaluate(None, ["response_evaluation_score"]) - mock_perform_eval.assert_not_called() # Ensure _perform_eval was not called - - def test_evaluate_empty_dataset_raises_value_error(self, mock_perform_eval): - """Test evaluate function raises ValueError for an empty list.""" - with pytest.raises(ValueError, match="The evaluation dataset is empty."): - ResponseEvaluator.evaluate([], ["response_evaluation_score"]) - mock_perform_eval.assert_not_called() # Ensure _perform_eval was not called - def test_evaluate_invocations_rouge_metric(self, mock_perform_eval): """Test evaluate_invocations function for Rouge metric.""" actual_invocations = [ @@ -107,190 +57,198 @@ class TestResponseEvaluator: evaluator = ResponseEvaluator( threshold=0.8, metric_name="response_match_score" ) + evaluation_result = evaluator.evaluate_invocations( actual_invocations, expected_invocations ) + assert evaluation_result.overall_score == pytest.approx(8 / 11) # ROUGE-1 F1 is approx. 0.73 < 0.8 threshold, so eval status is FAILED. assert evaluation_result.overall_eval_status == EvalStatus.FAILED + mock_perform_eval.assert_not_called() # Ensure _perform_eval was not called - def test_evaluate_determines_metrics_correctly_for_perform_eval( + def test_evaluate_invocations_coherence_metric_passed( self, mock_perform_eval ): - """Test that the correct metrics list is passed to _perform_eval based on criteria/keys.""" - mock_perform_eval.return_value = MOCK_EVAL_RESULT - - # Test case 1: Only Coherence - raw_data_1 = [[SAMPLE_TURN_1_ALL_KEYS]] - criteria_1 = ["response_evaluation_score"] - ResponseEvaluator.evaluate(raw_data_1, criteria_1) - _, kwargs = mock_perform_eval.call_args - assert kwargs["metrics"] == [ - MetricPromptTemplateExamples.Pointwise.COHERENCE + """Test evaluate_invocations function for Coherence metric.""" + actual_invocations = [ + Invocation( + user_content=genai_types.Content( + parts=[genai_types.Part(text="This is a test query.")] + ), + final_response=genai_types.Content( + parts=[ + genai_types.Part(text="This is a test candidate response.") + ] + ), + ) ] - mock_perform_eval.reset_mock() # Reset mock for next call - - # Test case 2: Only Rouge - raw_data_2 = [[SAMPLE_TURN_1_ALL_KEYS]] - criteria_2 = ["response_match_score"] - ResponseEvaluator.evaluate(raw_data_2, criteria_2) - _, kwargs = mock_perform_eval.call_args - assert kwargs["metrics"] == ["rouge_1"] - mock_perform_eval.reset_mock() - - # Test case 3: No metrics if keys missing in first turn - raw_data_3 = [[SAMPLE_TURN_4_MINIMAL, SAMPLE_TURN_1_ALL_KEYS]] - criteria_3 = ["response_evaluation_score", "response_match_score"] - ResponseEvaluator.evaluate(raw_data_3, criteria_3) - _, kwargs = mock_perform_eval.call_args - assert kwargs["metrics"] == [] - mock_perform_eval.reset_mock() - - # Test case 4: No metrics if criteria empty - raw_data_4 = [[SAMPLE_TURN_1_ALL_KEYS]] - criteria_4 = [] - ResponseEvaluator.evaluate(raw_data_4, criteria_4) - _, kwargs = mock_perform_eval.call_args - assert kwargs["metrics"] == [] - mock_perform_eval.reset_mock() - - def test_evaluate_calls_perform_eval_correctly_all_metrics( - self, mock_perform_eval - ): - """Test evaluate function calls _perform_eval with expected args when all criteria/keys are present.""" - # Arrange - mock_perform_eval.return_value = ( - MOCK_EVAL_RESULT # Configure the mock return value + expected_invocations = [ + Invocation( + user_content=genai_types.Content( + parts=[genai_types.Part(text="This is a test query.")] + ), + final_response=genai_types.Content( + parts=[genai_types.Part(text="This is a test reference.")] + ), + ) + ] + evaluator = ResponseEvaluator( + threshold=0.8, metric_name="response_evaluation_score" + ) + # Mock the return value of _perform_eval + mock_perform_eval.return_value = vertexai_types.EvaluationResult( + summary_metrics=[vertexai_types.AggregatedMetricResult(mean_score=0.9)], + eval_case_results=[], ) - raw_data = [[SAMPLE_TURN_1_ALL_KEYS]] - criteria = ["response_evaluation_score", "response_match_score"] - - # Act - summary = ResponseEvaluator.evaluate(raw_data, criteria) - - # Assert - # 1. Check metrics determined by _get_metrics (passed to _perform_eval) - expected_metrics_list = [ - MetricPromptTemplateExamples.Pointwise.COHERENCE, - "rouge_1", - ] - # 2. Check DataFrame prepared (passed to _perform_eval) - expected_df_data = [{ - "prompt": "query1", - "response": "response1", - "actual_tool_use": [{"tool_name": "tool_a", "tool_input": {}}], - "reference_trajectory": [{"tool_name": "tool_a", "tool_input": {}}], - "reference": "reference1", - }] - expected_df = pd.DataFrame(expected_df_data) - - # Assert _perform_eval was called once - mock_perform_eval.assert_called_once() - # Get the arguments passed to the mocked _perform_eval - _, kwargs = mock_perform_eval.call_args - # Check the 'dataset' keyword argument - pd.testing.assert_frame_equal(kwargs["dataset"], expected_df) - # Check the 'metrics' keyword argument - assert kwargs["metrics"] == expected_metrics_list - - # 3. Check the correct summary metrics are returned - # (from mock_perform_eval's return value) - assert summary == MOCK_EVAL_RESULT.summary_metrics - - def test_evaluate_prepares_dataframe_correctly_for_perform_eval( - self, mock_perform_eval - ): - """Test that the DataFrame is correctly flattened and renamed before passing to _perform_eval.""" - mock_perform_eval.return_value = MOCK_EVAL_RESULT - - raw_data = [ - [SAMPLE_TURN_1_ALL_KEYS], # Conversation 1 - [ - SAMPLE_TURN_2_MISSING_REF, - SAMPLE_TURN_3_MISSING_EXP_TOOLS, - ], # Conversation 2 - ] - criteria = [ - "response_match_score" - ] # Doesn't affect the DataFrame structure - - ResponseEvaluator.evaluate(raw_data, criteria) - - # Expected flattened and renamed data - expected_df_data = [ - # Turn 1 (from SAMPLE_TURN_1_ALL_KEYS) - { - "prompt": "query1", - "response": "response1", - "actual_tool_use": [{"tool_name": "tool_a", "tool_input": {}}], - "reference_trajectory": [{"tool_name": "tool_a", "tool_input": {}}], - "reference": "reference1", - }, - # Turn 2 (from SAMPLE_TURN_2_MISSING_REF) - { - "prompt": "query2", - "response": "response2", - "actual_tool_use": [], - "reference_trajectory": [], - # "reference": None # Missing key results in NaN in DataFrame - # usually - }, - # Turn 3 (from SAMPLE_TURN_3_MISSING_EXP_TOOLS) - { - "prompt": "query3", - "response": "response3", - "actual_tool_use": [{"tool_name": "tool_b", "tool_input": {}}], - # "reference_trajectory": None, # Missing key results in NaN - "reference": "reference3", - }, - ] - # Need to be careful with missing keys -> NaN when creating DataFrame - # Pandas handles this automatically when creating from list of dicts - expected_df = pd.DataFrame(expected_df_data) - - mock_perform_eval.assert_called_once() - _, kwargs = mock_perform_eval.call_args - # Compare the DataFrame passed to the mock - pd.testing.assert_frame_equal(kwargs["dataset"], expected_df) - - @patch( - "google.adk.evaluation.response_evaluator.ResponseEvaluator._print_results" - ) # Mock the private print method - def test_evaluate_print_detailed_results( - self, mock_print_results, mock_perform_eval - ): - """Test _print_results function is called when print_detailed_results=True.""" - mock_perform_eval.return_value = ( - MOCK_EVAL_RESULT # Ensure _perform_eval returns our mock result + evaluation_result = evaluator.evaluate_invocations( + actual_invocations, expected_invocations ) - raw_data = [[SAMPLE_TURN_1_ALL_KEYS]] - criteria = ["response_match_score"] - - ResponseEvaluator.evaluate(raw_data, criteria, print_detailed_results=True) - - # Assert _perform_eval was called + assert evaluation_result.overall_score == 0.9 + assert evaluation_result.overall_eval_status == EvalStatus.PASSED mock_perform_eval.assert_called_once() - # Assert _print_results was called once with the result object - # from _perform_eval - mock_print_results.assert_called_once_with(MOCK_EVAL_RESULT) - @patch( - "google.adk.evaluation.response_evaluator.ResponseEvaluator._print_results" - ) - def test_evaluate_no_print_detailed_results( - self, mock_print_results, mock_perform_eval + def test_evaluate_invocations_coherence_metric_failed( + self, mock_perform_eval ): - """Test _print_results function is NOT called when print_detailed_results=False (default).""" - mock_perform_eval.return_value = MOCK_EVAL_RESULT + """Test evaluate_invocations function for Coherence metric.""" + actual_invocations = [ + Invocation( + user_content=genai_types.Content( + parts=[genai_types.Part(text="This is a test query.")] + ), + final_response=genai_types.Content( + parts=[ + genai_types.Part(text="This is a test candidate response.") + ] + ), + ) + ] + expected_invocations = [ + Invocation( + user_content=genai_types.Content( + parts=[genai_types.Part(text="This is a test query.")] + ), + final_response=genai_types.Content( + parts=[genai_types.Part(text="This is a test reference.")] + ), + ) + ] + evaluator = ResponseEvaluator( + threshold=0.8, metric_name="response_evaluation_score" + ) + # Mock the return value of _perform_eval + mock_perform_eval.return_value = vertexai_types.EvaluationResult( + summary_metrics=[vertexai_types.AggregatedMetricResult(mean_score=0.7)], + eval_case_results=[], + ) - raw_data = [[SAMPLE_TURN_1_ALL_KEYS]] - criteria = ["response_match_score"] + evaluation_result = evaluator.evaluate_invocations( + actual_invocations, expected_invocations + ) - ResponseEvaluator.evaluate(raw_data, criteria, print_detailed_results=False) - - # Assert _perform_eval was called + assert evaluation_result.overall_score == 0.7 + assert evaluation_result.overall_eval_status == EvalStatus.FAILED mock_perform_eval.assert_called_once() - # Assert _print_results was NOT called - mock_print_results.assert_not_called() + + def test_evaluate_invocations_coherence_metric_no_score( + self, mock_perform_eval + ): + """Test evaluate_invocations function for Coherence metric.""" + actual_invocations = [ + Invocation( + user_content=genai_types.Content( + parts=[genai_types.Part(text="This is a test query.")] + ), + final_response=genai_types.Content( + parts=[ + genai_types.Part(text="This is a test candidate response.") + ] + ), + ) + ] + expected_invocations = [ + Invocation( + user_content=genai_types.Content( + parts=[genai_types.Part(text="This is a test query.")] + ), + final_response=genai_types.Content( + parts=[genai_types.Part(text="This is a test reference.")] + ), + ) + ] + evaluator = ResponseEvaluator( + threshold=0.8, metric_name="response_evaluation_score" + ) + # Mock the return value of _perform_eval + mock_perform_eval.return_value = vertexai_types.EvaluationResult( + summary_metrics=[], + eval_case_results=[], + ) + + evaluation_result = evaluator.evaluate_invocations( + actual_invocations, expected_invocations + ) + + assert evaluation_result.overall_score is None + assert evaluation_result.overall_eval_status == EvalStatus.NOT_EVALUATED + mock_perform_eval.assert_called_once() + + def test_evaluate_invocations_coherence_metric_multiple_invocations( + self, mock_perform_eval + ): + """Test evaluate_invocations function for Coherence metric with multiple invocations.""" + num_invocations = 6 + actual_invocations = [] + expected_invocations = [] + mock_eval_results = [] + random.seed(61553) + scores = [random.random() for _ in range(num_invocations)] + + for i in range(num_invocations): + actual_invocations.append( + Invocation( + user_content=genai_types.Content( + parts=[genai_types.Part(text=f"Query {i+1}")] + ), + final_response=genai_types.Content( + parts=[genai_types.Part(text=f"Response {i+1}")] + ), + ) + ) + expected_invocations.append( + Invocation( + user_content=genai_types.Content( + parts=[genai_types.Part(text=f"Query {i+1}")] + ), + final_response=genai_types.Content( + parts=[genai_types.Part(text=f"Reference {i+1}")] + ), + ) + ) + mock_eval_results.append( + vertexai_types.EvaluationResult( + summary_metrics=[ + vertexai_types.AggregatedMetricResult(mean_score=scores[i]) + ], + eval_case_results=[], + ) + ) + + evaluator = ResponseEvaluator( + threshold=0.8, metric_name="response_evaluation_score" + ) + # Mock the return value of _perform_eval + mock_perform_eval.side_effect = mock_eval_results + + evaluation_result = evaluator.evaluate_invocations( + actual_invocations, expected_invocations + ) + + assert evaluation_result.overall_score == pytest.approx( + sum(scores) / num_invocations + ) + assert evaluation_result.overall_eval_status == EvalStatus.FAILED + assert mock_perform_eval.call_count == num_invocations