chore: Update ResponseEvaluator to use newer version of Eval SDK

Also,
- removed functionality that was marked deprecated from the ResponseEvaluator class.
- Added unit test cases

PiperOrigin-RevId: 778568884

pyproject.toml

@@ -85,7 +85,7 @@ a2a = [
 
 eval = [
   # go/keep-sorted start
-  "google-cloud-aiplatform[evaluation]>=1.87.0",
+  "google-cloud-aiplatform[evaluation]>=1.100.0",
   "pandas>=2.2.3",
   "tabulate>=0.9.0",
   "rouge-score>=0.1.2",

src/google/adk/evaluation/response_evaluator.py

@@ -14,18 +14,15 @@
 
 from __future__ import annotations
 
-from typing import Any
+import os
 from typing import Optional
 
 from google.genai import types as genai_types
 import pandas as pd
-from tabulate import tabulate
-from typing_extensions import deprecated
 from typing_extensions import override
-from vertexai.preview.evaluation import EvalTask
+from vertexai import Client as VertexAiClient
-from vertexai.preview.evaluation import MetricPromptTemplateExamples
+from vertexai import types as vertexai_types
 
-from .eval_case import IntermediateData
 from .eval_case import Invocation
 from .eval_metrics import EvalMetric
 from .evaluator import EvalStatus
@@ -57,7 +54,7 @@ class ResponseEvaluator(Evaluator):
       metric_name = eval_metric.metric_name
 
     if "response_evaluation_score" == metric_name:
-      self._metric_name = MetricPromptTemplateExamples.Pointwise.COHERENCE
+      self._metric_name = vertexai_types.PrebuiltMetric.COHERENCE
     elif "response_match_score" == metric_name:
       self._metric_name = "response_match_score"
     else:
@@ -87,17 +84,11 @@ class ResponseEvaluator(Evaluator):
       prompt = self._get_text(expected.user_content)
       reference = self._get_text(expected.final_response)
       response = self._get_text(actual.final_response)
-      actual_tool_use = self._get_tool_use_trajectory(actual.intermediate_data)
-      reference_trajectory = self._get_tool_use_trajectory(
-          expected.intermediate_data
-      )
 
       eval_case = {
           "prompt": prompt,
           "reference": reference,
           "response": response,
-          "actual_tool_user": actual_tool_use,
-          "reference_trajectory": reference_trajectory,
       }
 
       eval_case_result = ResponseEvaluator._perform_eval(
@@ -112,11 +103,15 @@ class ResponseEvaluator(Evaluator):
               eval_status=self._get_eval_status(score),
           )
       )
-      total_score += score
+
-      num_invocations += 1
+      if score:
+        total_score += score
+        num_invocations += 1
 
     if per_invocation_results:
-      overall_score = total_score / num_invocations
+      overall_score = (
+          total_score / num_invocations if num_invocations > 0 else None
+      )
       return EvaluationResult(
           overall_score=overall_score,
           overall_eval_status=self._get_eval_status(overall_score),
@@ -131,138 +126,19 @@ class ResponseEvaluator(Evaluator):
 
     return ""
 
-  def _get_tool_use_trajectory(
+  def _get_score(self, eval_result) -> Optional[float]:
-      self, intermediate_data: Optional[IntermediateData]
+    if eval_result and eval_result.summary_metrics:
-  ) -> list[dict[str, Any]]:
+      return eval_result.summary_metrics[0].mean_score
-    tool_use_trajectory = []
-    if not intermediate_data:
-      return tool_use_trajectory
 
-    for function_call in intermediate_data.tool_uses:
+    return None
-      tool_use_trajectory.append({
-          "tool_name": function_call.name,
-          "tool_input": function_call.args or {},
-      })
 
-    return tool_use_trajectory
+  def _get_eval_status(self, score: Optional[float]):
+    if score:
+      return (
+          EvalStatus.PASSED if score >= self._threshold else EvalStatus.FAILED
+      )
 
-  def _get_score(self, eval_result) -> float:
+    return EvalStatus.NOT_EVALUATED
-    return eval_result.summary_metrics[f"{self._metric_name}/mean"].item()
-
-  def _get_eval_status(self, score: float):
-    return EvalStatus.PASSED if score >= self._threshold else EvalStatus.FAILED
-
-  @staticmethod
-  @deprecated(
-      "This method has been deprecated and will be removed soon. Please use"
-      " evaluate_invocations instead."
-  )
-  def evaluate(
-      raw_eval_dataset: list[list[dict[str, Any]]],
-      evaluation_criteria: list[str],
-      *,
-      print_detailed_results: bool = False,
-  ):
-    r"""Returns the value of requested evaluation metrics.
-
-    Args:
-      raw_eval_dataset: The dataset that will be evaluated.
-      evaluation_criteria: The evaluation criteria to be used. This method
-        support two criteria, `response_evaluation_score` and
-        `response_match_score`.
-      print_detailed_results: Prints detailed results on the console. This is
-        usually helpful during debugging.
-
-    A note on evaluation_criteria:
-      `response_match_score`: This metric compares the agents final natural
-        language response with the expected final response, stored in the
-        "reference" field in test/eval files. We use Rouge metric to compare the
-        two responses.
-
-        Value Range: [0, 1]. A score closer to 0 means poor similarity between
-          response and reference. A score closer to 1 means strong similarity
-          between response and reference.
-
-      `response_evaluation_score`: Uses LLM to evalaute coherence of the
-        response, including tool use. This is pointwise metric.
-
-        Value range: [0, 5], where 0 means that the agent's response is not
-        coherent, while 5 means it is . High values are good.
-    A note on raw_eval_dataset:
-      The dataset should be a list session, where each session is represented
-      as a list of interaction that need evaluation. Each evaluation is
-      represented as a dictionary that is expected to have values for the
-      following keys:
-
-        1) query
-        2) response
-        3) acutal_tool_use
-        4) expected_tool_use
-        5) reference
-
-      Here is a sample eval_dataset value with one entry:
-      [
-        [
-          {
-            "query": "roll a die for me",
-            "response": "I rolled a 16 sided die and got 13.\n",
-            "expected_tool_use": [
-              {
-                "tool_name": "roll_die",
-                "tool_input": {
-                  "sides": 16
-                }
-              }
-            ],
-            "acutal_tool_use": [
-              {
-                "tool_name": "roll_die",
-                "tool_input": {
-                  "sides": 16
-                }
-              }
-            ],
-            "reference": "I rolled a 16 sided die and got 13.\n"
-          }
-        ]
-      ]
-    """
-    if not raw_eval_dataset:
-      raise ValueError("The evaluation dataset is empty.")
-
-    metrics = ResponseEvaluator._get_metrics(
-        raw_eval_dataset, evaluation_criteria
-    )
-    flattened_queries = [
-        item for sublist in raw_eval_dataset for item in sublist
-    ]
-    eval_dataset = pd.DataFrame(flattened_queries).rename(
-        columns={"query": "prompt", "expected_tool_use": "reference_trajectory"}
-    )
-
-    eval_result = ResponseEvaluator._perform_eval(
-        dataset=eval_dataset, metrics=metrics
-    )
-
-    if print_detailed_results:
-      ResponseEvaluator._print_results(eval_result)
-    return eval_result.summary_metrics
-
-  @staticmethod
-  def _get_metrics(raw_eval_dataset, criteria):
-    metrics = []
-    if (
-        "response_evaluation_score" in criteria
-        and "query" in raw_eval_dataset[0][0]
-        and "expected_tool_use" in raw_eval_dataset[0][0]
-    ):
-      metrics.append(MetricPromptTemplateExamples.Pointwise.COHERENCE)
-    if (
-        "response_match_score" in criteria
-        and "reference" in raw_eval_dataset[0][0]
-    ):
-      metrics.append("rouge_1")
-    return metrics
 
   @staticmethod
   def _perform_eval(dataset, metrics):
@@ -270,11 +146,11 @@ class ResponseEvaluator(Evaluator):
 
     Primarily helps with unit testing.
     """
-    eval_task = EvalTask(dataset=dataset, metrics=metrics)
+    project_id = str(os.environ.get("GOOGLE_CLOUD_PROJECT"))
+    location = os.environ.get("GOOGLE_CLOUD_REGION")
+    client = VertexAiClient(project=project_id, location=location)
 
-    return eval_task.evaluate()
+    return client.evals.evaluate(
-
+        dataset=vertexai_types.EvaluationDataset(eval_dataset_df=dataset),
-  @staticmethod
+        metrics=metrics,
-  def _print_results(eval_result):
+    )
-    print("Evaluation Summary Metrics:", eval_result.summary_metrics)
-    print(tabulate(eval_result.metrics_table, headers="keys", tablefmt="grid"))

tests/unittests/evaluation/test_response_evaluator.py

@@ -13,53 +13,15 @@
 # limitations under the License.
 
 """Tests for the Response Evaluator."""
-from unittest.mock import MagicMock
+import random
 from unittest.mock import patch
 
 from google.adk.evaluation.eval_case import Invocation
 from google.adk.evaluation.evaluator import EvalStatus
 from google.adk.evaluation.response_evaluator import ResponseEvaluator
 from google.genai import types as genai_types
-import pandas as pd
 import pytest
-from vertexai.preview.evaluation import MetricPromptTemplateExamples
+from vertexai import types as vertexai_types
-
-# Mock object for the result normally returned by _perform_eval
-MOCK_EVAL_RESULT = MagicMock()
-MOCK_EVAL_RESULT.summary_metrics = {"mock_metric": 0.75, "another_mock": 3.5}
-# Add a metrics_table for testing _print_results interaction
-MOCK_EVAL_RESULT.metrics_table = pd.DataFrame({
-    "prompt": ["mock_query1"],
-    "response": ["mock_resp1"],
-    "mock_metric": [0.75],
-})
-
-SAMPLE_TURN_1_ALL_KEYS = {
-    "query": "query1",
-    "response": "response1",
-    "actual_tool_use": [{"tool_name": "tool_a", "tool_input": {}}],
-    "expected_tool_use": [{"tool_name": "tool_a", "tool_input": {}}],
-    "reference": "reference1",
-}
-SAMPLE_TURN_2_MISSING_REF = {
-    "query": "query2",
-    "response": "response2",
-    "actual_tool_use": [],
-    "expected_tool_use": [],
-    # "reference": "reference2" # Missing
-}
-SAMPLE_TURN_3_MISSING_EXP_TOOLS = {
-    "query": "query3",
-    "response": "response3",
-    "actual_tool_use": [{"tool_name": "tool_b", "tool_input": {}}],
-    # "expected_tool_use": [], # Missing
-    "reference": "reference3",
-}
-SAMPLE_TURN_4_MINIMAL = {
-    "query": "query4",
-    "response": "response4",
-    # Minimal keys, others missing
-}
 
 
 @patch(
@@ -68,18 +30,6 @@ SAMPLE_TURN_4_MINIMAL = {
 class TestResponseEvaluator:
   """A class to help organize "patch" that are applicable to all tests."""
 
-  def test_evaluate_none_dataset_raises_value_error(self, mock_perform_eval):
-    """Test evaluate function raises ValueError for an empty list."""
-    with pytest.raises(ValueError, match="The evaluation dataset is empty."):
-      ResponseEvaluator.evaluate(None, ["response_evaluation_score"])
-    mock_perform_eval.assert_not_called()  # Ensure _perform_eval was not called
-
-  def test_evaluate_empty_dataset_raises_value_error(self, mock_perform_eval):
-    """Test evaluate function raises ValueError for an empty list."""
-    with pytest.raises(ValueError, match="The evaluation dataset is empty."):
-      ResponseEvaluator.evaluate([], ["response_evaluation_score"])
-    mock_perform_eval.assert_not_called()  # Ensure _perform_eval was not called
-
   def test_evaluate_invocations_rouge_metric(self, mock_perform_eval):
     """Test evaluate_invocations function for Rouge metric."""
     actual_invocations = [
@@ -107,190 +57,198 @@ class TestResponseEvaluator:
     evaluator = ResponseEvaluator(
         threshold=0.8, metric_name="response_match_score"
     )
+
     evaluation_result = evaluator.evaluate_invocations(
         actual_invocations, expected_invocations
     )
+
     assert evaluation_result.overall_score == pytest.approx(8 / 11)
     # ROUGE-1 F1 is approx. 0.73 < 0.8 threshold, so eval status is FAILED.
     assert evaluation_result.overall_eval_status == EvalStatus.FAILED
+    mock_perform_eval.assert_not_called()  # Ensure _perform_eval was not called
 
-  def test_evaluate_determines_metrics_correctly_for_perform_eval(
+  def test_evaluate_invocations_coherence_metric_passed(
       self, mock_perform_eval
   ):
-    """Test that the correct metrics list is passed to _perform_eval based on criteria/keys."""
+    """Test evaluate_invocations function for Coherence metric."""
-    mock_perform_eval.return_value = MOCK_EVAL_RESULT
+    actual_invocations = [
-
+        Invocation(
-    # Test case 1: Only Coherence
+            user_content=genai_types.Content(
-    raw_data_1 = [[SAMPLE_TURN_1_ALL_KEYS]]
+                parts=[genai_types.Part(text="This is a test query.")]
-    criteria_1 = ["response_evaluation_score"]
+            ),
-    ResponseEvaluator.evaluate(raw_data_1, criteria_1)
+            final_response=genai_types.Content(
-    _, kwargs = mock_perform_eval.call_args
+                parts=[
-    assert kwargs["metrics"] == [
+                    genai_types.Part(text="This is a test candidate response.")
-        MetricPromptTemplateExamples.Pointwise.COHERENCE
+                ]
+            ),
+        )
     ]
-    mock_perform_eval.reset_mock()  # Reset mock for next call
+    expected_invocations = [
-
+        Invocation(
-    # Test case 2: Only Rouge
+            user_content=genai_types.Content(
-    raw_data_2 = [[SAMPLE_TURN_1_ALL_KEYS]]
+                parts=[genai_types.Part(text="This is a test query.")]
-    criteria_2 = ["response_match_score"]
+            ),
-    ResponseEvaluator.evaluate(raw_data_2, criteria_2)
+            final_response=genai_types.Content(
-    _, kwargs = mock_perform_eval.call_args
+                parts=[genai_types.Part(text="This is a test reference.")]
-    assert kwargs["metrics"] == ["rouge_1"]
+            ),
-    mock_perform_eval.reset_mock()
+        )
-
+    ]
-    # Test case 3: No metrics if keys missing in first turn
+    evaluator = ResponseEvaluator(
-    raw_data_3 = [[SAMPLE_TURN_4_MINIMAL, SAMPLE_TURN_1_ALL_KEYS]]
+        threshold=0.8, metric_name="response_evaluation_score"
-    criteria_3 = ["response_evaluation_score", "response_match_score"]
+    )
-    ResponseEvaluator.evaluate(raw_data_3, criteria_3)
+    # Mock the return value of _perform_eval
-    _, kwargs = mock_perform_eval.call_args
+    mock_perform_eval.return_value = vertexai_types.EvaluationResult(
-    assert kwargs["metrics"] == []
+        summary_metrics=[vertexai_types.AggregatedMetricResult(mean_score=0.9)],
-    mock_perform_eval.reset_mock()
+        eval_case_results=[],
-
-    # Test case 4: No metrics if criteria empty
-    raw_data_4 = [[SAMPLE_TURN_1_ALL_KEYS]]
-    criteria_4 = []
-    ResponseEvaluator.evaluate(raw_data_4, criteria_4)
-    _, kwargs = mock_perform_eval.call_args
-    assert kwargs["metrics"] == []
-    mock_perform_eval.reset_mock()
-
-  def test_evaluate_calls_perform_eval_correctly_all_metrics(
-      self, mock_perform_eval
-  ):
-    """Test evaluate function calls _perform_eval with expected args when all criteria/keys are present."""
-    # Arrange
-    mock_perform_eval.return_value = (
-        MOCK_EVAL_RESULT  # Configure the mock return value
     )
 
-    raw_data = [[SAMPLE_TURN_1_ALL_KEYS]]
+    evaluation_result = evaluator.evaluate_invocations(
-    criteria = ["response_evaluation_score", "response_match_score"]
+        actual_invocations, expected_invocations
-
-    # Act
-    summary = ResponseEvaluator.evaluate(raw_data, criteria)
-
-    # Assert
-    # 1. Check metrics determined by _get_metrics (passed to _perform_eval)
-    expected_metrics_list = [
-        MetricPromptTemplateExamples.Pointwise.COHERENCE,
-        "rouge_1",
-    ]
-    # 2. Check DataFrame prepared (passed to _perform_eval)
-    expected_df_data = [{
-        "prompt": "query1",
-        "response": "response1",
-        "actual_tool_use": [{"tool_name": "tool_a", "tool_input": {}}],
-        "reference_trajectory": [{"tool_name": "tool_a", "tool_input": {}}],
-        "reference": "reference1",
-    }]
-    expected_df = pd.DataFrame(expected_df_data)
-
-    # Assert _perform_eval was called once
-    mock_perform_eval.assert_called_once()
-    # Get the arguments passed to the mocked _perform_eval
-    _, kwargs = mock_perform_eval.call_args
-    # Check the 'dataset' keyword argument
-    pd.testing.assert_frame_equal(kwargs["dataset"], expected_df)
-    # Check the 'metrics' keyword argument
-    assert kwargs["metrics"] == expected_metrics_list
-
-    # 3. Check the correct summary metrics are returned
-    # (from mock_perform_eval's return value)
-    assert summary == MOCK_EVAL_RESULT.summary_metrics
-
-  def test_evaluate_prepares_dataframe_correctly_for_perform_eval(
-      self, mock_perform_eval
-  ):
-    """Test that the DataFrame is correctly flattened and renamed before passing to _perform_eval."""
-    mock_perform_eval.return_value = MOCK_EVAL_RESULT
-
-    raw_data = [
-        [SAMPLE_TURN_1_ALL_KEYS],  # Conversation 1
-        [
-            SAMPLE_TURN_2_MISSING_REF,
-            SAMPLE_TURN_3_MISSING_EXP_TOOLS,
-        ],  # Conversation 2
-    ]
-    criteria = [
-        "response_match_score"
-    ]  # Doesn't affect the DataFrame structure
-
-    ResponseEvaluator.evaluate(raw_data, criteria)
-
-    # Expected flattened and renamed data
-    expected_df_data = [
-        # Turn 1 (from SAMPLE_TURN_1_ALL_KEYS)
-        {
-            "prompt": "query1",
-            "response": "response1",
-            "actual_tool_use": [{"tool_name": "tool_a", "tool_input": {}}],
-            "reference_trajectory": [{"tool_name": "tool_a", "tool_input": {}}],
-            "reference": "reference1",
-        },
-        # Turn 2 (from SAMPLE_TURN_2_MISSING_REF)
-        {
-            "prompt": "query2",
-            "response": "response2",
-            "actual_tool_use": [],
-            "reference_trajectory": [],
-            # "reference": None # Missing key results in NaN in DataFrame
-            # usually
-        },
-        # Turn 3 (from SAMPLE_TURN_3_MISSING_EXP_TOOLS)
-        {
-            "prompt": "query3",
-            "response": "response3",
-            "actual_tool_use": [{"tool_name": "tool_b", "tool_input": {}}],
-            # "reference_trajectory": None, # Missing key results in NaN
-            "reference": "reference3",
-        },
-    ]
-    # Need to be careful with missing keys -> NaN when creating DataFrame
-    # Pandas handles this automatically when creating from list of dicts
-    expected_df = pd.DataFrame(expected_df_data)
-
-    mock_perform_eval.assert_called_once()
-    _, kwargs = mock_perform_eval.call_args
-    # Compare the DataFrame passed to the mock
-    pd.testing.assert_frame_equal(kwargs["dataset"], expected_df)
-
-  @patch(
-      "google.adk.evaluation.response_evaluator.ResponseEvaluator._print_results"
-  )  # Mock the private print method
-  def test_evaluate_print_detailed_results(
-      self, mock_print_results, mock_perform_eval
-  ):
-    """Test _print_results function is called when print_detailed_results=True."""
-    mock_perform_eval.return_value = (
-        MOCK_EVAL_RESULT  # Ensure _perform_eval returns our mock result
     )
 
-    raw_data = [[SAMPLE_TURN_1_ALL_KEYS]]
+    assert evaluation_result.overall_score == 0.9
-    criteria = ["response_match_score"]
+    assert evaluation_result.overall_eval_status == EvalStatus.PASSED
-
-    ResponseEvaluator.evaluate(raw_data, criteria, print_detailed_results=True)
-
-    # Assert _perform_eval was called
     mock_perform_eval.assert_called_once()
-    # Assert _print_results was called once with the result object
-    # from _perform_eval
-    mock_print_results.assert_called_once_with(MOCK_EVAL_RESULT)
 
-  @patch(
+  def test_evaluate_invocations_coherence_metric_failed(
-      "google.adk.evaluation.response_evaluator.ResponseEvaluator._print_results"
+      self, mock_perform_eval
-  )
-  def test_evaluate_no_print_detailed_results(
-      self, mock_print_results, mock_perform_eval
   ):
-    """Test _print_results function is NOT called when print_detailed_results=False (default)."""
+    """Test evaluate_invocations function for Coherence metric."""
-    mock_perform_eval.return_value = MOCK_EVAL_RESULT
+    actual_invocations = [
+        Invocation(
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test query.")]
+            ),
+            final_response=genai_types.Content(
+                parts=[
+                    genai_types.Part(text="This is a test candidate response.")
+                ]
+            ),
+        )
+    ]
+    expected_invocations = [
+        Invocation(
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test query.")]
+            ),
+            final_response=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test reference.")]
+            ),
+        )
+    ]
+    evaluator = ResponseEvaluator(
+        threshold=0.8, metric_name="response_evaluation_score"
+    )
+    # Mock the return value of _perform_eval
+    mock_perform_eval.return_value = vertexai_types.EvaluationResult(
+        summary_metrics=[vertexai_types.AggregatedMetricResult(mean_score=0.7)],
+        eval_case_results=[],
+    )
 
-    raw_data = [[SAMPLE_TURN_1_ALL_KEYS]]
+    evaluation_result = evaluator.evaluate_invocations(
-    criteria = ["response_match_score"]
+        actual_invocations, expected_invocations
+    )
 
-    ResponseEvaluator.evaluate(raw_data, criteria, print_detailed_results=False)
+    assert evaluation_result.overall_score == 0.7
-
+    assert evaluation_result.overall_eval_status == EvalStatus.FAILED
-    # Assert _perform_eval was called
     mock_perform_eval.assert_called_once()
-    # Assert _print_results was NOT called
+
-    mock_print_results.assert_not_called()
+  def test_evaluate_invocations_coherence_metric_no_score(
+      self, mock_perform_eval
+  ):
+    """Test evaluate_invocations function for Coherence metric."""
+    actual_invocations = [
+        Invocation(
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test query.")]
+            ),
+            final_response=genai_types.Content(
+                parts=[
+                    genai_types.Part(text="This is a test candidate response.")
+                ]
+            ),
+        )
+    ]
+    expected_invocations = [
+        Invocation(
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test query.")]
+            ),
+            final_response=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test reference.")]
+            ),
+        )
+    ]
+    evaluator = ResponseEvaluator(
+        threshold=0.8, metric_name="response_evaluation_score"
+    )
+    # Mock the return value of _perform_eval
+    mock_perform_eval.return_value = vertexai_types.EvaluationResult(
+        summary_metrics=[],
+        eval_case_results=[],
+    )
+
+    evaluation_result = evaluator.evaluate_invocations(
+        actual_invocations, expected_invocations
+    )
+
+    assert evaluation_result.overall_score is None
+    assert evaluation_result.overall_eval_status == EvalStatus.NOT_EVALUATED
+    mock_perform_eval.assert_called_once()
+
+  def test_evaluate_invocations_coherence_metric_multiple_invocations(
+      self, mock_perform_eval
+  ):
+    """Test evaluate_invocations function for Coherence metric with multiple invocations."""
+    num_invocations = 6
+    actual_invocations = []
+    expected_invocations = []
+    mock_eval_results = []
+    random.seed(61553)
+    scores = [random.random() for _ in range(num_invocations)]
+
+    for i in range(num_invocations):
+      actual_invocations.append(
+          Invocation(
+              user_content=genai_types.Content(
+                  parts=[genai_types.Part(text=f"Query {i+1}")]
+              ),
+              final_response=genai_types.Content(
+                  parts=[genai_types.Part(text=f"Response {i+1}")]
+              ),
+          )
+      )
+      expected_invocations.append(
+          Invocation(
+              user_content=genai_types.Content(
+                  parts=[genai_types.Part(text=f"Query {i+1}")]
+              ),
+              final_response=genai_types.Content(
+                  parts=[genai_types.Part(text=f"Reference {i+1}")]
+              ),
+          )
+      )
+      mock_eval_results.append(
+          vertexai_types.EvaluationResult(
+              summary_metrics=[
+                  vertexai_types.AggregatedMetricResult(mean_score=scores[i])
+              ],
+              eval_case_results=[],
+          )
+      )
+
+    evaluator = ResponseEvaluator(
+        threshold=0.8, metric_name="response_evaluation_score"
+    )
+    # Mock the return value of _perform_eval
+    mock_perform_eval.side_effect = mock_eval_results
+
+    evaluation_result = evaluator.evaluate_invocations(
+        actual_invocations, expected_invocations
+    )
+
+    assert evaluation_result.overall_score == pytest.approx(
+        sum(scores) / num_invocations
+    )
+    assert evaluation_result.overall_eval_status == EvalStatus.FAILED
+    assert mock_perform_eval.call_count == num_invocations