mirror of
https://github.com/encounter/adk-python.git
synced 2026-07-09 18:19:28 -07:00
chore: Update ResponseEvaluator to use newer version of Eval SDK
Also, - removed functionality that was marked deprecated from the ResponseEvaluator class. - Added unit test cases PiperOrigin-RevId: 778568884
This commit is contained in:
Ankur Sharma
committed by
Copybara-Service
Copybara-Service
parent
08869ccc07
commit
62c4a85917
+1
-1
@@ -85,7 +85,7 @@ a2a = [
|
||||
|
||||
eval = [
|
||||
# go/keep-sorted start
|
||||
"google-cloud-aiplatform[evaluation]>=1.87.0",
|
||||
"google-cloud-aiplatform[evaluation]>=1.100.0",
|
||||
"pandas>=2.2.3",
|
||||
"tabulate>=0.9.0",
|
||||
"rouge-score>=0.1.2",
|
||||
|
||||
@@ -14,18 +14,15 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
from google.genai import types as genai_types
|
||||
import pandas as pd
|
||||
from tabulate import tabulate
|
||||
from typing_extensions import deprecated
|
||||
from typing_extensions import override
|
||||
from vertexai.preview.evaluation import EvalTask
|
||||
from vertexai.preview.evaluation import MetricPromptTemplateExamples
|
||||
from vertexai import Client as VertexAiClient
|
||||
from vertexai import types as vertexai_types
|
||||
|
||||
from .eval_case import IntermediateData
|
||||
from .eval_case import Invocation
|
||||
from .eval_metrics import EvalMetric
|
||||
from .evaluator import EvalStatus
|
||||
@@ -57,7 +54,7 @@ class ResponseEvaluator(Evaluator):
|
||||
metric_name = eval_metric.metric_name
|
||||
|
||||
if "response_evaluation_score" == metric_name:
|
||||
self._metric_name = MetricPromptTemplateExamples.Pointwise.COHERENCE
|
||||
self._metric_name = vertexai_types.PrebuiltMetric.COHERENCE
|
||||
elif "response_match_score" == metric_name:
|
||||
self._metric_name = "response_match_score"
|
||||
else:
|
||||
@@ -87,17 +84,11 @@ class ResponseEvaluator(Evaluator):
|
||||
prompt = self._get_text(expected.user_content)
|
||||
reference = self._get_text(expected.final_response)
|
||||
response = self._get_text(actual.final_response)
|
||||
actual_tool_use = self._get_tool_use_trajectory(actual.intermediate_data)
|
||||
reference_trajectory = self._get_tool_use_trajectory(
|
||||
expected.intermediate_data
|
||||
)
|
||||
|
||||
eval_case = {
|
||||
"prompt": prompt,
|
||||
"reference": reference,
|
||||
"response": response,
|
||||
"actual_tool_user": actual_tool_use,
|
||||
"reference_trajectory": reference_trajectory,
|
||||
}
|
||||
|
||||
eval_case_result = ResponseEvaluator._perform_eval(
|
||||
@@ -112,11 +103,15 @@ class ResponseEvaluator(Evaluator):
|
||||
eval_status=self._get_eval_status(score),
|
||||
)
|
||||
)
|
||||
total_score += score
|
||||
num_invocations += 1
|
||||
|
||||
if score:
|
||||
total_score += score
|
||||
num_invocations += 1
|
||||
|
||||
if per_invocation_results:
|
||||
overall_score = total_score / num_invocations
|
||||
overall_score = (
|
||||
total_score / num_invocations if num_invocations > 0 else None
|
||||
)
|
||||
return EvaluationResult(
|
||||
overall_score=overall_score,
|
||||
overall_eval_status=self._get_eval_status(overall_score),
|
||||
@@ -131,138 +126,19 @@ class ResponseEvaluator(Evaluator):
|
||||
|
||||
return ""
|
||||
|
||||
def _get_tool_use_trajectory(
|
||||
self, intermediate_data: Optional[IntermediateData]
|
||||
) -> list[dict[str, Any]]:
|
||||
tool_use_trajectory = []
|
||||
if not intermediate_data:
|
||||
return tool_use_trajectory
|
||||
def _get_score(self, eval_result) -> Optional[float]:
|
||||
if eval_result and eval_result.summary_metrics:
|
||||
return eval_result.summary_metrics[0].mean_score
|
||||
|
||||
for function_call in intermediate_data.tool_uses:
|
||||
tool_use_trajectory.append({
|
||||
"tool_name": function_call.name,
|
||||
"tool_input": function_call.args or {},
|
||||
})
|
||||
return None
|
||||
|
||||
return tool_use_trajectory
|
||||
def _get_eval_status(self, score: Optional[float]):
|
||||
if score:
|
||||
return (
|
||||
EvalStatus.PASSED if score >= self._threshold else EvalStatus.FAILED
|
||||
)
|
||||
|
||||
def _get_score(self, eval_result) -> float:
|
||||
return eval_result.summary_metrics[f"{self._metric_name}/mean"].item()
|
||||
|
||||
def _get_eval_status(self, score: float):
|
||||
return EvalStatus.PASSED if score >= self._threshold else EvalStatus.FAILED
|
||||
|
||||
@staticmethod
|
||||
@deprecated(
|
||||
"This method has been deprecated and will be removed soon. Please use"
|
||||
" evaluate_invocations instead."
|
||||
)
|
||||
def evaluate(
|
||||
raw_eval_dataset: list[list[dict[str, Any]]],
|
||||
evaluation_criteria: list[str],
|
||||
*,
|
||||
print_detailed_results: bool = False,
|
||||
):
|
||||
r"""Returns the value of requested evaluation metrics.
|
||||
|
||||
Args:
|
||||
raw_eval_dataset: The dataset that will be evaluated.
|
||||
evaluation_criteria: The evaluation criteria to be used. This method
|
||||
support two criteria, `response_evaluation_score` and
|
||||
`response_match_score`.
|
||||
print_detailed_results: Prints detailed results on the console. This is
|
||||
usually helpful during debugging.
|
||||
|
||||
A note on evaluation_criteria:
|
||||
`response_match_score`: This metric compares the agents final natural
|
||||
language response with the expected final response, stored in the
|
||||
"reference" field in test/eval files. We use Rouge metric to compare the
|
||||
two responses.
|
||||
|
||||
Value Range: [0, 1]. A score closer to 0 means poor similarity between
|
||||
response and reference. A score closer to 1 means strong similarity
|
||||
between response and reference.
|
||||
|
||||
`response_evaluation_score`: Uses LLM to evalaute coherence of the
|
||||
response, including tool use. This is pointwise metric.
|
||||
|
||||
Value range: [0, 5], where 0 means that the agent's response is not
|
||||
coherent, while 5 means it is . High values are good.
|
||||
A note on raw_eval_dataset:
|
||||
The dataset should be a list session, where each session is represented
|
||||
as a list of interaction that need evaluation. Each evaluation is
|
||||
represented as a dictionary that is expected to have values for the
|
||||
following keys:
|
||||
|
||||
1) query
|
||||
2) response
|
||||
3) acutal_tool_use
|
||||
4) expected_tool_use
|
||||
5) reference
|
||||
|
||||
Here is a sample eval_dataset value with one entry:
|
||||
[
|
||||
[
|
||||
{
|
||||
"query": "roll a die for me",
|
||||
"response": "I rolled a 16 sided die and got 13.\n",
|
||||
"expected_tool_use": [
|
||||
{
|
||||
"tool_name": "roll_die",
|
||||
"tool_input": {
|
||||
"sides": 16
|
||||
}
|
||||
}
|
||||
],
|
||||
"acutal_tool_use": [
|
||||
{
|
||||
"tool_name": "roll_die",
|
||||
"tool_input": {
|
||||
"sides": 16
|
||||
}
|
||||
}
|
||||
],
|
||||
"reference": "I rolled a 16 sided die and got 13.\n"
|
||||
}
|
||||
]
|
||||
]
|
||||
"""
|
||||
if not raw_eval_dataset:
|
||||
raise ValueError("The evaluation dataset is empty.")
|
||||
|
||||
metrics = ResponseEvaluator._get_metrics(
|
||||
raw_eval_dataset, evaluation_criteria
|
||||
)
|
||||
flattened_queries = [
|
||||
item for sublist in raw_eval_dataset for item in sublist
|
||||
]
|
||||
eval_dataset = pd.DataFrame(flattened_queries).rename(
|
||||
columns={"query": "prompt", "expected_tool_use": "reference_trajectory"}
|
||||
)
|
||||
|
||||
eval_result = ResponseEvaluator._perform_eval(
|
||||
dataset=eval_dataset, metrics=metrics
|
||||
)
|
||||
|
||||
if print_detailed_results:
|
||||
ResponseEvaluator._print_results(eval_result)
|
||||
return eval_result.summary_metrics
|
||||
|
||||
@staticmethod
|
||||
def _get_metrics(raw_eval_dataset, criteria):
|
||||
metrics = []
|
||||
if (
|
||||
"response_evaluation_score" in criteria
|
||||
and "query" in raw_eval_dataset[0][0]
|
||||
and "expected_tool_use" in raw_eval_dataset[0][0]
|
||||
):
|
||||
metrics.append(MetricPromptTemplateExamples.Pointwise.COHERENCE)
|
||||
if (
|
||||
"response_match_score" in criteria
|
||||
and "reference" in raw_eval_dataset[0][0]
|
||||
):
|
||||
metrics.append("rouge_1")
|
||||
return metrics
|
||||
return EvalStatus.NOT_EVALUATED
|
||||
|
||||
@staticmethod
|
||||
def _perform_eval(dataset, metrics):
|
||||
@@ -270,11 +146,11 @@ class ResponseEvaluator(Evaluator):
|
||||
|
||||
Primarily helps with unit testing.
|
||||
"""
|
||||
eval_task = EvalTask(dataset=dataset, metrics=metrics)
|
||||
project_id = str(os.environ.get("GOOGLE_CLOUD_PROJECT"))
|
||||
location = os.environ.get("GOOGLE_CLOUD_REGION")
|
||||
client = VertexAiClient(project=project_id, location=location)
|
||||
|
||||
return eval_task.evaluate()
|
||||
|
||||
@staticmethod
|
||||
def _print_results(eval_result):
|
||||
print("Evaluation Summary Metrics:", eval_result.summary_metrics)
|
||||
print(tabulate(eval_result.metrics_table, headers="keys", tablefmt="grid"))
|
||||
return client.evals.evaluate(
|
||||
dataset=vertexai_types.EvaluationDataset(eval_dataset_df=dataset),
|
||||
metrics=metrics,
|
||||
)
|
||||
|
||||
@@ -13,53 +13,15 @@
|
||||
# limitations under the License.
|
||||
|
||||
"""Tests for the Response Evaluator."""
|
||||
from unittest.mock import MagicMock
|
||||
import random
|
||||
from unittest.mock import patch
|
||||
|
||||
from google.adk.evaluation.eval_case import Invocation
|
||||
from google.adk.evaluation.evaluator import EvalStatus
|
||||
from google.adk.evaluation.response_evaluator import ResponseEvaluator
|
||||
from google.genai import types as genai_types
|
||||
import pandas as pd
|
||||
import pytest
|
||||
from vertexai.preview.evaluation import MetricPromptTemplateExamples
|
||||
|
||||
# Mock object for the result normally returned by _perform_eval
|
||||
MOCK_EVAL_RESULT = MagicMock()
|
||||
MOCK_EVAL_RESULT.summary_metrics = {"mock_metric": 0.75, "another_mock": 3.5}
|
||||
# Add a metrics_table for testing _print_results interaction
|
||||
MOCK_EVAL_RESULT.metrics_table = pd.DataFrame({
|
||||
"prompt": ["mock_query1"],
|
||||
"response": ["mock_resp1"],
|
||||
"mock_metric": [0.75],
|
||||
})
|
||||
|
||||
SAMPLE_TURN_1_ALL_KEYS = {
|
||||
"query": "query1",
|
||||
"response": "response1",
|
||||
"actual_tool_use": [{"tool_name": "tool_a", "tool_input": {}}],
|
||||
"expected_tool_use": [{"tool_name": "tool_a", "tool_input": {}}],
|
||||
"reference": "reference1",
|
||||
}
|
||||
SAMPLE_TURN_2_MISSING_REF = {
|
||||
"query": "query2",
|
||||
"response": "response2",
|
||||
"actual_tool_use": [],
|
||||
"expected_tool_use": [],
|
||||
# "reference": "reference2" # Missing
|
||||
}
|
||||
SAMPLE_TURN_3_MISSING_EXP_TOOLS = {
|
||||
"query": "query3",
|
||||
"response": "response3",
|
||||
"actual_tool_use": [{"tool_name": "tool_b", "tool_input": {}}],
|
||||
# "expected_tool_use": [], # Missing
|
||||
"reference": "reference3",
|
||||
}
|
||||
SAMPLE_TURN_4_MINIMAL = {
|
||||
"query": "query4",
|
||||
"response": "response4",
|
||||
# Minimal keys, others missing
|
||||
}
|
||||
from vertexai import types as vertexai_types
|
||||
|
||||
|
||||
@patch(
|
||||
@@ -68,18 +30,6 @@ SAMPLE_TURN_4_MINIMAL = {
|
||||
class TestResponseEvaluator:
|
||||
"""A class to help organize "patch" that are applicable to all tests."""
|
||||
|
||||
def test_evaluate_none_dataset_raises_value_error(self, mock_perform_eval):
|
||||
"""Test evaluate function raises ValueError for an empty list."""
|
||||
with pytest.raises(ValueError, match="The evaluation dataset is empty."):
|
||||
ResponseEvaluator.evaluate(None, ["response_evaluation_score"])
|
||||
mock_perform_eval.assert_not_called() # Ensure _perform_eval was not called
|
||||
|
||||
def test_evaluate_empty_dataset_raises_value_error(self, mock_perform_eval):
|
||||
"""Test evaluate function raises ValueError for an empty list."""
|
||||
with pytest.raises(ValueError, match="The evaluation dataset is empty."):
|
||||
ResponseEvaluator.evaluate([], ["response_evaluation_score"])
|
||||
mock_perform_eval.assert_not_called() # Ensure _perform_eval was not called
|
||||
|
||||
def test_evaluate_invocations_rouge_metric(self, mock_perform_eval):
|
||||
"""Test evaluate_invocations function for Rouge metric."""
|
||||
actual_invocations = [
|
||||
@@ -107,190 +57,198 @@ class TestResponseEvaluator:
|
||||
evaluator = ResponseEvaluator(
|
||||
threshold=0.8, metric_name="response_match_score"
|
||||
)
|
||||
|
||||
evaluation_result = evaluator.evaluate_invocations(
|
||||
actual_invocations, expected_invocations
|
||||
)
|
||||
|
||||
assert evaluation_result.overall_score == pytest.approx(8 / 11)
|
||||
# ROUGE-1 F1 is approx. 0.73 < 0.8 threshold, so eval status is FAILED.
|
||||
assert evaluation_result.overall_eval_status == EvalStatus.FAILED
|
||||
mock_perform_eval.assert_not_called() # Ensure _perform_eval was not called
|
||||
|
||||
def test_evaluate_determines_metrics_correctly_for_perform_eval(
|
||||
def test_evaluate_invocations_coherence_metric_passed(
|
||||
self, mock_perform_eval
|
||||
):
|
||||
"""Test that the correct metrics list is passed to _perform_eval based on criteria/keys."""
|
||||
mock_perform_eval.return_value = MOCK_EVAL_RESULT
|
||||
|
||||
# Test case 1: Only Coherence
|
||||
raw_data_1 = [[SAMPLE_TURN_1_ALL_KEYS]]
|
||||
criteria_1 = ["response_evaluation_score"]
|
||||
ResponseEvaluator.evaluate(raw_data_1, criteria_1)
|
||||
_, kwargs = mock_perform_eval.call_args
|
||||
assert kwargs["metrics"] == [
|
||||
MetricPromptTemplateExamples.Pointwise.COHERENCE
|
||||
"""Test evaluate_invocations function for Coherence metric."""
|
||||
actual_invocations = [
|
||||
Invocation(
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="This is a test query.")]
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[
|
||||
genai_types.Part(text="This is a test candidate response.")
|
||||
]
|
||||
),
|
||||
)
|
||||
]
|
||||
mock_perform_eval.reset_mock() # Reset mock for next call
|
||||
|
||||
# Test case 2: Only Rouge
|
||||
raw_data_2 = [[SAMPLE_TURN_1_ALL_KEYS]]
|
||||
criteria_2 = ["response_match_score"]
|
||||
ResponseEvaluator.evaluate(raw_data_2, criteria_2)
|
||||
_, kwargs = mock_perform_eval.call_args
|
||||
assert kwargs["metrics"] == ["rouge_1"]
|
||||
mock_perform_eval.reset_mock()
|
||||
|
||||
# Test case 3: No metrics if keys missing in first turn
|
||||
raw_data_3 = [[SAMPLE_TURN_4_MINIMAL, SAMPLE_TURN_1_ALL_KEYS]]
|
||||
criteria_3 = ["response_evaluation_score", "response_match_score"]
|
||||
ResponseEvaluator.evaluate(raw_data_3, criteria_3)
|
||||
_, kwargs = mock_perform_eval.call_args
|
||||
assert kwargs["metrics"] == []
|
||||
mock_perform_eval.reset_mock()
|
||||
|
||||
# Test case 4: No metrics if criteria empty
|
||||
raw_data_4 = [[SAMPLE_TURN_1_ALL_KEYS]]
|
||||
criteria_4 = []
|
||||
ResponseEvaluator.evaluate(raw_data_4, criteria_4)
|
||||
_, kwargs = mock_perform_eval.call_args
|
||||
assert kwargs["metrics"] == []
|
||||
mock_perform_eval.reset_mock()
|
||||
|
||||
def test_evaluate_calls_perform_eval_correctly_all_metrics(
|
||||
self, mock_perform_eval
|
||||
):
|
||||
"""Test evaluate function calls _perform_eval with expected args when all criteria/keys are present."""
|
||||
# Arrange
|
||||
mock_perform_eval.return_value = (
|
||||
MOCK_EVAL_RESULT # Configure the mock return value
|
||||
expected_invocations = [
|
||||
Invocation(
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="This is a test query.")]
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[genai_types.Part(text="This is a test reference.")]
|
||||
),
|
||||
)
|
||||
]
|
||||
evaluator = ResponseEvaluator(
|
||||
threshold=0.8, metric_name="response_evaluation_score"
|
||||
)
|
||||
# Mock the return value of _perform_eval
|
||||
mock_perform_eval.return_value = vertexai_types.EvaluationResult(
|
||||
summary_metrics=[vertexai_types.AggregatedMetricResult(mean_score=0.9)],
|
||||
eval_case_results=[],
|
||||
)
|
||||
|
||||
raw_data = [[SAMPLE_TURN_1_ALL_KEYS]]
|
||||
criteria = ["response_evaluation_score", "response_match_score"]
|
||||
|
||||
# Act
|
||||
summary = ResponseEvaluator.evaluate(raw_data, criteria)
|
||||
|
||||
# Assert
|
||||
# 1. Check metrics determined by _get_metrics (passed to _perform_eval)
|
||||
expected_metrics_list = [
|
||||
MetricPromptTemplateExamples.Pointwise.COHERENCE,
|
||||
"rouge_1",
|
||||
]
|
||||
# 2. Check DataFrame prepared (passed to _perform_eval)
|
||||
expected_df_data = [{
|
||||
"prompt": "query1",
|
||||
"response": "response1",
|
||||
"actual_tool_use": [{"tool_name": "tool_a", "tool_input": {}}],
|
||||
"reference_trajectory": [{"tool_name": "tool_a", "tool_input": {}}],
|
||||
"reference": "reference1",
|
||||
}]
|
||||
expected_df = pd.DataFrame(expected_df_data)
|
||||
|
||||
# Assert _perform_eval was called once
|
||||
mock_perform_eval.assert_called_once()
|
||||
# Get the arguments passed to the mocked _perform_eval
|
||||
_, kwargs = mock_perform_eval.call_args
|
||||
# Check the 'dataset' keyword argument
|
||||
pd.testing.assert_frame_equal(kwargs["dataset"], expected_df)
|
||||
# Check the 'metrics' keyword argument
|
||||
assert kwargs["metrics"] == expected_metrics_list
|
||||
|
||||
# 3. Check the correct summary metrics are returned
|
||||
# (from mock_perform_eval's return value)
|
||||
assert summary == MOCK_EVAL_RESULT.summary_metrics
|
||||
|
||||
def test_evaluate_prepares_dataframe_correctly_for_perform_eval(
|
||||
self, mock_perform_eval
|
||||
):
|
||||
"""Test that the DataFrame is correctly flattened and renamed before passing to _perform_eval."""
|
||||
mock_perform_eval.return_value = MOCK_EVAL_RESULT
|
||||
|
||||
raw_data = [
|
||||
[SAMPLE_TURN_1_ALL_KEYS], # Conversation 1
|
||||
[
|
||||
SAMPLE_TURN_2_MISSING_REF,
|
||||
SAMPLE_TURN_3_MISSING_EXP_TOOLS,
|
||||
], # Conversation 2
|
||||
]
|
||||
criteria = [
|
||||
"response_match_score"
|
||||
] # Doesn't affect the DataFrame structure
|
||||
|
||||
ResponseEvaluator.evaluate(raw_data, criteria)
|
||||
|
||||
# Expected flattened and renamed data
|
||||
expected_df_data = [
|
||||
# Turn 1 (from SAMPLE_TURN_1_ALL_KEYS)
|
||||
{
|
||||
"prompt": "query1",
|
||||
"response": "response1",
|
||||
"actual_tool_use": [{"tool_name": "tool_a", "tool_input": {}}],
|
||||
"reference_trajectory": [{"tool_name": "tool_a", "tool_input": {}}],
|
||||
"reference": "reference1",
|
||||
},
|
||||
# Turn 2 (from SAMPLE_TURN_2_MISSING_REF)
|
||||
{
|
||||
"prompt": "query2",
|
||||
"response": "response2",
|
||||
"actual_tool_use": [],
|
||||
"reference_trajectory": [],
|
||||
# "reference": None # Missing key results in NaN in DataFrame
|
||||
# usually
|
||||
},
|
||||
# Turn 3 (from SAMPLE_TURN_3_MISSING_EXP_TOOLS)
|
||||
{
|
||||
"prompt": "query3",
|
||||
"response": "response3",
|
||||
"actual_tool_use": [{"tool_name": "tool_b", "tool_input": {}}],
|
||||
# "reference_trajectory": None, # Missing key results in NaN
|
||||
"reference": "reference3",
|
||||
},
|
||||
]
|
||||
# Need to be careful with missing keys -> NaN when creating DataFrame
|
||||
# Pandas handles this automatically when creating from list of dicts
|
||||
expected_df = pd.DataFrame(expected_df_data)
|
||||
|
||||
mock_perform_eval.assert_called_once()
|
||||
_, kwargs = mock_perform_eval.call_args
|
||||
# Compare the DataFrame passed to the mock
|
||||
pd.testing.assert_frame_equal(kwargs["dataset"], expected_df)
|
||||
|
||||
@patch(
|
||||
"google.adk.evaluation.response_evaluator.ResponseEvaluator._print_results"
|
||||
) # Mock the private print method
|
||||
def test_evaluate_print_detailed_results(
|
||||
self, mock_print_results, mock_perform_eval
|
||||
):
|
||||
"""Test _print_results function is called when print_detailed_results=True."""
|
||||
mock_perform_eval.return_value = (
|
||||
MOCK_EVAL_RESULT # Ensure _perform_eval returns our mock result
|
||||
evaluation_result = evaluator.evaluate_invocations(
|
||||
actual_invocations, expected_invocations
|
||||
)
|
||||
|
||||
raw_data = [[SAMPLE_TURN_1_ALL_KEYS]]
|
||||
criteria = ["response_match_score"]
|
||||
|
||||
ResponseEvaluator.evaluate(raw_data, criteria, print_detailed_results=True)
|
||||
|
||||
# Assert _perform_eval was called
|
||||
assert evaluation_result.overall_score == 0.9
|
||||
assert evaluation_result.overall_eval_status == EvalStatus.PASSED
|
||||
mock_perform_eval.assert_called_once()
|
||||
# Assert _print_results was called once with the result object
|
||||
# from _perform_eval
|
||||
mock_print_results.assert_called_once_with(MOCK_EVAL_RESULT)
|
||||
|
||||
@patch(
|
||||
"google.adk.evaluation.response_evaluator.ResponseEvaluator._print_results"
|
||||
)
|
||||
def test_evaluate_no_print_detailed_results(
|
||||
self, mock_print_results, mock_perform_eval
|
||||
def test_evaluate_invocations_coherence_metric_failed(
|
||||
self, mock_perform_eval
|
||||
):
|
||||
"""Test _print_results function is NOT called when print_detailed_results=False (default)."""
|
||||
mock_perform_eval.return_value = MOCK_EVAL_RESULT
|
||||
"""Test evaluate_invocations function for Coherence metric."""
|
||||
actual_invocations = [
|
||||
Invocation(
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="This is a test query.")]
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[
|
||||
genai_types.Part(text="This is a test candidate response.")
|
||||
]
|
||||
),
|
||||
)
|
||||
]
|
||||
expected_invocations = [
|
||||
Invocation(
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="This is a test query.")]
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[genai_types.Part(text="This is a test reference.")]
|
||||
),
|
||||
)
|
||||
]
|
||||
evaluator = ResponseEvaluator(
|
||||
threshold=0.8, metric_name="response_evaluation_score"
|
||||
)
|
||||
# Mock the return value of _perform_eval
|
||||
mock_perform_eval.return_value = vertexai_types.EvaluationResult(
|
||||
summary_metrics=[vertexai_types.AggregatedMetricResult(mean_score=0.7)],
|
||||
eval_case_results=[],
|
||||
)
|
||||
|
||||
raw_data = [[SAMPLE_TURN_1_ALL_KEYS]]
|
||||
criteria = ["response_match_score"]
|
||||
evaluation_result = evaluator.evaluate_invocations(
|
||||
actual_invocations, expected_invocations
|
||||
)
|
||||
|
||||
ResponseEvaluator.evaluate(raw_data, criteria, print_detailed_results=False)
|
||||
|
||||
# Assert _perform_eval was called
|
||||
assert evaluation_result.overall_score == 0.7
|
||||
assert evaluation_result.overall_eval_status == EvalStatus.FAILED
|
||||
mock_perform_eval.assert_called_once()
|
||||
# Assert _print_results was NOT called
|
||||
mock_print_results.assert_not_called()
|
||||
|
||||
def test_evaluate_invocations_coherence_metric_no_score(
|
||||
self, mock_perform_eval
|
||||
):
|
||||
"""Test evaluate_invocations function for Coherence metric."""
|
||||
actual_invocations = [
|
||||
Invocation(
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="This is a test query.")]
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[
|
||||
genai_types.Part(text="This is a test candidate response.")
|
||||
]
|
||||
),
|
||||
)
|
||||
]
|
||||
expected_invocations = [
|
||||
Invocation(
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="This is a test query.")]
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[genai_types.Part(text="This is a test reference.")]
|
||||
),
|
||||
)
|
||||
]
|
||||
evaluator = ResponseEvaluator(
|
||||
threshold=0.8, metric_name="response_evaluation_score"
|
||||
)
|
||||
# Mock the return value of _perform_eval
|
||||
mock_perform_eval.return_value = vertexai_types.EvaluationResult(
|
||||
summary_metrics=[],
|
||||
eval_case_results=[],
|
||||
)
|
||||
|
||||
evaluation_result = evaluator.evaluate_invocations(
|
||||
actual_invocations, expected_invocations
|
||||
)
|
||||
|
||||
assert evaluation_result.overall_score is None
|
||||
assert evaluation_result.overall_eval_status == EvalStatus.NOT_EVALUATED
|
||||
mock_perform_eval.assert_called_once()
|
||||
|
||||
def test_evaluate_invocations_coherence_metric_multiple_invocations(
|
||||
self, mock_perform_eval
|
||||
):
|
||||
"""Test evaluate_invocations function for Coherence metric with multiple invocations."""
|
||||
num_invocations = 6
|
||||
actual_invocations = []
|
||||
expected_invocations = []
|
||||
mock_eval_results = []
|
||||
random.seed(61553)
|
||||
scores = [random.random() for _ in range(num_invocations)]
|
||||
|
||||
for i in range(num_invocations):
|
||||
actual_invocations.append(
|
||||
Invocation(
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text=f"Query {i+1}")]
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[genai_types.Part(text=f"Response {i+1}")]
|
||||
),
|
||||
)
|
||||
)
|
||||
expected_invocations.append(
|
||||
Invocation(
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text=f"Query {i+1}")]
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[genai_types.Part(text=f"Reference {i+1}")]
|
||||
),
|
||||
)
|
||||
)
|
||||
mock_eval_results.append(
|
||||
vertexai_types.EvaluationResult(
|
||||
summary_metrics=[
|
||||
vertexai_types.AggregatedMetricResult(mean_score=scores[i])
|
||||
],
|
||||
eval_case_results=[],
|
||||
)
|
||||
)
|
||||
|
||||
evaluator = ResponseEvaluator(
|
||||
threshold=0.8, metric_name="response_evaluation_score"
|
||||
)
|
||||
# Mock the return value of _perform_eval
|
||||
mock_perform_eval.side_effect = mock_eval_results
|
||||
|
||||
evaluation_result = evaluator.evaluate_invocations(
|
||||
actual_invocations, expected_invocations
|
||||
)
|
||||
|
||||
assert evaluation_result.overall_score == pytest.approx(
|
||||
sum(scores) / num_invocations
|
||||
)
|
||||
assert evaluation_result.overall_eval_status == EvalStatus.FAILED
|
||||
assert mock_perform_eval.call_count == num_invocations
|
||||
|
||||
Reference in New Issue
Block a user