mirror of
https://github.com/encounter/adk-python.git
synced 2026-07-09 18:19:28 -07:00
chore: Update ResponseEvaluator to use newer version of Eval SDK
Also, - removed functionality that was marked deprecated from the ResponseEvaluator class. - Added unit test cases PiperOrigin-RevId: 778568884
This commit is contained in:
Ankur Sharma
committed by
Copybara-Service
Copybara-Service
parent
08869ccc07
commit
62c4a85917
@@ -13,53 +13,15 @@
|
||||
# limitations under the License.
|
||||
|
||||
"""Tests for the Response Evaluator."""
|
||||
from unittest.mock import MagicMock
|
||||
import random
|
||||
from unittest.mock import patch
|
||||
|
||||
from google.adk.evaluation.eval_case import Invocation
|
||||
from google.adk.evaluation.evaluator import EvalStatus
|
||||
from google.adk.evaluation.response_evaluator import ResponseEvaluator
|
||||
from google.genai import types as genai_types
|
||||
import pandas as pd
|
||||
import pytest
|
||||
from vertexai.preview.evaluation import MetricPromptTemplateExamples
|
||||
|
||||
# Mock object for the result normally returned by _perform_eval
|
||||
MOCK_EVAL_RESULT = MagicMock()
|
||||
MOCK_EVAL_RESULT.summary_metrics = {"mock_metric": 0.75, "another_mock": 3.5}
|
||||
# Add a metrics_table for testing _print_results interaction
|
||||
MOCK_EVAL_RESULT.metrics_table = pd.DataFrame({
|
||||
"prompt": ["mock_query1"],
|
||||
"response": ["mock_resp1"],
|
||||
"mock_metric": [0.75],
|
||||
})
|
||||
|
||||
SAMPLE_TURN_1_ALL_KEYS = {
|
||||
"query": "query1",
|
||||
"response": "response1",
|
||||
"actual_tool_use": [{"tool_name": "tool_a", "tool_input": {}}],
|
||||
"expected_tool_use": [{"tool_name": "tool_a", "tool_input": {}}],
|
||||
"reference": "reference1",
|
||||
}
|
||||
SAMPLE_TURN_2_MISSING_REF = {
|
||||
"query": "query2",
|
||||
"response": "response2",
|
||||
"actual_tool_use": [],
|
||||
"expected_tool_use": [],
|
||||
# "reference": "reference2" # Missing
|
||||
}
|
||||
SAMPLE_TURN_3_MISSING_EXP_TOOLS = {
|
||||
"query": "query3",
|
||||
"response": "response3",
|
||||
"actual_tool_use": [{"tool_name": "tool_b", "tool_input": {}}],
|
||||
# "expected_tool_use": [], # Missing
|
||||
"reference": "reference3",
|
||||
}
|
||||
SAMPLE_TURN_4_MINIMAL = {
|
||||
"query": "query4",
|
||||
"response": "response4",
|
||||
# Minimal keys, others missing
|
||||
}
|
||||
from vertexai import types as vertexai_types
|
||||
|
||||
|
||||
@patch(
|
||||
@@ -68,18 +30,6 @@ SAMPLE_TURN_4_MINIMAL = {
|
||||
class TestResponseEvaluator:
|
||||
"""A class to help organize "patch" that are applicable to all tests."""
|
||||
|
||||
def test_evaluate_none_dataset_raises_value_error(self, mock_perform_eval):
|
||||
"""Test evaluate function raises ValueError for an empty list."""
|
||||
with pytest.raises(ValueError, match="The evaluation dataset is empty."):
|
||||
ResponseEvaluator.evaluate(None, ["response_evaluation_score"])
|
||||
mock_perform_eval.assert_not_called() # Ensure _perform_eval was not called
|
||||
|
||||
def test_evaluate_empty_dataset_raises_value_error(self, mock_perform_eval):
|
||||
"""Test evaluate function raises ValueError for an empty list."""
|
||||
with pytest.raises(ValueError, match="The evaluation dataset is empty."):
|
||||
ResponseEvaluator.evaluate([], ["response_evaluation_score"])
|
||||
mock_perform_eval.assert_not_called() # Ensure _perform_eval was not called
|
||||
|
||||
def test_evaluate_invocations_rouge_metric(self, mock_perform_eval):
|
||||
"""Test evaluate_invocations function for Rouge metric."""
|
||||
actual_invocations = [
|
||||
@@ -107,190 +57,198 @@ class TestResponseEvaluator:
|
||||
evaluator = ResponseEvaluator(
|
||||
threshold=0.8, metric_name="response_match_score"
|
||||
)
|
||||
|
||||
evaluation_result = evaluator.evaluate_invocations(
|
||||
actual_invocations, expected_invocations
|
||||
)
|
||||
|
||||
assert evaluation_result.overall_score == pytest.approx(8 / 11)
|
||||
# ROUGE-1 F1 is approx. 0.73 < 0.8 threshold, so eval status is FAILED.
|
||||
assert evaluation_result.overall_eval_status == EvalStatus.FAILED
|
||||
mock_perform_eval.assert_not_called() # Ensure _perform_eval was not called
|
||||
|
||||
def test_evaluate_determines_metrics_correctly_for_perform_eval(
|
||||
def test_evaluate_invocations_coherence_metric_passed(
|
||||
self, mock_perform_eval
|
||||
):
|
||||
"""Test that the correct metrics list is passed to _perform_eval based on criteria/keys."""
|
||||
mock_perform_eval.return_value = MOCK_EVAL_RESULT
|
||||
|
||||
# Test case 1: Only Coherence
|
||||
raw_data_1 = [[SAMPLE_TURN_1_ALL_KEYS]]
|
||||
criteria_1 = ["response_evaluation_score"]
|
||||
ResponseEvaluator.evaluate(raw_data_1, criteria_1)
|
||||
_, kwargs = mock_perform_eval.call_args
|
||||
assert kwargs["metrics"] == [
|
||||
MetricPromptTemplateExamples.Pointwise.COHERENCE
|
||||
"""Test evaluate_invocations function for Coherence metric."""
|
||||
actual_invocations = [
|
||||
Invocation(
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="This is a test query.")]
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[
|
||||
genai_types.Part(text="This is a test candidate response.")
|
||||
]
|
||||
),
|
||||
)
|
||||
]
|
||||
mock_perform_eval.reset_mock() # Reset mock for next call
|
||||
|
||||
# Test case 2: Only Rouge
|
||||
raw_data_2 = [[SAMPLE_TURN_1_ALL_KEYS]]
|
||||
criteria_2 = ["response_match_score"]
|
||||
ResponseEvaluator.evaluate(raw_data_2, criteria_2)
|
||||
_, kwargs = mock_perform_eval.call_args
|
||||
assert kwargs["metrics"] == ["rouge_1"]
|
||||
mock_perform_eval.reset_mock()
|
||||
|
||||
# Test case 3: No metrics if keys missing in first turn
|
||||
raw_data_3 = [[SAMPLE_TURN_4_MINIMAL, SAMPLE_TURN_1_ALL_KEYS]]
|
||||
criteria_3 = ["response_evaluation_score", "response_match_score"]
|
||||
ResponseEvaluator.evaluate(raw_data_3, criteria_3)
|
||||
_, kwargs = mock_perform_eval.call_args
|
||||
assert kwargs["metrics"] == []
|
||||
mock_perform_eval.reset_mock()
|
||||
|
||||
# Test case 4: No metrics if criteria empty
|
||||
raw_data_4 = [[SAMPLE_TURN_1_ALL_KEYS]]
|
||||
criteria_4 = []
|
||||
ResponseEvaluator.evaluate(raw_data_4, criteria_4)
|
||||
_, kwargs = mock_perform_eval.call_args
|
||||
assert kwargs["metrics"] == []
|
||||
mock_perform_eval.reset_mock()
|
||||
|
||||
def test_evaluate_calls_perform_eval_correctly_all_metrics(
|
||||
self, mock_perform_eval
|
||||
):
|
||||
"""Test evaluate function calls _perform_eval with expected args when all criteria/keys are present."""
|
||||
# Arrange
|
||||
mock_perform_eval.return_value = (
|
||||
MOCK_EVAL_RESULT # Configure the mock return value
|
||||
expected_invocations = [
|
||||
Invocation(
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="This is a test query.")]
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[genai_types.Part(text="This is a test reference.")]
|
||||
),
|
||||
)
|
||||
]
|
||||
evaluator = ResponseEvaluator(
|
||||
threshold=0.8, metric_name="response_evaluation_score"
|
||||
)
|
||||
# Mock the return value of _perform_eval
|
||||
mock_perform_eval.return_value = vertexai_types.EvaluationResult(
|
||||
summary_metrics=[vertexai_types.AggregatedMetricResult(mean_score=0.9)],
|
||||
eval_case_results=[],
|
||||
)
|
||||
|
||||
raw_data = [[SAMPLE_TURN_1_ALL_KEYS]]
|
||||
criteria = ["response_evaluation_score", "response_match_score"]
|
||||
|
||||
# Act
|
||||
summary = ResponseEvaluator.evaluate(raw_data, criteria)
|
||||
|
||||
# Assert
|
||||
# 1. Check metrics determined by _get_metrics (passed to _perform_eval)
|
||||
expected_metrics_list = [
|
||||
MetricPromptTemplateExamples.Pointwise.COHERENCE,
|
||||
"rouge_1",
|
||||
]
|
||||
# 2. Check DataFrame prepared (passed to _perform_eval)
|
||||
expected_df_data = [{
|
||||
"prompt": "query1",
|
||||
"response": "response1",
|
||||
"actual_tool_use": [{"tool_name": "tool_a", "tool_input": {}}],
|
||||
"reference_trajectory": [{"tool_name": "tool_a", "tool_input": {}}],
|
||||
"reference": "reference1",
|
||||
}]
|
||||
expected_df = pd.DataFrame(expected_df_data)
|
||||
|
||||
# Assert _perform_eval was called once
|
||||
mock_perform_eval.assert_called_once()
|
||||
# Get the arguments passed to the mocked _perform_eval
|
||||
_, kwargs = mock_perform_eval.call_args
|
||||
# Check the 'dataset' keyword argument
|
||||
pd.testing.assert_frame_equal(kwargs["dataset"], expected_df)
|
||||
# Check the 'metrics' keyword argument
|
||||
assert kwargs["metrics"] == expected_metrics_list
|
||||
|
||||
# 3. Check the correct summary metrics are returned
|
||||
# (from mock_perform_eval's return value)
|
||||
assert summary == MOCK_EVAL_RESULT.summary_metrics
|
||||
|
||||
def test_evaluate_prepares_dataframe_correctly_for_perform_eval(
|
||||
self, mock_perform_eval
|
||||
):
|
||||
"""Test that the DataFrame is correctly flattened and renamed before passing to _perform_eval."""
|
||||
mock_perform_eval.return_value = MOCK_EVAL_RESULT
|
||||
|
||||
raw_data = [
|
||||
[SAMPLE_TURN_1_ALL_KEYS], # Conversation 1
|
||||
[
|
||||
SAMPLE_TURN_2_MISSING_REF,
|
||||
SAMPLE_TURN_3_MISSING_EXP_TOOLS,
|
||||
], # Conversation 2
|
||||
]
|
||||
criteria = [
|
||||
"response_match_score"
|
||||
] # Doesn't affect the DataFrame structure
|
||||
|
||||
ResponseEvaluator.evaluate(raw_data, criteria)
|
||||
|
||||
# Expected flattened and renamed data
|
||||
expected_df_data = [
|
||||
# Turn 1 (from SAMPLE_TURN_1_ALL_KEYS)
|
||||
{
|
||||
"prompt": "query1",
|
||||
"response": "response1",
|
||||
"actual_tool_use": [{"tool_name": "tool_a", "tool_input": {}}],
|
||||
"reference_trajectory": [{"tool_name": "tool_a", "tool_input": {}}],
|
||||
"reference": "reference1",
|
||||
},
|
||||
# Turn 2 (from SAMPLE_TURN_2_MISSING_REF)
|
||||
{
|
||||
"prompt": "query2",
|
||||
"response": "response2",
|
||||
"actual_tool_use": [],
|
||||
"reference_trajectory": [],
|
||||
# "reference": None # Missing key results in NaN in DataFrame
|
||||
# usually
|
||||
},
|
||||
# Turn 3 (from SAMPLE_TURN_3_MISSING_EXP_TOOLS)
|
||||
{
|
||||
"prompt": "query3",
|
||||
"response": "response3",
|
||||
"actual_tool_use": [{"tool_name": "tool_b", "tool_input": {}}],
|
||||
# "reference_trajectory": None, # Missing key results in NaN
|
||||
"reference": "reference3",
|
||||
},
|
||||
]
|
||||
# Need to be careful with missing keys -> NaN when creating DataFrame
|
||||
# Pandas handles this automatically when creating from list of dicts
|
||||
expected_df = pd.DataFrame(expected_df_data)
|
||||
|
||||
mock_perform_eval.assert_called_once()
|
||||
_, kwargs = mock_perform_eval.call_args
|
||||
# Compare the DataFrame passed to the mock
|
||||
pd.testing.assert_frame_equal(kwargs["dataset"], expected_df)
|
||||
|
||||
@patch(
|
||||
"google.adk.evaluation.response_evaluator.ResponseEvaluator._print_results"
|
||||
) # Mock the private print method
|
||||
def test_evaluate_print_detailed_results(
|
||||
self, mock_print_results, mock_perform_eval
|
||||
):
|
||||
"""Test _print_results function is called when print_detailed_results=True."""
|
||||
mock_perform_eval.return_value = (
|
||||
MOCK_EVAL_RESULT # Ensure _perform_eval returns our mock result
|
||||
evaluation_result = evaluator.evaluate_invocations(
|
||||
actual_invocations, expected_invocations
|
||||
)
|
||||
|
||||
raw_data = [[SAMPLE_TURN_1_ALL_KEYS]]
|
||||
criteria = ["response_match_score"]
|
||||
|
||||
ResponseEvaluator.evaluate(raw_data, criteria, print_detailed_results=True)
|
||||
|
||||
# Assert _perform_eval was called
|
||||
assert evaluation_result.overall_score == 0.9
|
||||
assert evaluation_result.overall_eval_status == EvalStatus.PASSED
|
||||
mock_perform_eval.assert_called_once()
|
||||
# Assert _print_results was called once with the result object
|
||||
# from _perform_eval
|
||||
mock_print_results.assert_called_once_with(MOCK_EVAL_RESULT)
|
||||
|
||||
@patch(
|
||||
"google.adk.evaluation.response_evaluator.ResponseEvaluator._print_results"
|
||||
)
|
||||
def test_evaluate_no_print_detailed_results(
|
||||
self, mock_print_results, mock_perform_eval
|
||||
def test_evaluate_invocations_coherence_metric_failed(
|
||||
self, mock_perform_eval
|
||||
):
|
||||
"""Test _print_results function is NOT called when print_detailed_results=False (default)."""
|
||||
mock_perform_eval.return_value = MOCK_EVAL_RESULT
|
||||
"""Test evaluate_invocations function for Coherence metric."""
|
||||
actual_invocations = [
|
||||
Invocation(
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="This is a test query.")]
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[
|
||||
genai_types.Part(text="This is a test candidate response.")
|
||||
]
|
||||
),
|
||||
)
|
||||
]
|
||||
expected_invocations = [
|
||||
Invocation(
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="This is a test query.")]
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[genai_types.Part(text="This is a test reference.")]
|
||||
),
|
||||
)
|
||||
]
|
||||
evaluator = ResponseEvaluator(
|
||||
threshold=0.8, metric_name="response_evaluation_score"
|
||||
)
|
||||
# Mock the return value of _perform_eval
|
||||
mock_perform_eval.return_value = vertexai_types.EvaluationResult(
|
||||
summary_metrics=[vertexai_types.AggregatedMetricResult(mean_score=0.7)],
|
||||
eval_case_results=[],
|
||||
)
|
||||
|
||||
raw_data = [[SAMPLE_TURN_1_ALL_KEYS]]
|
||||
criteria = ["response_match_score"]
|
||||
evaluation_result = evaluator.evaluate_invocations(
|
||||
actual_invocations, expected_invocations
|
||||
)
|
||||
|
||||
ResponseEvaluator.evaluate(raw_data, criteria, print_detailed_results=False)
|
||||
|
||||
# Assert _perform_eval was called
|
||||
assert evaluation_result.overall_score == 0.7
|
||||
assert evaluation_result.overall_eval_status == EvalStatus.FAILED
|
||||
mock_perform_eval.assert_called_once()
|
||||
# Assert _print_results was NOT called
|
||||
mock_print_results.assert_not_called()
|
||||
|
||||
def test_evaluate_invocations_coherence_metric_no_score(
|
||||
self, mock_perform_eval
|
||||
):
|
||||
"""Test evaluate_invocations function for Coherence metric."""
|
||||
actual_invocations = [
|
||||
Invocation(
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="This is a test query.")]
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[
|
||||
genai_types.Part(text="This is a test candidate response.")
|
||||
]
|
||||
),
|
||||
)
|
||||
]
|
||||
expected_invocations = [
|
||||
Invocation(
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="This is a test query.")]
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[genai_types.Part(text="This is a test reference.")]
|
||||
),
|
||||
)
|
||||
]
|
||||
evaluator = ResponseEvaluator(
|
||||
threshold=0.8, metric_name="response_evaluation_score"
|
||||
)
|
||||
# Mock the return value of _perform_eval
|
||||
mock_perform_eval.return_value = vertexai_types.EvaluationResult(
|
||||
summary_metrics=[],
|
||||
eval_case_results=[],
|
||||
)
|
||||
|
||||
evaluation_result = evaluator.evaluate_invocations(
|
||||
actual_invocations, expected_invocations
|
||||
)
|
||||
|
||||
assert evaluation_result.overall_score is None
|
||||
assert evaluation_result.overall_eval_status == EvalStatus.NOT_EVALUATED
|
||||
mock_perform_eval.assert_called_once()
|
||||
|
||||
def test_evaluate_invocations_coherence_metric_multiple_invocations(
|
||||
self, mock_perform_eval
|
||||
):
|
||||
"""Test evaluate_invocations function for Coherence metric with multiple invocations."""
|
||||
num_invocations = 6
|
||||
actual_invocations = []
|
||||
expected_invocations = []
|
||||
mock_eval_results = []
|
||||
random.seed(61553)
|
||||
scores = [random.random() for _ in range(num_invocations)]
|
||||
|
||||
for i in range(num_invocations):
|
||||
actual_invocations.append(
|
||||
Invocation(
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text=f"Query {i+1}")]
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[genai_types.Part(text=f"Response {i+1}")]
|
||||
),
|
||||
)
|
||||
)
|
||||
expected_invocations.append(
|
||||
Invocation(
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text=f"Query {i+1}")]
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[genai_types.Part(text=f"Reference {i+1}")]
|
||||
),
|
||||
)
|
||||
)
|
||||
mock_eval_results.append(
|
||||
vertexai_types.EvaluationResult(
|
||||
summary_metrics=[
|
||||
vertexai_types.AggregatedMetricResult(mean_score=scores[i])
|
||||
],
|
||||
eval_case_results=[],
|
||||
)
|
||||
)
|
||||
|
||||
evaluator = ResponseEvaluator(
|
||||
threshold=0.8, metric_name="response_evaluation_score"
|
||||
)
|
||||
# Mock the return value of _perform_eval
|
||||
mock_perform_eval.side_effect = mock_eval_results
|
||||
|
||||
evaluation_result = evaluator.evaluate_invocations(
|
||||
actual_invocations, expected_invocations
|
||||
)
|
||||
|
||||
assert evaluation_result.overall_score == pytest.approx(
|
||||
sum(scores) / num_invocations
|
||||
)
|
||||
assert evaluation_result.overall_eval_status == EvalStatus.FAILED
|
||||
assert mock_perform_eval.call_count == num_invocations
|
||||
|
||||
Reference in New Issue
Block a user