mirror of
https://github.com/encounter/adk-python.git
synced 2026-07-09 18:19:28 -07:00
feat: Introduce User Personas to the ADK evaluation framework
PiperOrigin-RevId: 871366815
This commit is contained in:
committed by
Copybara-Service
parent
976a238544
commit
6a808c60b3
@@ -763,7 +763,7 @@ def test_cli_add_eval_case_with_session(tmp_path: Path):
|
||||
eval_set_data = json.load(f)
|
||||
assert len(eval_set_data["eval_cases"]) == 1
|
||||
eval_case = eval_set_data["eval_cases"][0]
|
||||
assert eval_case["eval_id"] == "0a1a5048"
|
||||
assert eval_case["eval_id"] == "734909ff"
|
||||
assert eval_case["session_input"]["app_name"] == "test_app_add_2"
|
||||
|
||||
|
||||
|
||||
@@ -18,6 +18,8 @@ from google.adk.evaluation import conversation_scenarios
|
||||
from google.adk.evaluation.simulation.llm_backed_user_simulator import LlmBackedUserSimulator
|
||||
from google.adk.evaluation.simulation.llm_backed_user_simulator import LlmBackedUserSimulatorConfig
|
||||
from google.adk.evaluation.simulation.user_simulator import Status
|
||||
from google.adk.evaluation.simulation.user_simulator_personas import UserBehavior
|
||||
from google.adk.evaluation.simulation.user_simulator_personas import UserPersona
|
||||
from google.adk.events.event import Event
|
||||
from google.genai import types
|
||||
from pydantic import ValidationError
|
||||
@@ -94,7 +96,7 @@ def test_llm_backed_user_simulator_config_validation():
|
||||
config = LlmBackedUserSimulatorConfig(custom_instructions=None)
|
||||
assert config.custom_instructions is None
|
||||
valid_instructions = (
|
||||
"{stop_signal} {conversation_plan} {conversation_history}"
|
||||
"{{ stop_signal }} {{ conversation_plan }} {{ conversation_history }}"
|
||||
)
|
||||
config = LlmBackedUserSimulatorConfig(custom_instructions=valid_instructions)
|
||||
assert config.custom_instructions == valid_instructions
|
||||
@@ -144,12 +146,38 @@ def conversation_scenario():
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def user_persona():
|
||||
"""Provides a test user persona."""
|
||||
return UserPersona(
|
||||
id="test_persona",
|
||||
description="A test persona",
|
||||
behaviors=[
|
||||
UserBehavior(
|
||||
name="polite",
|
||||
description="is polite",
|
||||
behavior_instructions=["Always say please and thank you."],
|
||||
violation_rubrics=["is rude"],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def conversation_scenario_with_persona(user_persona):
|
||||
"""Provides a test conversation scenario with a user persona."""
|
||||
return conversation_scenarios.ConversationScenario(
|
||||
starting_prompt="Hello",
|
||||
conversation_plan="test plan with persona",
|
||||
user_persona=user_persona,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def simulator(mock_llm_agent, conversation_scenario):
|
||||
"""Provides an LlmBackedUserSimulator instance for testing."""
|
||||
config = LlmBackedUserSimulatorConfig(
|
||||
model="test-model",
|
||||
model_configuration=types.GenerateContentConfig(),
|
||||
)
|
||||
sim = LlmBackedUserSimulator(
|
||||
config=config, conversation_scenario=conversation_scenario
|
||||
@@ -158,6 +186,19 @@ def simulator(mock_llm_agent, conversation_scenario):
|
||||
return sim
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def simulator_with_persona(mock_llm_agent, conversation_scenario_with_persona):
|
||||
"""Provides an LlmBackedUserSimulator instance for testing."""
|
||||
config = LlmBackedUserSimulatorConfig(
|
||||
model="test-model",
|
||||
)
|
||||
sim = LlmBackedUserSimulator(
|
||||
config=config, conversation_scenario=conversation_scenario_with_persona
|
||||
)
|
||||
sim._invocation_count = 1 # Bypass starting prompt by default for tests
|
||||
return sim
|
||||
|
||||
|
||||
class TestLlmBackedUserSimulator:
|
||||
"""Test cases for LlmBackedUserSimulator main methods."""
|
||||
|
||||
@@ -262,3 +303,27 @@ class TestLlmBackedUserSimulator:
|
||||
|
||||
assert next_user_message.status == Status.SUCCESS
|
||||
assert next_user_message.user_message == expected_user_message
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_next_user_message_with_persona_success(
|
||||
self, simulator_with_persona, mock_llm_agent, mocker
|
||||
):
|
||||
"""Tests get_next_user_message when the user message is generated successfully."""
|
||||
mock_llm_response = mocker.MagicMock()
|
||||
mock_llm_response.content = types.Content(
|
||||
parts=[types.Part(text="I need to book a flight.")]
|
||||
)
|
||||
mock_llm_agent.generate_content_async.return_value = to_async_iter(
|
||||
[mock_llm_response]
|
||||
)
|
||||
|
||||
next_user_message = await simulator_with_persona.get_next_user_message(
|
||||
events=_INPUT_EVENTS
|
||||
)
|
||||
|
||||
expected_user_message = types.Content(
|
||||
parts=[types.Part(text="I need to book a flight.")], role="user"
|
||||
)
|
||||
|
||||
assert next_user_message.status == Status.SUCCESS
|
||||
assert next_user_message.user_message == expected_user_message
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import textwrap
|
||||
|
||||
from google.adk.evaluation.simulation.llm_backed_user_simulator_prompts import _DEFAULT_USER_SIMULATOR_INSTRUCTIONS_TEMPLATE
|
||||
from google.adk.evaluation.simulation.llm_backed_user_simulator_prompts import _get_user_simulator_instructions_template
|
||||
from google.adk.evaluation.simulation.llm_backed_user_simulator_prompts import _USER_SIMULATOR_INSTRUCTIONS_WITH_PERSONA_TEMPLATE
|
||||
from google.adk.evaluation.simulation.llm_backed_user_simulator_prompts import get_llm_backed_user_simulator_prompt
|
||||
from google.adk.evaluation.simulation.llm_backed_user_simulator_prompts import is_valid_user_simulator_template
|
||||
from google.adk.evaluation.simulation.user_simulator_personas import UserBehavior
|
||||
from google.adk.evaluation.simulation.user_simulator_personas import UserPersona
|
||||
import pytest
|
||||
|
||||
_MOCK_DEFAULT_TEMPLATE = textwrap.dedent("""\
|
||||
Default template
|
||||
|
||||
# Conversation Plan
|
||||
{{conversation_plan}}
|
||||
|
||||
# Conversation History
|
||||
{{conversation_history}}
|
||||
|
||||
# Stop signal
|
||||
{{stop_signal}}
|
||||
""").strip()
|
||||
|
||||
_MOCK_PERSONA_TEMPLATE = textwrap.dedent("""\
|
||||
Persona template
|
||||
|
||||
# Persona Description
|
||||
{{persona.description}}
|
||||
{% for b in persona.behaviors %}
|
||||
## {{ b.name }}
|
||||
{{ b.description }}
|
||||
|
||||
Instructions:
|
||||
{{ b.get_behavior_instructions_str() }}
|
||||
{% endfor %}
|
||||
# Conversation Plan
|
||||
{{conversation_plan}}
|
||||
|
||||
# Conversation History
|
||||
{{conversation_history}}
|
||||
|
||||
# Stop signal
|
||||
{{stop_signal}}
|
||||
""").strip()
|
||||
|
||||
|
||||
class TestGetUserSimulatorInstructionsTemplate:
|
||||
"""Test cases for _get_user_simulator_instructions_template."""
|
||||
|
||||
def test_get_user_simulator_instructions_template_default(self):
|
||||
assert (
|
||||
_get_user_simulator_instructions_template()
|
||||
== _DEFAULT_USER_SIMULATOR_INSTRUCTIONS_TEMPLATE
|
||||
)
|
||||
|
||||
def test_get_user_simulator_instructions_template_with_custom_instructions(
|
||||
self,
|
||||
):
|
||||
custom_instructions = "custom instructions"
|
||||
assert (
|
||||
_get_user_simulator_instructions_template(
|
||||
custom_instructions=custom_instructions
|
||||
)
|
||||
== custom_instructions
|
||||
)
|
||||
|
||||
def test_get_user_simulator_instructions_template_with_persona(self):
|
||||
user_persona = UserPersona(
|
||||
id="test_persona", description="Test persona", behaviors=[]
|
||||
)
|
||||
assert (
|
||||
_get_user_simulator_instructions_template(user_persona=user_persona)
|
||||
== _USER_SIMULATOR_INSTRUCTIONS_WITH_PERSONA_TEMPLATE
|
||||
)
|
||||
|
||||
def test_get_user_simulator_instructions_template_with_bad_custom_instructions_raises_error(
|
||||
self,
|
||||
):
|
||||
custom_instructions = "custom instructions"
|
||||
user_persona = UserPersona(
|
||||
id="test_persona", description="Test persona", behaviors=[]
|
||||
)
|
||||
with pytest.raises(ValueError):
|
||||
_get_user_simulator_instructions_template(
|
||||
custom_instructions=custom_instructions, user_persona=user_persona
|
||||
)
|
||||
|
||||
|
||||
sample_persona = UserPersona(
|
||||
id="test_persona",
|
||||
description="Test persona description",
|
||||
behaviors=[
|
||||
UserBehavior(
|
||||
name="Test behavior",
|
||||
description="Test behavior description",
|
||||
behavior_instructions=["instruction 1", "instruction 2"],
|
||||
violation_rubrics=["rubric 1"],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class TestGetLlmBackedUserSimulatorPrompt:
|
||||
"""Test cases for get_llm_backed_user_simulator_prompt."""
|
||||
|
||||
def test_get_llm_backed_user_simulator_prompt_default(self, mocker):
|
||||
mocker.patch(
|
||||
"google.adk.evaluation.simulation.llm_backed_user_simulator_prompts._DEFAULT_USER_SIMULATOR_INSTRUCTIONS_TEMPLATE",
|
||||
_MOCK_DEFAULT_TEMPLATE,
|
||||
)
|
||||
prompt = get_llm_backed_user_simulator_prompt(
|
||||
conversation_plan="test plan",
|
||||
conversation_history="test history",
|
||||
stop_signal="test stop",
|
||||
)
|
||||
expected_prompt = textwrap.dedent("""\
|
||||
Default template
|
||||
|
||||
# Conversation Plan
|
||||
test plan
|
||||
|
||||
# Conversation History
|
||||
test history
|
||||
|
||||
# Stop signal
|
||||
test stop""").strip()
|
||||
|
||||
assert prompt == expected_prompt
|
||||
|
||||
def test_get_llm_backed_user_simulator_prompt_with_custom_instructions(self):
|
||||
custom_instructions = textwrap.dedent("""\
|
||||
Custom instructions:
|
||||
|
||||
# Past history
|
||||
{{conversation_plan}}
|
||||
|
||||
# Plan
|
||||
{{conversation_plan}}
|
||||
|
||||
# Finished!
|
||||
{{stop_signal}}""").strip()
|
||||
prompt = get_llm_backed_user_simulator_prompt(
|
||||
conversation_plan="test plan",
|
||||
conversation_history="test history",
|
||||
stop_signal="test stop",
|
||||
custom_instructions=custom_instructions,
|
||||
)
|
||||
|
||||
expected_prompt = textwrap.dedent("""\
|
||||
Custom instructions:
|
||||
|
||||
# Past history
|
||||
test plan
|
||||
|
||||
# Plan
|
||||
test plan
|
||||
|
||||
# Finished!
|
||||
test stop""").strip()
|
||||
assert prompt == expected_prompt
|
||||
|
||||
def test_get_llm_backed_user_simulator_prompt_with_persona(self, mocker):
|
||||
mocker.patch(
|
||||
"google.adk.evaluation.simulation.llm_backed_user_simulator_prompts._USER_SIMULATOR_INSTRUCTIONS_WITH_PERSONA_TEMPLATE",
|
||||
_MOCK_PERSONA_TEMPLATE,
|
||||
)
|
||||
prompt = get_llm_backed_user_simulator_prompt(
|
||||
conversation_plan="test plan",
|
||||
conversation_history="test history",
|
||||
stop_signal="test stop",
|
||||
user_persona=sample_persona,
|
||||
)
|
||||
expected_prompt = textwrap.dedent("""\
|
||||
Persona template
|
||||
|
||||
# Persona Description
|
||||
Test persona description
|
||||
|
||||
## Test behavior
|
||||
Test behavior description
|
||||
|
||||
Instructions:
|
||||
* instruction 1
|
||||
* instruction 2
|
||||
|
||||
# Conversation Plan
|
||||
test plan
|
||||
|
||||
# Conversation History
|
||||
test history
|
||||
|
||||
# Stop signal
|
||||
test stop""").strip()
|
||||
assert prompt == expected_prompt
|
||||
|
||||
|
||||
class TestIsValidUserSimulatorTemplate:
|
||||
"""Test cases for is_valid_user_simulator_template."""
|
||||
|
||||
def test_valid_template(self):
|
||||
template = "Hello {{ name }}"
|
||||
params = ["name"]
|
||||
assert is_valid_user_simulator_template(template, params) is True
|
||||
|
||||
def test_invalid_syntax(self):
|
||||
template = "Hello {{ name"
|
||||
params = ["name"]
|
||||
assert is_valid_user_simulator_template(template, params) is False
|
||||
|
||||
def test_missing_parameter(self):
|
||||
template = "Hello"
|
||||
params = ["name"]
|
||||
assert is_valid_user_simulator_template(template, params) is False
|
||||
+184
@@ -0,0 +1,184 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import textwrap
|
||||
|
||||
from google.adk.evaluation.simulation.per_turn_user_simulator_quality_prompts import _get_latest_turn_user_simulator_quality_prompt_template
|
||||
from google.adk.evaluation.simulation.per_turn_user_simulator_quality_prompts import _LATEST_TURN_USER_SIMULATOR_EVALUATOR_PROMPT_TEMPLATE
|
||||
from google.adk.evaluation.simulation.per_turn_user_simulator_quality_prompts import _LATEST_TURN_USER_SIMULATOR_WITH_PERSONA_EVALUATOR_PROMPT_TEMPLATE
|
||||
from google.adk.evaluation.simulation.per_turn_user_simulator_quality_prompts import get_per_turn_user_simulator_quality_prompt
|
||||
from google.adk.evaluation.simulation.user_simulator_personas import UserBehavior
|
||||
from google.adk.evaluation.simulation.user_simulator_personas import UserPersona
|
||||
|
||||
_MOCK_DEFAULT_TEMPLATE = textwrap.dedent("""\
|
||||
Default template
|
||||
|
||||
# Conversation Plan
|
||||
{{conversation_plan}}
|
||||
|
||||
# Conversation History
|
||||
{{conversation_history}}
|
||||
|
||||
# Generated User Response
|
||||
{{generated_user_response}}
|
||||
|
||||
# Stop signal
|
||||
{{stop_signal}}
|
||||
""").strip()
|
||||
|
||||
_MOCK_PERSONA_TEMPLATE = textwrap.dedent("""\
|
||||
Persona template
|
||||
|
||||
# Persona Description
|
||||
{{persona.description}}
|
||||
{% for b in persona.behaviors %}
|
||||
## Criteria: {{ b.name | render_string_filter}}
|
||||
{{ b.description | render_string_filter}}
|
||||
|
||||
Mark as FAIL if any of the following Violations occur:
|
||||
{{ b.get_violation_rubrics_str() | render_string_filter}}
|
||||
{% endfor %}
|
||||
# Conversation Plan
|
||||
{{conversation_plan}}
|
||||
|
||||
# Conversation History
|
||||
{{conversation_history}}
|
||||
|
||||
# Generated User Response
|
||||
{{generated_user_response}}
|
||||
|
||||
# Stop signal
|
||||
{{stop_signal}}
|
||||
""").strip()
|
||||
|
||||
|
||||
class TestGetLatestTurnUserSimulatorQualityPrompt:
|
||||
"""Test cases for get_latest_turn_user_simulator_quality_prompt."""
|
||||
|
||||
def test_get_get_latest_turn_user_simulator_quality_prompt_template_default(
|
||||
self,
|
||||
):
|
||||
prompt = _get_latest_turn_user_simulator_quality_prompt_template(
|
||||
user_persona=None
|
||||
)
|
||||
assert prompt == _LATEST_TURN_USER_SIMULATOR_EVALUATOR_PROMPT_TEMPLATE
|
||||
|
||||
def test_get_latest_turn_user_simulator_quality_prompt_template_with_persona(
|
||||
self,
|
||||
):
|
||||
"""Tests that the correct prompt is returned when a persona is provided."""
|
||||
persona = UserPersona(
|
||||
id="test_persona",
|
||||
description="Test persona description.",
|
||||
behaviors=[
|
||||
UserBehavior(
|
||||
name="test_behavior",
|
||||
description="Test behavior description.",
|
||||
behavior_instructions=["instruction1"],
|
||||
violation_rubrics=["violation1"],
|
||||
)
|
||||
],
|
||||
)
|
||||
prompt = _get_latest_turn_user_simulator_quality_prompt_template(
|
||||
user_persona=persona
|
||||
)
|
||||
assert (
|
||||
prompt
|
||||
== _LATEST_TURN_USER_SIMULATOR_WITH_PERSONA_EVALUATOR_PROMPT_TEMPLATE
|
||||
)
|
||||
|
||||
|
||||
class TestGetPerTurnUserSimulatorQualityPrompt:
|
||||
"""Test cases for get_per_turn_user_simulator_quality_prompt."""
|
||||
|
||||
def test_get_per_turn_user_simulator_quality_prompt_default(self, mocker):
|
||||
"""Tests that the correct prompt is returned when no persona is provided."""
|
||||
mocker.patch(
|
||||
"google.adk.evaluation.simulation.per_turn_user_simulator_quality_prompts._LATEST_TURN_USER_SIMULATOR_EVALUATOR_PROMPT_TEMPLATE",
|
||||
_MOCK_DEFAULT_TEMPLATE,
|
||||
)
|
||||
prompt = get_per_turn_user_simulator_quality_prompt(
|
||||
conversation_plan="plan",
|
||||
conversation_history="history",
|
||||
generated_user_response="response",
|
||||
stop_signal="stop",
|
||||
user_persona=None,
|
||||
)
|
||||
expected_prompt = textwrap.dedent("""\
|
||||
Default template
|
||||
|
||||
# Conversation Plan
|
||||
plan
|
||||
|
||||
# Conversation History
|
||||
history
|
||||
|
||||
# Generated User Response
|
||||
response
|
||||
|
||||
# Stop signal
|
||||
stop""").strip()
|
||||
assert prompt == expected_prompt
|
||||
|
||||
def test_get_per_turn_user_simulator_quality_prompt_with_persona(
|
||||
self, mocker
|
||||
):
|
||||
"""Tests that the correct prompt is returned when a persona is provided."""
|
||||
mocker.patch(
|
||||
"google.adk.evaluation.simulation.per_turn_user_simulator_quality_prompts._LATEST_TURN_USER_SIMULATOR_WITH_PERSONA_EVALUATOR_PROMPT_TEMPLATE",
|
||||
_MOCK_PERSONA_TEMPLATE,
|
||||
)
|
||||
persona = UserPersona(
|
||||
id="test_persona",
|
||||
description="Test persona description.",
|
||||
behaviors=[
|
||||
UserBehavior(
|
||||
name="test_behavior",
|
||||
description="Test behavior description.",
|
||||
behavior_instructions=["instruction1"],
|
||||
violation_rubrics=["violation1"],
|
||||
)
|
||||
],
|
||||
)
|
||||
prompt = get_per_turn_user_simulator_quality_prompt(
|
||||
conversation_plan="plan",
|
||||
conversation_history="history",
|
||||
generated_user_response="response",
|
||||
stop_signal="stop",
|
||||
user_persona=persona,
|
||||
)
|
||||
expected_prompt = textwrap.dedent("""\
|
||||
Persona template
|
||||
|
||||
# Persona Description
|
||||
Test persona description.
|
||||
|
||||
## Criteria: test_behavior
|
||||
Test behavior description.
|
||||
|
||||
Mark as FAIL if any of the following Violations occur:
|
||||
* violation1
|
||||
|
||||
# Conversation Plan
|
||||
plan
|
||||
|
||||
# Conversation History
|
||||
history
|
||||
|
||||
# Generated User Response
|
||||
response
|
||||
|
||||
# Stop signal
|
||||
stop""").strip()
|
||||
assert prompt == expected_prompt
|
||||
@@ -26,7 +26,10 @@ from google.adk.evaluation.llm_as_judge_utils import Label
|
||||
from google.adk.evaluation.simulation.per_turn_user_simulator_quality_v1 import _format_conversation_history
|
||||
from google.adk.evaluation.simulation.per_turn_user_simulator_quality_v1 import _parse_llm_response
|
||||
from google.adk.evaluation.simulation.per_turn_user_simulator_quality_v1 import PerTurnUserSimulatorQualityV1
|
||||
from google.adk.evaluation.simulation.user_simulator_personas import UserBehavior
|
||||
from google.adk.evaluation.simulation.user_simulator_personas import UserPersona
|
||||
from google.adk.models.llm_response import LlmResponse
|
||||
from google.genai import types
|
||||
from google.genai import types as genai_types
|
||||
import pytest
|
||||
|
||||
@@ -149,6 +152,54 @@ def test_parse_llm_response_label_valid(response_text):
|
||||
],
|
||||
"is_valid": "invalid",
|
||||
}
|
||||
```""",
|
||||
"""```json
|
||||
{
|
||||
"criteria": [
|
||||
{
|
||||
"name": "TEST_NAME",
|
||||
"reasoning": "test_resonining",
|
||||
"passes": False
|
||||
}
|
||||
],
|
||||
"is_valid": "almost",
|
||||
}
|
||||
```""",
|
||||
"""```json
|
||||
{
|
||||
"criteria": [
|
||||
{
|
||||
"name": "TEST_NAME",
|
||||
"reasoning": "test_resonining",
|
||||
"passes": False
|
||||
}
|
||||
],
|
||||
"is_valid": "partially_valid",
|
||||
}
|
||||
```""",
|
||||
"""```json
|
||||
{
|
||||
"criteria": [
|
||||
{
|
||||
"name": "TEST_NAME",
|
||||
"reasoning": "test_resonining",
|
||||
"passes": False
|
||||
}
|
||||
],
|
||||
"is_valid": "partially valid",
|
||||
}
|
||||
```""",
|
||||
"""```json
|
||||
{
|
||||
"criteria": [
|
||||
{
|
||||
"name": "TEST_NAME",
|
||||
"reasoning": "test_resonining",
|
||||
"passes": False
|
||||
}
|
||||
],
|
||||
"is_valid": "partially",
|
||||
}
|
||||
```""",
|
||||
],
|
||||
)
|
||||
@@ -158,16 +209,16 @@ def test_parse_llm_response_label_invalid(response_text):
|
||||
|
||||
|
||||
def create_test_template() -> str:
|
||||
return """This is a test template with stop signal: `{stop_signal}`.
|
||||
return """This is a test template with stop signal: `{{stop_signal}}`.
|
||||
|
||||
# Conversation Plan
|
||||
{conversation_plan}
|
||||
{{conversation_plan}}
|
||||
|
||||
# Conversation History
|
||||
{conversation_history}
|
||||
{{conversation_history}}
|
||||
|
||||
# Generated User Response
|
||||
{generated_user_response}
|
||||
{{generated_user_response}}
|
||||
""".strip()
|
||||
|
||||
|
||||
@@ -189,18 +240,19 @@ def _create_test_evaluator(
|
||||
),
|
||||
),
|
||||
)
|
||||
evaluator._prompt_template = create_test_template()
|
||||
return evaluator
|
||||
|
||||
|
||||
def _create_test_conversation_scenario(
|
||||
conversation_plan: str = "test conversation plan",
|
||||
starting_prompt: str = "test starting prompt",
|
||||
user_persona: UserPersona = None,
|
||||
) -> ConversationScenario:
|
||||
"""Returns a ConversationScenario."""
|
||||
return ConversationScenario(
|
||||
starting_prompt=starting_prompt,
|
||||
conversation_plan=conversation_plan,
|
||||
user_persona=user_persona,
|
||||
)
|
||||
|
||||
|
||||
@@ -243,48 +295,28 @@ def _create_test_invocations(
|
||||
return invocations
|
||||
|
||||
|
||||
def test_format_llm_prompt():
|
||||
evaluator = _create_test_evaluator(stop_signal="test stop signal")
|
||||
def test_format_llm_prompt_raises_error_if_previous_invocations_is_none():
|
||||
evaluator = _create_test_evaluator()
|
||||
with pytest.raises(
|
||||
ValueError, match="Previous invocations should have a set value"
|
||||
):
|
||||
evaluator._format_llm_prompt(
|
||||
invocation=_create_test_invocation("1"),
|
||||
conversation_scenario=_create_test_conversation_scenario(),
|
||||
previous_invocations=None,
|
||||
)
|
||||
|
||||
starting_prompt = "first user prompt."
|
||||
conversation_scenario = _create_test_conversation_scenario(
|
||||
conversation_plan="test conversation plan.",
|
||||
starting_prompt=starting_prompt,
|
||||
)
|
||||
invocation_history = _create_test_invocations([
|
||||
starting_prompt,
|
||||
"first agent response.",
|
||||
"second user prompt.",
|
||||
"second agent response.",
|
||||
"third user prompt.",
|
||||
"third agent response.",
|
||||
])
|
||||
|
||||
prompt = evaluator._format_llm_prompt(
|
||||
invocation=invocation_history[-1],
|
||||
conversation_scenario=conversation_scenario,
|
||||
previous_invocations=invocation_history[:-1],
|
||||
)
|
||||
|
||||
assert (
|
||||
prompt == """This is a test template with stop signal: `test stop signal`.
|
||||
|
||||
# Conversation Plan
|
||||
test conversation plan.
|
||||
|
||||
# Conversation History
|
||||
user: first user prompt.
|
||||
|
||||
model: first agent response.
|
||||
|
||||
user: second user prompt.
|
||||
|
||||
model: second agent response.
|
||||
|
||||
# Generated User Response
|
||||
third user prompt.
|
||||
""".strip()
|
||||
)
|
||||
def test_format_llm_prompt_raises_error_if_conversation_scenario_is_none():
|
||||
evaluator = _create_test_evaluator()
|
||||
with pytest.raises(
|
||||
ValueError, match="Conversation scenario should have a set value"
|
||||
):
|
||||
evaluator._format_llm_prompt(
|
||||
invocation=_create_test_invocation("1"),
|
||||
conversation_scenario=None,
|
||||
previous_invocations=[],
|
||||
)
|
||||
|
||||
|
||||
def test_convert_llm_response_to_score_pass():
|
||||
@@ -419,6 +451,19 @@ def test_aggregate_samples_failure():
|
||||
assert aggregation_result.eval_status == EvalStatus.FAILED
|
||||
|
||||
|
||||
def test_format_conversation_history_with_none_values():
|
||||
"""Tests that _format_conversation_history handles None values."""
|
||||
invocations = [
|
||||
Invocation(
|
||||
invocation_id="1",
|
||||
user_content=types.Content(),
|
||||
final_response=None,
|
||||
)
|
||||
]
|
||||
formatted_history = _format_conversation_history(invocations)
|
||||
assert formatted_history == ""
|
||||
|
||||
|
||||
def test_format_conversation_history():
|
||||
conversation_history = [
|
||||
"first user prompt.",
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from google.adk.evaluation.simulation import pre_built_personas
|
||||
|
||||
|
||||
def test_get_default_persona_registry():
|
||||
"""Tests that the default persona registry can be loaded."""
|
||||
assert pre_built_personas.DEFAULT_USER_PERSONA_REGISTRY is not None
|
||||
@@ -0,0 +1,133 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from google.adk.errors.not_found_error import NotFoundError
|
||||
from google.adk.evaluation.simulation.user_simulator_personas import UserBehavior
|
||||
from google.adk.evaluation.simulation.user_simulator_personas import UserPersona
|
||||
from google.adk.evaluation.simulation.user_simulator_personas import UserPersonaRegistry
|
||||
import pytest
|
||||
|
||||
|
||||
class TestUserBehavior:
|
||||
"""Test cases for UserBehavior."""
|
||||
|
||||
def test_create_user_behavior(self):
|
||||
"""Tests UserBehavior creation."""
|
||||
behavior = UserBehavior(
|
||||
name="test_behavior",
|
||||
description="Test behavior description.",
|
||||
behavior_instructions=["instruction1", "instruction2"],
|
||||
violation_rubrics=["violation1", "violation2"],
|
||||
)
|
||||
assert behavior.name == "test_behavior"
|
||||
assert behavior.description == "Test behavior description."
|
||||
assert behavior.behavior_instructions == ["instruction1", "instruction2"]
|
||||
assert behavior.violation_rubrics == ["violation1", "violation2"]
|
||||
|
||||
def test_get_behavior_instructions_str(self):
|
||||
"""Tests get_behavior_instructions_str method."""
|
||||
behavior = UserBehavior(
|
||||
name="test_behavior",
|
||||
description="Test behavior description.",
|
||||
behavior_instructions=["instruction1", "instruction2"],
|
||||
violation_rubrics=[],
|
||||
)
|
||||
assert (
|
||||
behavior.get_behavior_instructions_str()
|
||||
== " * instruction1\n * instruction2"
|
||||
)
|
||||
|
||||
def test_get_violation_rubrics_str(self):
|
||||
"""Tests get_violation_rubrics_str method."""
|
||||
behavior = UserBehavior(
|
||||
name="test_behavior",
|
||||
description="Test behavior description.",
|
||||
behavior_instructions=[],
|
||||
violation_rubrics=["violation1", "violation2"],
|
||||
)
|
||||
assert (
|
||||
behavior.get_violation_rubrics_str() == " * violation1\n * violation2"
|
||||
)
|
||||
|
||||
|
||||
class TestUserPersona:
|
||||
"""Test cases for UserPersona."""
|
||||
|
||||
def test_create_user_persona(self):
|
||||
"""Tests UserPersona creation."""
|
||||
behavior = UserBehavior(
|
||||
name="test_behavior",
|
||||
description="Test behavior description.",
|
||||
behavior_instructions=["instruction1"],
|
||||
violation_rubrics=["violation1"],
|
||||
)
|
||||
persona = UserPersona(
|
||||
id="test_persona",
|
||||
description="Test persona description.",
|
||||
behaviors=[behavior],
|
||||
)
|
||||
assert persona.id == "test_persona"
|
||||
assert persona.description == "Test persona description."
|
||||
assert persona.behaviors == [behavior]
|
||||
|
||||
|
||||
class TestUserPersonaRegistry:
|
||||
"""Test cases for UserPersonaRegistry."""
|
||||
|
||||
def test_register_and_get_persona(self):
|
||||
"""Tests register_persona and get_persona methods."""
|
||||
registry = UserPersonaRegistry()
|
||||
persona = UserPersona(
|
||||
id="test_persona", description="Test persona", behaviors=[]
|
||||
)
|
||||
registry.register_persona("persona1", persona)
|
||||
assert registry.get_persona("persona1") == persona
|
||||
|
||||
def test_get_persona_not_found(self):
|
||||
"""Tests get_persona for a non-existent persona."""
|
||||
registry = UserPersonaRegistry()
|
||||
with pytest.raises(NotFoundError, match="persona2 not found in registry."):
|
||||
registry.get_persona("persona2")
|
||||
|
||||
def test_update_persona(self):
|
||||
"""Tests updating an existing persona in the registry."""
|
||||
registry = UserPersonaRegistry()
|
||||
persona1 = UserPersona(
|
||||
id="test_persona1", description="Test persona 1", behaviors=[]
|
||||
)
|
||||
persona2 = UserPersona(
|
||||
id="test_persona2", description="Test persona 2", behaviors=[]
|
||||
)
|
||||
registry.register_persona("persona1", persona1)
|
||||
assert registry.get_persona("persona1") == persona1
|
||||
registry.register_persona("persona1", persona2)
|
||||
assert registry.get_persona("persona1") == persona2
|
||||
|
||||
def test_get_registered_personas(self):
|
||||
"""Tests get_registered_personas method."""
|
||||
registry = UserPersonaRegistry()
|
||||
persona1 = UserPersona(
|
||||
id="test_persona1", description="Test persona 1", behaviors=[]
|
||||
)
|
||||
persona2 = UserPersona(
|
||||
id="test_persona2", description="Test persona 2", behaviors=[]
|
||||
)
|
||||
registry.register_persona("persona1", persona1)
|
||||
registry.register_persona("persona2", persona2)
|
||||
registered_personas = registry.get_registered_personas()
|
||||
assert len(registered_personas) == 2
|
||||
assert persona1 in registered_personas
|
||||
assert persona2 in registered_personas
|
||||
Reference in New Issue
Block a user