mirror of
https://github.com/encounter/adk-python.git
synced 2026-07-09 18:19:28 -07:00
feat: Adds Static User Simulator and User Simulator Provider
Details: - Adds the `StaticUserSimulator` which implements the current functionality of supplying a fixed set of user prompts for an EvalCase. - Adds the `UserSimulatorProvider` which determines the type of user simulator required for an EvalCase (StaticUserSimulator or LlmBackedUserSimulator). - Integrates the UserSimulatorProvider and UserSimulator into the CLI and evaluation infrastructure. - Updates and adds unit tests for the new functionality. - Miscellaneous updates to lay groundwork for a full implementation of the LlmBackedUserSimulator in the future. PiperOrigin-RevId: 822198401
This commit is contained in:
Google Team Member
committed by
Copybara-Service
Copybara-Service
parent
4a842c5a13
commit
aeaec859bf
@@ -14,6 +14,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from google.adk.evaluation.conversation_scenarios import ConversationScenario
|
||||
from google.adk.evaluation.eval_case import EvalCase
|
||||
from google.adk.evaluation.eval_case import get_all_tool_calls
|
||||
from google.adk.evaluation.eval_case import get_all_tool_calls_with_responses
|
||||
from google.adk.evaluation.eval_case import get_all_tool_responses
|
||||
@@ -246,3 +248,36 @@ def test_get_all_tool_calls_with_responses_with_invocation_events():
|
||||
(tool_call1, tool_response1),
|
||||
(tool_call2, None),
|
||||
]
|
||||
|
||||
|
||||
def test_conversation_and_conversation_scenario_mutual_exclusion():
|
||||
"""Tests the ensure_conversation_xor_conversation_scenario validator."""
|
||||
test_conversation_scenario = ConversationScenario(
|
||||
starting_prompt='', conversation_plan=''
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=(
|
||||
'Exactly one of conversation and conversation_scenario must be'
|
||||
' provided in an EvalCase.'
|
||||
),
|
||||
):
|
||||
EvalCase(eval_id='test_id')
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=(
|
||||
'Exactly one of conversation and conversation_scenario must be'
|
||||
' provided in an EvalCase.'
|
||||
),
|
||||
):
|
||||
EvalCase(
|
||||
eval_id='test_id',
|
||||
conversation=[],
|
||||
conversation_scenario=test_conversation_scenario,
|
||||
)
|
||||
|
||||
# these two should not cause exceptions
|
||||
EvalCase(eval_id='test_id', conversation=[])
|
||||
EvalCase(eval_id='test_id', conversation_scenario=test_conversation_scenario)
|
||||
|
||||
@@ -18,9 +18,13 @@ from google.adk.evaluation.app_details import AgentDetails
|
||||
from google.adk.evaluation.app_details import AppDetails
|
||||
from google.adk.evaluation.evaluation_generator import EvaluationGenerator
|
||||
from google.adk.evaluation.request_intercepter_plugin import _RequestIntercepterPlugin
|
||||
from google.adk.evaluation.user_simulator import NextUserMessage
|
||||
from google.adk.evaluation.user_simulator import Status as UserSimulatorStatus
|
||||
from google.adk.evaluation.user_simulator import UserSimulator
|
||||
from google.adk.events.event import Event
|
||||
from google.adk.models.llm_request import LlmRequest
|
||||
from google.genai import types
|
||||
import pytest
|
||||
|
||||
|
||||
def _build_event(
|
||||
@@ -324,3 +328,130 @@ class TestGetAppDetailsByInvocationId:
|
||||
}
|
||||
assert app_details == expected_app_details
|
||||
assert mock_request_intercepter.get_model_request.call_count == 3
|
||||
|
||||
|
||||
class TestGenerateInferencesForSingleUserInvocation:
|
||||
"""Test cases for EvaluationGenerator._generate_inferences_for_single_user_invocation method."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_inferences_with_mock_runner(self, mocker):
|
||||
"""Tests inference generation with a mocked runner."""
|
||||
runner = mocker.MagicMock()
|
||||
|
||||
agent_parts = [types.Part(text="Agent response")]
|
||||
|
||||
async def mock_run_async(*args, **kwargs):
|
||||
yield _build_event(
|
||||
author="agent",
|
||||
parts=agent_parts,
|
||||
invocation_id="inv1",
|
||||
)
|
||||
|
||||
runner.run_async.return_value = mock_run_async()
|
||||
|
||||
user_content = types.Content(parts=[types.Part(text="User query")])
|
||||
|
||||
events = [
|
||||
event
|
||||
async for event in EvaluationGenerator._generate_inferences_for_single_user_invocation(
|
||||
runner, "test_user", "test_session", user_content
|
||||
)
|
||||
]
|
||||
|
||||
assert len(events) == 2
|
||||
assert events[0].author == "user"
|
||||
assert events[0].content == user_content
|
||||
assert events[0].invocation_id == "inv1"
|
||||
assert events[1].author == "agent"
|
||||
assert events[1].content.parts == agent_parts
|
||||
|
||||
runner.run_async.assert_called_once_with(
|
||||
user_id="test_user",
|
||||
session_id="test_session",
|
||||
new_message=user_content,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_runner(mocker):
|
||||
"""Provides a mock Runner for testing."""
|
||||
mock_runner_cls = mocker.patch(
|
||||
"google.adk.evaluation.evaluation_generator.Runner"
|
||||
)
|
||||
mock_runner_instance = mocker.AsyncMock()
|
||||
mock_runner_instance.__aenter__.return_value = mock_runner_instance
|
||||
mock_runner_cls.return_value = mock_runner_instance
|
||||
yield mock_runner_instance
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_session_service(mocker):
|
||||
"""Provides a mock InMemorySessionService for testing."""
|
||||
mock_session_service_cls = mocker.patch(
|
||||
"google.adk.evaluation.evaluation_generator.InMemorySessionService"
|
||||
)
|
||||
mock_session_service_instance = mocker.MagicMock()
|
||||
mock_session_service_instance.create_session = mocker.AsyncMock()
|
||||
mock_session_service_cls.return_value = mock_session_service_instance
|
||||
yield mock_session_service_instance
|
||||
|
||||
|
||||
class TestGenerateInferencesFromRootAgent:
|
||||
"""Test cases for EvaluationGenerator._generate_inferences_from_root_agent method."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generates_inferences_with_user_simulator(
|
||||
self, mocker, mock_runner, mock_session_service
|
||||
):
|
||||
"""Tests that inferences are generated by interacting with a user simulator."""
|
||||
mock_agent = mocker.MagicMock()
|
||||
mock_user_sim = mocker.MagicMock(spec=UserSimulator)
|
||||
|
||||
# Mock user simulator will produce one message, then stop.
|
||||
async def get_next_user_message_side_effect(*args, **kwargs):
|
||||
if mock_user_sim.get_next_user_message.call_count == 1:
|
||||
return NextUserMessage(
|
||||
status=UserSimulatorStatus.SUCCESS,
|
||||
user_message=types.Content(parts=[types.Part(text="message 1")]),
|
||||
)
|
||||
return NextUserMessage(status=UserSimulatorStatus.STOP_SIGNAL_DETECTED)
|
||||
|
||||
mock_user_sim.get_next_user_message = mocker.AsyncMock(
|
||||
side_effect=get_next_user_message_side_effect
|
||||
)
|
||||
|
||||
mock_generate_inferences = mocker.patch(
|
||||
"google.adk.evaluation.evaluation_generator.EvaluationGenerator._generate_inferences_for_single_user_invocation"
|
||||
)
|
||||
mocker.patch(
|
||||
"google.adk.evaluation.evaluation_generator.EvaluationGenerator._get_app_details_by_invocation_id"
|
||||
)
|
||||
mocker.patch(
|
||||
"google.adk.evaluation.evaluation_generator.EvaluationGenerator.convert_events_to_eval_invocations"
|
||||
)
|
||||
|
||||
# Each call to _generate_inferences_for_single_user_invocation will
|
||||
# yield one user and one agent event.
|
||||
async def mock_generate_inferences_side_effect(
|
||||
runner, user_id, session_id, user_content
|
||||
):
|
||||
yield _build_event("user", user_content.parts, "inv1")
|
||||
yield _build_event("agent", [types.Part(text="agent_response")], "inv1")
|
||||
|
||||
mock_generate_inferences.side_effect = mock_generate_inferences_side_effect
|
||||
|
||||
await EvaluationGenerator._generate_inferences_from_root_agent(
|
||||
root_agent=mock_agent,
|
||||
user_simulator=mock_user_sim,
|
||||
)
|
||||
|
||||
# Check that user simulator was called until it stopped.
|
||||
assert mock_user_sim.get_next_user_message.call_count == 2
|
||||
|
||||
# Check that we generated inferences for each user message.
|
||||
assert mock_generate_inferences.call_count == 1
|
||||
|
||||
# Check the content of the user messages passed to inference generation
|
||||
mock_generate_inferences.assert_called_once()
|
||||
called_with_content = mock_generate_inferences.call_args.args[3]
|
||||
assert called_with_content.parts[0].text == "message 1"
|
||||
|
||||
@@ -12,6 +12,8 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
@@ -246,6 +248,7 @@ async def test_evaluate_success(
|
||||
|
||||
mock_eval_case = mocker.MagicMock(spec=EvalCase)
|
||||
mock_eval_case.conversation = []
|
||||
mock_eval_case.conversation_scenario = None
|
||||
mock_eval_case.session_input = None
|
||||
mock_eval_sets_manager.get_eval_case.return_value = mock_eval_case
|
||||
|
||||
@@ -321,6 +324,7 @@ async def test_evaluate_single_inference_result(
|
||||
invocation.model_copy(deep=True),
|
||||
invocation.model_copy(deep=True),
|
||||
]
|
||||
mock_eval_case.conversation_scenario = None
|
||||
mock_eval_case.session_input = None
|
||||
mock_eval_sets_manager.get_eval_case.return_value = mock_eval_case
|
||||
|
||||
@@ -352,6 +356,51 @@ async def test_evaluate_single_inference_result(
|
||||
assert metric_result.eval_status == EvalStatus.PASSED
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_evaluate_single_inference_result_skipped_for_conversation_scenario(
|
||||
eval_service, mock_eval_sets_manager, mocker
|
||||
):
|
||||
"""To be removed once evaluation is implemented for conversation scenarios."""
|
||||
invocation = Invocation(
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="test user content.")]
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[genai_types.Part(text="test final response.")]
|
||||
),
|
||||
)
|
||||
inference_result = InferenceResult(
|
||||
app_name="test_app",
|
||||
eval_set_id="test_eval_set",
|
||||
eval_case_id="case1",
|
||||
inferences=[invocation.model_copy(deep=True)],
|
||||
session_id="session1",
|
||||
)
|
||||
eval_metric = EvalMetric(metric_name="fake_metric", threshold=0.5)
|
||||
evaluate_config = EvaluateConfig(eval_metrics=[eval_metric], parallelism=1)
|
||||
|
||||
mock_eval_case = mocker.MagicMock(spec=EvalCase)
|
||||
mock_eval_case.conversation = None
|
||||
mock_eval_case.conversation_scenario = mocker.MagicMock()
|
||||
mock_eval_case.session_input = None
|
||||
mock_eval_sets_manager.get_eval_case.return_value = mock_eval_case
|
||||
|
||||
_, result = await eval_service._evaluate_single_inference_result(
|
||||
inference_result=inference_result, evaluate_config=evaluate_config
|
||||
)
|
||||
assert isinstance(result, EvalCaseResult)
|
||||
assert result.eval_id == "case1"
|
||||
assert result.final_eval_status == EvalStatus.NOT_EVALUATED
|
||||
assert not result.overall_eval_metric_results
|
||||
assert len(result.eval_metric_result_per_invocation) == 1
|
||||
invocation_result = result.eval_metric_result_per_invocation[0]
|
||||
assert not invocation_result.eval_metric_results
|
||||
assert (
|
||||
invocation_result.expected_invocation.final_response.parts[0].text
|
||||
== "N/A"
|
||||
)
|
||||
|
||||
|
||||
def test_generate_final_eval_status_doesn_t_throw_on(eval_service):
|
||||
# How to fix if this test case fails?
|
||||
# This test case has failed mainly because a new EvalStatus got added. You
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from google.adk.evaluation import static_user_simulator
|
||||
from google.adk.evaluation import user_simulator
|
||||
from google.adk.evaluation.eval_case import Invocation
|
||||
from google.genai import types
|
||||
import pytest
|
||||
|
||||
|
||||
class TestStaticUserSimulator:
|
||||
"""Test cases for StaticUserSimulator."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_next_user_message(self):
|
||||
"""Tests that the provided messages are returned in order followed by the stop signal."""
|
||||
conversation = [
|
||||
Invocation(
|
||||
invocation_id="inv1",
|
||||
user_content=types.Content(parts=[types.Part(text="message 1")]),
|
||||
),
|
||||
Invocation(
|
||||
invocation_id="inv2",
|
||||
user_content=types.Content(parts=[types.Part(text="message 2")]),
|
||||
),
|
||||
]
|
||||
simulator = static_user_simulator.StaticUserSimulator(
|
||||
static_conversation=conversation
|
||||
)
|
||||
|
||||
next_message_1 = await simulator.get_next_user_message(events=[])
|
||||
assert user_simulator.Status.SUCCESS == next_message_1.status
|
||||
assert "message 1" == next_message_1.user_message.parts[0].text
|
||||
|
||||
next_message_2 = await simulator.get_next_user_message(events=[])
|
||||
assert user_simulator.Status.SUCCESS == next_message_2.status
|
||||
assert "message 2" == next_message_2.user_message.parts[0].text
|
||||
|
||||
next_message_3 = await simulator.get_next_user_message(events=[])
|
||||
assert user_simulator.Status.STOP_SIGNAL_DETECTED == next_message_3.status
|
||||
assert next_message_3.user_message is None
|
||||
@@ -0,0 +1,45 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from google.adk.evaluation.user_simulator import NextUserMessage
|
||||
from google.adk.evaluation.user_simulator import Status
|
||||
from google.genai.types import Content
|
||||
import pytest
|
||||
|
||||
|
||||
def test_next_user_message_validation():
|
||||
"""Tests post-init validation of NextUserMessage."""
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=(
|
||||
"A user_message should be provided if and only if the status is"
|
||||
" SUCCESS"
|
||||
),
|
||||
):
|
||||
NextUserMessage(status=Status.SUCCESS)
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=(
|
||||
"A user_message should be provided if and only if the status is"
|
||||
" SUCCESS"
|
||||
),
|
||||
):
|
||||
NextUserMessage(status=Status.TURN_LIMIT_REACHED, user_message=Content())
|
||||
|
||||
# these two should not cause exceptions
|
||||
NextUserMessage(status=Status.SUCCESS, user_message=Content())
|
||||
NextUserMessage(status=Status.TURN_LIMIT_REACHED)
|
||||
@@ -0,0 +1,79 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from google.adk.evaluation import conversation_scenarios
|
||||
from google.adk.evaluation import eval_case
|
||||
from google.adk.evaluation import user_simulator_provider
|
||||
from google.adk.evaluation.llm_backed_user_simulator import LlmBackedUserSimulator
|
||||
from google.adk.evaluation.llm_backed_user_simulator import LlmBackedUserSimulatorConfig
|
||||
from google.adk.evaluation.static_user_simulator import StaticUserSimulator
|
||||
from google.genai import types
|
||||
import pytest
|
||||
|
||||
_TEST_CONVERSATION = [
|
||||
eval_case.Invocation(
|
||||
invocation_id='inv1',
|
||||
user_content=types.Content(parts=[types.Part(text='Hello!')]),
|
||||
),
|
||||
]
|
||||
|
||||
_TEST_CONVERSATION_SCENARIO = conversation_scenarios.ConversationScenario(
|
||||
starting_prompt='Hello!', conversation_plan='test plan'
|
||||
)
|
||||
|
||||
|
||||
class TestUserSimulatorProvider:
|
||||
"""Test cases for the UserSimulatorProvider."""
|
||||
|
||||
def test_provide_static_user_simulator(self):
|
||||
"""Tests the case when a StaticUserSimulator should be provided."""
|
||||
provider = user_simulator_provider.UserSimulatorProvider()
|
||||
test_eval_case = eval_case.EvalCase(
|
||||
eval_id='test_eval_id',
|
||||
conversation=_TEST_CONVERSATION,
|
||||
)
|
||||
simulator = provider.provide(test_eval_case)
|
||||
assert isinstance(simulator, StaticUserSimulator)
|
||||
assert simulator.static_conversation == _TEST_CONVERSATION
|
||||
|
||||
def test_provide_llm_backed_user_simulator(self, mocker):
|
||||
"""Tests the case when a LlmBackedUserSimulator should be provided."""
|
||||
mock_llm_registry = mocker.patch(
|
||||
'google.adk.evaluation.llm_backed_user_simulator.LLMRegistry',
|
||||
autospec=True,
|
||||
)
|
||||
mock_llm_registry.return_value.resolve.return_value = mocker.Mock()
|
||||
# Test case 1: No config in provider.
|
||||
provider = user_simulator_provider.UserSimulatorProvider()
|
||||
test_eval_case = eval_case.EvalCase(
|
||||
eval_id='test_eval_id',
|
||||
conversation_scenario=_TEST_CONVERSATION_SCENARIO,
|
||||
)
|
||||
simulator = provider.provide(test_eval_case)
|
||||
assert isinstance(simulator, LlmBackedUserSimulator)
|
||||
assert simulator._conversation_scenario == _TEST_CONVERSATION_SCENARIO
|
||||
|
||||
# Test case 2: Config in provider.
|
||||
llm_config = LlmBackedUserSimulatorConfig(
|
||||
model='test_model',
|
||||
)
|
||||
provider = user_simulator_provider.UserSimulatorProvider(
|
||||
user_simulator_config=llm_config
|
||||
)
|
||||
simulator = provider.provide(test_eval_case)
|
||||
assert isinstance(simulator, LlmBackedUserSimulator)
|
||||
assert simulator._conversation_scenario == _TEST_CONVERSATION_SCENARIO
|
||||
assert simulator._config.model == 'test_model'
|
||||
Reference in New Issue
Block a user