chore: Move simulation related modules to a sub-package in evaluation

Co-authored-by: Keyur Joshi <keyurj@google.com>
PiperOrigin-RevId: 838904075

tests/unittests/evaluation/simulation/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/unittests/evaluation/simulation/test_llm_backed_user_simulator.py

@@ -0,0 +1,249 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.evaluation import conversation_scenarios
+from google.adk.evaluation.simulation.llm_backed_user_simulator import LlmBackedUserSimulator
+from google.adk.evaluation.simulation.llm_backed_user_simulator import LlmBackedUserSimulatorConfig
+from google.adk.evaluation.simulation.user_simulator import Status
+from google.adk.events.event import Event
+from google.genai import types
+import pytest
+
+_INPUT_EVENTS = [
+    Event(
+        author="user",
+        content=types.Content(
+            parts=[types.Part(text="Can you help me?")], role="user"
+        ),
+        invocation_id="inv1",
+    ),
+    Event(
+        author="helpful_assistant",
+        content=types.Content(
+            parts=[
+                types.Part(
+                    text="I'll get the user's name and greet them first.",
+                    thought=True,
+                ),
+                types.Part(
+                    function_call=types.FunctionCall(name="get_user_name")
+                ),
+                types.Part(
+                    function_response=types.FunctionResponse(
+                        name="get_user_name",
+                        response={"name": "John Doe"},
+                    )
+                ),
+                types.Part(text="Hi John, what can I do for you?"),
+            ],
+            role="model",
+        ),
+        invocation_id="inv1",
+    ),
+]
+
+_INPUT_EVENTS_LONG = _INPUT_EVENTS + [
+    Event(
+        author="user",
+        content=types.Content(
+            parts=[types.Part(text="I need to book a flight.")], role="user"
+        ),
+        invocation_id="inv2",
+    ),
+    Event(
+        author="helpful_assistant",
+        content=types.Content(
+            parts=[
+                types.Part(
+                    text="Sure, what is your departure date and destination?",
+                ),
+            ],
+            role="model",
+        ),
+        invocation_id="inv2",
+    ),
+]
+
+_EXPECTED_REWRITTEN_DIALOGUE = """user: Can you help me?
+
+helpful_assistant: Hi John, what can I do for you?"""
+
+_EXPECTED_REWRITTEN_DIALOGUE_LONG = _EXPECTED_REWRITTEN_DIALOGUE + """
+
+user: I need to book a flight.
+
+helpful_assistant: Sure, what is your departure date and destination?"""
+
+
+class TestHelperMethods:
+  """Test cases for LlmBackedUserSimulator helper methods."""
+
+  def test_convert_conversation_to_user_sim_pov(self):
+    """Tests _convert_conversation_to_user_sim_pov method."""
+    rewritten_dialogue = LlmBackedUserSimulator._summarize_conversation(
+        _INPUT_EVENTS
+    )
+    assert rewritten_dialogue == _EXPECTED_REWRITTEN_DIALOGUE
+    rewritten_dialogue = LlmBackedUserSimulator._summarize_conversation(
+        _INPUT_EVENTS_LONG
+    )
+    assert rewritten_dialogue == _EXPECTED_REWRITTEN_DIALOGUE_LONG
+
+
+async def to_async_iter(items):
+  for item in items:
+    yield item
+
+
+@pytest.fixture
+def mock_llm_agent(mocker):
+  """Provides a mock LLM agent."""
+  mock_llm_registry_cls = mocker.patch(
+      "google.adk.evaluation.simulation.llm_backed_user_simulator.LLMRegistry"
+  )
+  mock_llm_registry = mocker.MagicMock()
+  mock_llm_registry_cls.return_value = mock_llm_registry
+  mock_agent = mocker.MagicMock()
+  mock_llm_registry.resolve.return_value.return_value = mock_agent
+  return mock_agent
+
+
+@pytest.fixture
+def conversation_scenario():
+  """Provides a test conversation scenario."""
+  return conversation_scenarios.ConversationScenario(
+      starting_prompt="Hello", conversation_plan="test plan"
+  )
+
+
+@pytest.fixture
+def simulator(mock_llm_agent, conversation_scenario):
+  """Provides an LlmBackedUserSimulator instance for testing."""
+  config = LlmBackedUserSimulatorConfig(
+      model="test-model",
+      model_configuration=types.GenerateContentConfig(),
+  )
+  sim = LlmBackedUserSimulator(
+      config=config, conversation_scenario=conversation_scenario
+  )
+  sim._invocation_count = 1  # Bypass starting prompt by default for tests
+  return sim
+
+
+class TestLlmBackedUserSimulator:
+  """Test cases for LlmBackedUserSimulator main methods."""
+
+  @pytest.mark.asyncio
+  async def test_get_llm_response_return_value(
+      self, simulator, mock_llm_agent, mocker
+  ):
+    """Tests that _get_llm_response returns the full response correctly."""
+    mock_llm_response = mocker.MagicMock()
+    mock_llm_response.content = types.Content(
+        parts=[
+            types.Part(text="some thought", thought=True),
+            types.Part(text="Hello world!"),
+        ]
+    )
+    mock_llm_agent.generate_content_async.return_value = to_async_iter(
+        [mock_llm_response]
+    )
+    response = await simulator._get_llm_response(rewritten_dialogue="")
+    assert response == "Hello world!"
+
+  @pytest.mark.asyncio
+  async def test_get_next_user_message_first_invocation(
+      self, simulator, mock_llm_agent, conversation_scenario
+  ):
+    """Tests that the first invocation returns the starting prompt."""
+    simulator._invocation_count = 0  # override testing default
+    next_user_message = await simulator.get_next_user_message(events=[])
+
+    expected_user_message = types.Content(
+        parts=[types.Part(text=conversation_scenario.starting_prompt)],
+        role="user",
+    )
+    assert next_user_message.status == Status.SUCCESS
+    assert next_user_message.user_message == expected_user_message
+    mock_llm_agent.generate_content_async.assert_not_called()
+
+  @pytest.mark.asyncio
+  async def test_turn_limit_reached(self, conversation_scenario):
+    """Tests get_next_user_message when the turn limit is reached."""
+    config = LlmBackedUserSimulatorConfig(
+        max_allowed_invocations=1,
+    )
+    simulator = LlmBackedUserSimulator(
+        config=config, conversation_scenario=conversation_scenario
+    )
+    simulator._invocation_count = 1
+
+    next_user_message = await simulator.get_next_user_message(
+        events=_INPUT_EVENTS
+    )
+
+    assert next_user_message.status == Status.TURN_LIMIT_REACHED
+    assert next_user_message.user_message is None
+
+  @pytest.mark.asyncio
+  async def test_stop_signal_detected(self, simulator, mock_llm_agent, mocker):
+    """Tests get_next_user_message when the stop signal is detected."""
+    mock_llm_response = mocker.MagicMock()
+    mock_llm_response.content = types.Content(
+        parts=[types.Part(text="Thanks! Bye!</finished>")]
+    )
+    mock_llm_agent.generate_content_async.return_value = to_async_iter(
+        [mock_llm_response]
+    )
+
+    next_user_message = await simulator.get_next_user_message(
+        events=_INPUT_EVENTS
+    )
+
+    assert next_user_message.status == Status.STOP_SIGNAL_DETECTED
+    assert next_user_message.user_message is None
+
+  @pytest.mark.asyncio
+  async def test_no_message_generated(self, simulator, mock_llm_agent):
+    """Tests get_next_user_message when no message is generated."""
+    mock_llm_agent.generate_content_async.return_value = to_async_iter([])
+
+    with pytest.raises(RuntimeError, match="Failed to generate a user message"):
+      await simulator.get_next_user_message(events=_INPUT_EVENTS)
+
+  @pytest.mark.asyncio
+  async def test_get_next_user_message_success(
+      self, simulator, mock_llm_agent, mocker
+  ):
+    """Tests get_next_user_message when the user message is generated successfully."""
+    mock_llm_response = mocker.MagicMock()
+    mock_llm_response.content = types.Content(
+        parts=[types.Part(text="I need to book a flight.")]
+    )
+    mock_llm_agent.generate_content_async.return_value = to_async_iter(
+        [mock_llm_response]
+    )
+
+    next_user_message = await simulator.get_next_user_message(
+        events=_INPUT_EVENTS
+    )
+
+    expected_user_message = types.Content(
+        parts=[types.Part(text="I need to book a flight.")], role="user"
+    )
+
+    assert next_user_message.status == Status.SUCCESS
+    assert next_user_message.user_message == expected_user_message

tests/unittests/evaluation/simulation/test_static_user_simulator.py

@@ -0,0 +1,54 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.simulation import static_user_simulator
+from google.adk.evaluation.simulation import user_simulator
+from google.genai import types
+import pytest
+
+
+class TestStaticUserSimulator:
+  """Test cases for StaticUserSimulator."""
+
+  @pytest.mark.asyncio
+  async def test_get_next_user_message(self):
+    """Tests that the provided messages are returned in order followed by the stop signal."""
+    conversation = [
+        Invocation(
+            invocation_id="inv1",
+            user_content=types.Content(parts=[types.Part(text="message 1")]),
+        ),
+        Invocation(
+            invocation_id="inv2",
+            user_content=types.Content(parts=[types.Part(text="message 2")]),
+        ),
+    ]
+    simulator = static_user_simulator.StaticUserSimulator(
+        static_conversation=conversation
+    )
+
+    next_message_1 = await simulator.get_next_user_message(events=[])
+    assert user_simulator.Status.SUCCESS == next_message_1.status
+    assert "message 1" == next_message_1.user_message.parts[0].text
+
+    next_message_2 = await simulator.get_next_user_message(events=[])
+    assert user_simulator.Status.SUCCESS == next_message_2.status
+    assert "message 2" == next_message_2.user_message.parts[0].text
+
+    next_message_3 = await simulator.get_next_user_message(events=[])
+    assert user_simulator.Status.STOP_SIGNAL_DETECTED == next_message_3.status
+    assert next_message_3.user_message is None

tests/unittests/evaluation/simulation/test_user_simulator.py

@@ -0,0 +1,45 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.evaluation.simulation.user_simulator import NextUserMessage
+from google.adk.evaluation.simulation.user_simulator import Status
+from google.genai.types import Content
+import pytest
+
+
+def test_next_user_message_validation():
+  """Tests post-init validation of NextUserMessage."""
+  with pytest.raises(
+      ValueError,
+      match=(
+          "A user_message should be provided if and only if the status is"
+          " SUCCESS"
+      ),
+  ):
+    NextUserMessage(status=Status.SUCCESS)
+
+  with pytest.raises(
+      ValueError,
+      match=(
+          "A user_message should be provided if and only if the status is"
+          " SUCCESS"
+      ),
+  ):
+    NextUserMessage(status=Status.TURN_LIMIT_REACHED, user_message=Content())
+
+  # these two should not cause exceptions
+  NextUserMessage(status=Status.SUCCESS, user_message=Content())
+  NextUserMessage(status=Status.TURN_LIMIT_REACHED)

tests/unittests/evaluation/simulation/test_user_simulator_provider.py

@@ -0,0 +1,79 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.evaluation import conversation_scenarios
+from google.adk.evaluation import eval_case
+from google.adk.evaluation.simulation import user_simulator_provider
+from google.adk.evaluation.simulation.llm_backed_user_simulator import LlmBackedUserSimulator
+from google.adk.evaluation.simulation.llm_backed_user_simulator import LlmBackedUserSimulatorConfig
+from google.adk.evaluation.simulation.static_user_simulator import StaticUserSimulator
+from google.genai import types
+import pytest
+
+_TEST_CONVERSATION = [
+    eval_case.Invocation(
+        invocation_id='inv1',
+        user_content=types.Content(parts=[types.Part(text='Hello!')]),
+    ),
+]
+
+_TEST_CONVERSATION_SCENARIO = conversation_scenarios.ConversationScenario(
+    starting_prompt='Hello!', conversation_plan='test plan'
+)
+
+
+class TestUserSimulatorProvider:
+  """Test cases for the UserSimulatorProvider."""
+
+  def test_provide_static_user_simulator(self):
+    """Tests the case when a StaticUserSimulator should be provided."""
+    provider = user_simulator_provider.UserSimulatorProvider()
+    test_eval_case = eval_case.EvalCase(
+        eval_id='test_eval_id',
+        conversation=_TEST_CONVERSATION,
+    )
+    simulator = provider.provide(test_eval_case)
+    assert isinstance(simulator, StaticUserSimulator)
+    assert simulator.static_conversation == _TEST_CONVERSATION
+
+  def test_provide_llm_backed_user_simulator(self, mocker):
+    """Tests the case when a LlmBackedUserSimulator should be provided."""
+    mock_llm_registry = mocker.patch(
+        'google.adk.evaluation.simulation.llm_backed_user_simulator.LLMRegistry',
+        autospec=True,
+    )
+    mock_llm_registry.return_value.resolve.return_value = mocker.Mock()
+    # Test case 1: No config in provider.
+    provider = user_simulator_provider.UserSimulatorProvider()
+    test_eval_case = eval_case.EvalCase(
+        eval_id='test_eval_id',
+        conversation_scenario=_TEST_CONVERSATION_SCENARIO,
+    )
+    simulator = provider.provide(test_eval_case)
+    assert isinstance(simulator, LlmBackedUserSimulator)
+    assert simulator._conversation_scenario == _TEST_CONVERSATION_SCENARIO
+
+    # Test case 2: Config in provider.
+    llm_config = LlmBackedUserSimulatorConfig(
+        model='test_model',
+    )
+    provider = user_simulator_provider.UserSimulatorProvider(
+        user_simulator_config=llm_config
+    )
+    simulator = provider.provide(test_eval_case)
+    assert isinstance(simulator, LlmBackedUserSimulator)
+    assert simulator._conversation_scenario == _TEST_CONVERSATION_SCENARIO
+    assert simulator._config.model == 'test_model'