diff --git a/src/google/adk/cli/utils/evals.py b/src/google/adk/cli/utils/evals.py index 305d4754..8b2a3f2f 100644 --- a/src/google/adk/cli/utils/evals.py +++ b/src/google/adk/cli/utils/evals.py @@ -19,14 +19,13 @@ import os from typing import Any from typing import Tuple -from google.genai import types as genai_types from pydantic import alias_generators from pydantic import BaseModel from pydantic import ConfigDict from typing_extensions import deprecated -from ...evaluation.eval_case import IntermediateData from ...evaluation.eval_case import Invocation +from ...evaluation.evaluation_generator import EvaluationGenerator from ...evaluation.gcs_eval_set_results_manager import GcsEvalSetResultsManager from ...evaluation.gcs_eval_sets_manager import GcsEvalSetsManager from ...sessions.session import Session @@ -130,71 +129,8 @@ def convert_session_to_eval_invocations(session: Session) -> list[Invocation]: Returns: list: A list of invocation. """ - invocations: list[Invocation] = [] events = session.events if session and session.events else [] - - for event in events: - if event.author == 'user': - if not event.content or not event.content.parts: - continue - - # The content present in this event is the user content. - user_content = event.content - invocation_id = event.invocation_id - invocaton_timestamp = event.timestamp - - # Find the corresponding tool usage or response for the query - tool_uses: list[genai_types.FunctionCall] = [] - intermediate_responses: list[Tuple[str, list[genai_types.Part]]] = [] - - # Check subsequent events to extract tool uses or responses for this turn. - for subsequent_event in events[events.index(event) + 1 :]: - event_author = subsequent_event.author or 'agent' - if event_author == 'user': - # We found an event where the author was the user. This means that a - # new turn has started. So close this turn here. - break - - if not subsequent_event.content or not subsequent_event.content.parts: - continue - - intermediate_response_parts = [] - for subsequent_part in subsequent_event.content.parts: - # Some events have both function call and reference - if subsequent_part.function_call: - tool_uses.append(subsequent_part.function_call) - elif subsequent_part.text: - # Also keep track of all the natural language responses that - # agent (or sub agents) generated. - intermediate_response_parts.append(subsequent_part) - - if intermediate_response_parts: - # Only add an entry if there any intermediate entries. - intermediate_responses.append( - (event_author, intermediate_response_parts) - ) - - # If we are here then either we are done reading all the events or we - # encountered an event that had content authored by the end-user. - # This, basically means an end of turn. - # We assume that the last natural language intermediate response is the - # final response from the agent/model. We treat that as a reference. - invocations.append( - Invocation( - user_content=user_content, - invocation_id=invocation_id, - creation_timestamp=invocaton_timestamp, - intermediate_data=IntermediateData( - tool_uses=tool_uses, - intermediate_responses=intermediate_responses[:-1], - ), - final_response=genai_types.Content( - parts=intermediate_responses[-1][1] - ), - ) - ) - - return invocations + return EvaluationGenerator.convert_events_to_eval_invocations(events) def create_gcs_eval_managers_from_uri( diff --git a/src/google/adk/evaluation/agent_evaluator.py b/src/google/adk/evaluation/agent_evaluator.py index 12431571..0114dfbb 100644 --- a/src/google/adk/evaluation/agent_evaluator.py +++ b/src/google/adk/evaluation/agent_evaluator.py @@ -34,7 +34,8 @@ from pydantic import ValidationError from ..agents.base_agent import BaseAgent from ..utils.context_utils import Aclosing from .constants import MISSING_EVAL_DEPENDENCIES_MESSAGE -from .eval_case import IntermediateData +from .eval_case import get_all_tool_calls +from .eval_case import IntermediateDataType from .eval_case import Invocation from .eval_metrics import EvalMetric from .eval_metrics import EvalMetricResult @@ -457,12 +458,11 @@ class AgentEvaluator: @staticmethod def _convert_tool_calls_to_text( - intermediate_data: Optional[IntermediateData], + intermediate_data: Optional[IntermediateDataType], ) -> str: - if intermediate_data and intermediate_data.tool_uses: - return "\n".join([str(t) for t in intermediate_data.tool_uses]) + tool_calls = get_all_tool_calls(intermediate_data) - return "" + return "\n".join([str(t) for t in tool_calls]) @staticmethod def _get_agent_for_eval( diff --git a/src/google/adk/evaluation/app_details.py b/src/google/adk/evaluation/app_details.py new file mode 100644 index 00000000..534dbf94 --- /dev/null +++ b/src/google/adk/evaluation/app_details.py @@ -0,0 +1,49 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from google.genai import types as genai_types +from pydantic import Field + +from .common import EvalBaseModel + + +class AgentDetails(EvalBaseModel): + """Details about the individual agent in the App. + + This could be a root agent or the sub-agents in the Agent Tree. + """ + + name: str + """The name of the Agent that uniquely identifies it in the App.""" + + instructions: str = Field(default="") + """The instructions set on the Agent.""" + + tool_declarations: genai_types.ToolListUnion = Field(default_factory=list) + """A list of tools available to the Agent.""" + + +class AppDetails(EvalBaseModel): + """Contains details about the App (the agentic system). + + This structure is only a projection of the acutal app. Only details + that are relevant to the Eval System are captured here. + """ + + agent_details: dict[str, AgentDetails] = Field( + default_factory=dict, + ) + """A mapping from the agent name to the details of that agent.""" diff --git a/src/google/adk/evaluation/common.py b/src/google/adk/evaluation/common.py index 3f349d57..2193fb62 100644 --- a/src/google/adk/evaluation/common.py +++ b/src/google/adk/evaluation/common.py @@ -23,4 +23,5 @@ class EvalBaseModel(pydantic.BaseModel): alias_generator=alias_generators.to_camel, populate_by_name=True, extra='forbid', + arbitrary_types_allowed=True, ) diff --git a/src/google/adk/evaluation/eval_case.py b/src/google/adk/evaluation/eval_case.py index 2229230c..df2c478e 100644 --- a/src/google/adk/evaluation/eval_case.py +++ b/src/google/adk/evaluation/eval_case.py @@ -16,23 +16,17 @@ from __future__ import annotations from typing import Any from typing import Optional +from typing import Union from google.genai import types as genai_types -from pydantic import alias_generators -from pydantic import BaseModel -from pydantic import ConfigDict from pydantic import Field +from typing_extensions import TypeAlias +from .app_details import AppDetails +from .common import EvalBaseModel from .eval_rubrics import Rubric -class EvalBaseModel(BaseModel): - model_config = ConfigDict( - alias_generator=alias_generators.to_camel, - populate_by_name=True, - ) - - class IntermediateData(EvalBaseModel): """Container for intermediate data that an agent would generate as it responds with a final answer.""" @@ -54,6 +48,33 @@ class IntermediateData(EvalBaseModel): """ +class InvocationEvent(EvalBaseModel): + """An immutable record representing a specific point in the agent's invocation. + + It captures agent's replies, requests to use tools (function calls), and tool + results. + + This structure is a simple projection of the actual `Event` datamodel that + is intended for the Eval System. + """ + + author: str + """The name of the agent that authored/owned this event.""" + + content: Optional[genai_types.Content] + """The content of the event.""" + + +class InvocationEvents(EvalBaseModel): + """A container for events that occur during the course of an invocation.""" + + invocation_events: list[InvocationEvent] = Field(default_factory=list) + """A list of invocation events.""" + + +IntermediateDataType: TypeAlias = Union[IntermediateData, InvocationEvents] + + class Invocation(EvalBaseModel): """Represents a single invocation.""" @@ -66,7 +87,7 @@ class Invocation(EvalBaseModel): final_response: Optional[genai_types.Content] = None """Final response from the agent.""" - intermediate_data: Optional[IntermediateData] = None + intermediate_data: Optional[IntermediateDataType] = None """Intermediate steps generated as a part of Agent execution. For a multi-agent system, it is also helpful to inspect the route that @@ -81,6 +102,9 @@ class Invocation(EvalBaseModel): ) """A list of rubrics that are applicable to only this invocation.""" + app_details: Optional[AppDetails] = Field(default=None) + """Details about the App that was used for this invocation.""" + class SessionInput(EvalBaseModel): """Values that help initialize a Session.""" @@ -117,3 +141,30 @@ class EvalCase(EvalBaseModel): default=None, ) """A list of rubrics that are applicable to all the invocations in the conversation of this eval case.""" + + +def get_all_tool_calls( + intermediate_data: Optional[IntermediateDataType], +) -> list[genai_types.FunctionCall]: + """A utility method to retrieve tools calls from intermediate data.""" + if not intermediate_data: + return [] + + tool_calls = [] + if isinstance(intermediate_data, IntermediateData): + tool_calls = intermediate_data.tool_uses + elif isinstance(intermediate_data, InvocationEvents): + # Go over each event in the list of events + for invocation_event in intermediate_data.invocation_events: + # Check if the event has content and some parts. + if invocation_event.content and invocation_event.content.parts: + for p in invocation_event.content.parts: + # For each part, we check if any of those part is a function call. + if p.function_call: + tool_calls.append(p.function_call) + else: + raise ValueError( + f"Unsupported type for intermediate_data `{intermediate_data}`" + ) + + return tool_calls diff --git a/src/google/adk/evaluation/evaluation_generator.py b/src/google/adk/evaluation/evaluation_generator.py index 85742f36..5b955fdb 100644 --- a/src/google/adk/evaluation/evaluation_generator.py +++ b/src/google/adk/evaluation/evaluation_generator.py @@ -24,6 +24,7 @@ from pydantic import BaseModel from ..agents.llm_agent import Agent from ..artifacts.base_artifact_service import BaseArtifactService from ..artifacts.in_memory_artifact_service import InMemoryArtifactService +from ..events.event import Event from ..memory.base_memory_service import BaseMemoryService from ..memory.in_memory_memory_service import InMemoryMemoryService from ..runners import Runner @@ -31,12 +32,17 @@ from ..sessions.base_session_service import BaseSessionService from ..sessions.in_memory_session_service import InMemorySessionService from ..sessions.session import Session from ..utils.context_utils import Aclosing +from .app_details import AppDetails from .eval_case import EvalCase -from .eval_case import IntermediateData from .eval_case import Invocation +from .eval_case import InvocationEvent +from .eval_case import InvocationEvents from .eval_case import SessionInput from .eval_set import EvalSet +_USER_AUTHOR = "user" +_DEFAULT_AUTHOR = "agent" + class EvalCaseResponses(BaseModel): """Contains multiple responses associated with an EvalCase. @@ -174,8 +180,6 @@ class EvaluationGenerator: if callable(reset_func): reset_func() - response_invocations = [] - async with Runner( app_name=app_name, agent=root_agent, @@ -183,42 +187,94 @@ class EvaluationGenerator: session_service=session_service, memory_service=memory_service, ) as runner: + events = [] + for invocation in invocations: - final_response = None user_content = invocation.user_content - tool_uses = [] - invocation_id = "" + invocation_id = None async with Aclosing( runner.run_async( user_id=user_id, session_id=session_id, new_message=user_content ) ) as agen: + async for event in agen: - invocation_id = ( - event.invocation_id if not invocation_id else invocation_id - ) + if not invocation_id: + invocation_id = event.invocation_id + events.append( + Event( + content=user_content, + author=_USER_AUTHOR, + invocation_id=invocation_id, + ) + ) - if ( - event.is_final_response() - and event.content - and event.content.parts - ): - final_response = event.content - elif event.get_function_calls(): - for call in event.get_function_calls(): - tool_uses.append(call) + events.append(event) - response_invocations.append( - Invocation( - invocation_id=invocation_id, - user_content=user_content, - final_response=final_response, - intermediate_data=IntermediateData(tool_uses=tool_uses), - ) - ) + return EvaluationGenerator.convert_events_to_eval_invocations(events) - return response_invocations + @staticmethod + def convert_events_to_eval_invocations( + events: list[Event], + ) -> list[Invocation]: + """Converts a list of events to eval invocations.""" + # Group Events by invocation id. Events that share the same invocation id + # belong to the same invocation. + events_by_invocation_id: dict[str, list[Event]] = {} + + for event in events: + invocation_id = event.invocation_id + + if invocation_id not in events_by_invocation_id: + events_by_invocation_id[invocation_id] = [] + + events_by_invocation_id[invocation_id].append(event) + + invocations = [] + for invocation_id, events in events_by_invocation_id.items(): + final_response = None + user_content = "" + invocation_timestamp = 0 + + events_to_add = [] + + for event in events: + current_author = (event.author or _DEFAULT_AUTHOR).lower() + + if current_author == _USER_AUTHOR: + # If the author is the user, then we just identify it and move on + # to the next event. + user_content = event.content + invocation_timestamp = event.timestamp + continue + + if event.content and event.content.parts: + if event.is_final_response(): + final_response = event.content + else: + for p in event.content.parts: + if p.function_call or p.function_response or p.text: + events_to_add.append(event) + break + + invocation_events = [ + InvocationEvent(author=e.author, content=e.content) + for e in events_to_add + ] + invocations.append( + Invocation( + invocation_id=invocation_id, + user_content=user_content, + final_response=final_response, + intermediate_data=InvocationEvents( + invocation_events=invocation_events + ), + creation_timestamp=invocation_timestamp, + ) + ) + + return invocations @staticmethod def _process_query_with_session(session_data, data): diff --git a/src/google/adk/evaluation/gcs_eval_sets_manager.py b/src/google/adk/evaluation/gcs_eval_sets_manager.py index b7b5b8bc..86039c17 100644 --- a/src/google/adk/evaluation/gcs_eval_sets_manager.py +++ b/src/google/adk/evaluation/gcs_eval_sets_manager.py @@ -84,7 +84,12 @@ class GcsEvalSetsManager(EvalSetsManager): """Writes an EvalSet to GCS.""" blob = self.bucket.blob(blob_name) blob.upload_from_string( - eval_set.model_dump_json(indent=2), + eval_set.model_dump_json( + indent=2, + exclude_unset=True, + exclude_defaults=True, + exclude_none=True, + ), content_type="application/json", ) diff --git a/src/google/adk/evaluation/local_eval_sets_manager.py b/src/google/adk/evaluation/local_eval_sets_manager.py index a68eb853..e42eb80e 100644 --- a/src/google/adk/evaluation/local_eval_sets_manager.py +++ b/src/google/adk/evaluation/local_eval_sets_manager.py @@ -325,7 +325,14 @@ class LocalEvalSetsManager(EvalSetsManager): def _write_eval_set_to_path(self, eval_set_path: str, eval_set: EvalSet): with open(eval_set_path, "w", encoding="utf-8") as f: - f.write(eval_set.model_dump_json(indent=2)) + f.write( + eval_set.model_dump_json( + indent=2, + exclude_unset=True, + exclude_defaults=True, + exclude_none=True, + ) + ) def _save_eval_set(self, app_name: str, eval_set_id: str, eval_set: EvalSet): eval_set_file_path = self._get_eval_set_file_path(app_name, eval_set_id) diff --git a/src/google/adk/evaluation/trajectory_evaluator.py b/src/google/adk/evaluation/trajectory_evaluator.py index 8f7508d4..9ea78a44 100644 --- a/src/google/adk/evaluation/trajectory_evaluator.py +++ b/src/google/adk/evaluation/trajectory_evaluator.py @@ -23,6 +23,7 @@ from tabulate import tabulate from typing_extensions import deprecated from typing_extensions import override +from .eval_case import get_all_tool_calls from .eval_case import Invocation from .eval_metrics import EvalMetric from .eval_metrics import Interval @@ -83,14 +84,9 @@ class TrajectoryEvaluator(Evaluator): per_invocation_results = [] for actual, expected in zip(actual_invocations, expected_invocations): - actual_tool_uses = ( - actual.intermediate_data.tool_uses if actual.intermediate_data else [] - ) - expected_tool_uses = ( - expected.intermediate_data.tool_uses - if expected.intermediate_data - else [] - ) + actual_tool_uses = get_all_tool_calls(actual.intermediate_data) + expected_tool_uses = get_all_tool_calls(expected.intermediate_data) + tool_use_accuracy = ( 1.0 if self._are_tool_calls_equal(actual_tool_uses, expected_tool_uses) diff --git a/tests/unittests/evaluation/test_eval_case.py b/tests/unittests/evaluation/test_eval_case.py new file mode 100644 index 00000000..01cb9b62 --- /dev/null +++ b/tests/unittests/evaluation/test_eval_case.py @@ -0,0 +1,99 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from google.adk.evaluation.eval_case import get_all_tool_calls +from google.adk.evaluation.eval_case import IntermediateData +from google.adk.evaluation.eval_case import InvocationEvent +from google.adk.evaluation.eval_case import InvocationEvents +from google.genai import types as genai_types +import pytest + + +def test_get_all_tool_calls_with_none_input(): + """Tests that an empty list is returned when intermediate_data is None.""" + assert get_all_tool_calls(None) == [] + + +def test_get_all_tool_calls_with_intermediate_data_no_tools(): + """Tests IntermediateData with no tool calls.""" + intermediate_data = IntermediateData(tool_uses=[]) + assert get_all_tool_calls(intermediate_data) == [] + + +def test_get_all_tool_calls_with_intermediate_data(): + """Tests that tool calls are correctly extracted from IntermediateData.""" + tool_call1 = genai_types.FunctionCall( + name='search', args={'query': 'weather'} + ) + tool_call2 = genai_types.FunctionCall(name='lookup', args={'id': '123'}) + intermediate_data = IntermediateData(tool_uses=[tool_call1, tool_call2]) + assert get_all_tool_calls(intermediate_data) == [tool_call1, tool_call2] + + +def test_get_all_tool_calls_with_empty_invocation_events(): + """Tests InvocationEvents with an empty list of invocation events.""" + intermediate_data = InvocationEvents(invocation_events=[]) + assert get_all_tool_calls(intermediate_data) == [] + + +def test_get_all_tool_calls_with_invocation_events_no_tools(): + """Tests InvocationEvents containing events without any tool calls.""" + invocation_event = InvocationEvent( + author='agent', + content=genai_types.Content( + parts=[genai_types.Part(text='Thinking...')], role='model' + ), + ) + intermediate_data = InvocationEvents(invocation_events=[invocation_event]) + assert get_all_tool_calls(intermediate_data) == [] + + +def test_get_all_tool_calls_with_invocation_events(): + """Tests that tool calls are correctly extracted from a InvocationSteps object.""" + tool_call1 = genai_types.FunctionCall( + name='search', args={'query': 'weather'} + ) + tool_call2 = genai_types.FunctionCall(name='lookup', args={'id': '123'}) + + invocation_event1 = InvocationEvent( + author='agent1', + content=genai_types.Content( + parts=[genai_types.Part(function_call=tool_call1)], + role='model', + ), + ) + invocation_event2 = InvocationEvent( + author='agent2', + content=genai_types.Content( + parts=[ + genai_types.Part(text='Found something.'), + genai_types.Part(function_call=tool_call2), + ], + role='model', + ), + ) + intermediate_data = InvocationEvents( + invocation_events=[invocation_event1, invocation_event2] + ) + assert get_all_tool_calls(intermediate_data) == [tool_call1, tool_call2] + + +def test_get_all_tool_calls_with_unsupported_type(): + """Tests that a ValueError is raised for unsupported intermediate_data types.""" + with pytest.raises( + ValueError, match='Unsupported type for intermediate_data' + ): + get_all_tool_calls('this is not a valid type') diff --git a/tests/unittests/evaluation/test_evaluation_generator.py b/tests/unittests/evaluation/test_evaluation_generator.py new file mode 100644 index 00000000..2b649aa8 --- /dev/null +++ b/tests/unittests/evaluation/test_evaluation_generator.py @@ -0,0 +1,197 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from google.adk.evaluation.evaluation_generator import EvaluationGenerator +from google.adk.events.event import Event +from google.genai import types + + +def _build_event( + author: str, parts: list[types.Part], invocation_id: str +) -> Event: + """Builds an Event object with specified parts.""" + + return Event( + author=author, + content=types.Content(parts=parts), + invocation_id=invocation_id, + ) + + +class TestConvertEventsToEvalInvocation: + """Test cases for EvaluationGenerator.convert_events_to_eval_invocations method.""" + + def test_convert_events_to_eval_invocations_empty( + self, + ): + """Tests conversion with an empty list of events.""" + invocations = EvaluationGenerator.convert_events_to_eval_invocations([]) + assert invocations == [] + + def test_convert_single_turn_text_only( + self, + ): + """Tests a single turn with a text response.""" + events = [ + _build_event("user", [types.Part(text="Hello")], "inv1"), + _build_event("agent", [types.Part(text="Hi there!")], "inv1"), + ] + + invocations = EvaluationGenerator.convert_events_to_eval_invocations(events) + + assert len(invocations) == 1 + invocation = invocations[0] + assert invocation.invocation_id == "inv1" + assert invocation.user_content.parts[0].text == "Hello" + assert invocation.final_response.parts[0].text == "Hi there!" + assert len(invocation.intermediate_data.invocation_events) == 0 + + def test_convert_single_turn_tool_call( + self, + ): + """Tests a single turn with a tool call.""" + events = [ + _build_event("user", [types.Part(text="what is the weather?")], "inv1"), + _build_event( + "agent", + [ + types.Part( + function_call=types.FunctionCall( + name="get_weather", args={} + ) + ) + ], + "inv1", + ), + ] + + invocations = EvaluationGenerator.convert_events_to_eval_invocations(events) + + assert len(invocations) == 1 + invocation = invocations[0] + assert invocation.user_content.parts[0].text == "what is the weather?" + assert invocation.final_response is None + events = invocation.intermediate_data.invocation_events + assert len(events) == 1 + assert events[0].author == "agent" + assert events[0].content.parts[0].function_call.name == "get_weather" + + def test_convert_single_turn_tool_and_text_response( + self, + ): + """Tests a single turn with a tool call and a final text response.""" + events = [ + _build_event("user", [types.Part(text="what is the weather?")], "inv1"), + _build_event( + "agent", + [ + types.Part( + function_call=types.FunctionCall( + name="get_weather", args={} + ) + ) + ], + "inv1", + ), + _build_event("agent", [types.Part(text="It is sunny in SF.")], "inv1"), + ] + + invocations = EvaluationGenerator.convert_events_to_eval_invocations(events) + + assert len(invocations) == 1 + invocation = invocations[0] + assert invocation.final_response.parts[0].text == "It is sunny in SF." + events = invocation.intermediate_data.invocation_events + assert len(events) == 1 + assert events[0].content.parts[0].function_call.name == "get_weather" + + def test_multi_turn( + self, + ): + """Tests a conversation with multiple turns.""" + events = [ + _build_event("user", [types.Part(text="Hello")], "inv1"), + _build_event("agent", [types.Part(text="Hi there!")], "inv1"), + _build_event("user", [types.Part(text="How are you?")], "inv2"), + _build_event("agent", [types.Part(text="I am fine.")], "inv2"), + ] + + invocations = EvaluationGenerator.convert_events_to_eval_invocations(events) + + assert len(invocations) == 2 + assert invocations[0].user_content.parts[0].text == "Hello" + assert invocations[0].final_response.parts[0].text == "Hi there!" + assert invocations[1].user_content.parts[0].text == "How are you?" + assert invocations[1].final_response.parts[0].text == "I am fine." + + def test_multi_agent( + self, + ): + """Tests a multi-agent scenario creating multiple steps.""" + events = [ + _build_event("user", [types.Part(text="Do something")], "inv1"), + _build_event( + "root_agent", + [ + types.Part( + function_call=types.FunctionCall(name="tool1", args={}) + ) + ], + "inv1", + ), + _build_event( + "sub_agent_1", + [ + types.Part( + function_call=types.FunctionCall(name="tool2", args={}) + ) + ], + "inv1", + ), + _build_event( + "sub_agent_1", + [ + types.Part( + function_call=types.FunctionCall(name="tool3", args={}) + ), + types.Part(text="intermediate response"), + ], + "inv1", + ), + _build_event( + "sub_agent_2", + [ + types.Part( + function_call=types.FunctionCall(name="tool4", args={}) + ) + ], + "inv1", + ), + _build_event("root_agent", [types.Part(text="All done.")], "inv1"), + ] + + invocations = EvaluationGenerator.convert_events_to_eval_invocations(events) + + assert len(invocations) == 1 + invocation = invocations[0] + assert invocation.final_response.parts[0].text == "All done." + events = invocation.intermediate_data.invocation_events + + assert len(events) == 4 + assert events[0].author == "root_agent" + assert events[1].author == "sub_agent_1" + assert events[2].author == "sub_agent_1" + assert events[3].author == "sub_agent_2" diff --git a/tests/unittests/evaluation/test_local_eval_service.py b/tests/unittests/evaluation/test_local_eval_service.py index 347a0053..f68136ca 100644 --- a/tests/unittests/evaluation/test_local_eval_service.py +++ b/tests/unittests/evaluation/test_local_eval_service.py @@ -429,7 +429,6 @@ async def test_mcp_stdio_agent_no_runtime_error(): user_content=genai_types.Content( parts=[genai_types.Part(text="List directory contents")] ), - expected_response="", ) ], )