diff --git a/src/google/adk/evaluation/evaluation_generator.py b/src/google/adk/evaluation/evaluation_generator.py index 5b955fdb..97986dfd 100644 --- a/src/google/adk/evaluation/evaluation_generator.py +++ b/src/google/adk/evaluation/evaluation_generator.py @@ -32,6 +32,7 @@ from ..sessions.base_session_service import BaseSessionService from ..sessions.in_memory_session_service import InMemorySessionService from ..sessions.session import Session from ..utils.context_utils import Aclosing +from .app_details import AgentDetails from .app_details import AppDetails from .eval_case import EvalCase from .eval_case import Invocation @@ -39,6 +40,7 @@ from .eval_case import InvocationEvent from .eval_case import InvocationEvents from .eval_case import SessionInput from .eval_set import EvalSet +from .request_intercepter_plugin import _RequestIntercepterPlugin _USER_AUTHOR = "user" _DEFAULT_AUTHOR = "agent" @@ -180,12 +182,16 @@ class EvaluationGenerator: if callable(reset_func): reset_func() + request_intercepter_plugin = _RequestIntercepterPlugin( + name="request_intercepter_plugin" + ) async with Runner( app_name=app_name, agent=root_agent, artifact_service=artifact_service, session_service=session_service, memory_service=memory_service, + plugins=[request_intercepter_plugin], ) as runner: events = [] @@ -212,30 +218,36 @@ class EvaluationGenerator: events.append(event) - return EvaluationGenerator.convert_events_to_eval_invocations(events) + app_details_by_invocation_id = ( + EvaluationGenerator._get_app_details_by_invocation_id( + events, request_intercepter_plugin + ) + ) + return EvaluationGenerator.convert_events_to_eval_invocations( + events, app_details_by_invocation_id + ) @staticmethod def convert_events_to_eval_invocations( events: list[Event], + app_details_per_invocation: Optional[dict[str, AppDetails]] = None, ) -> list[Invocation]: """Converts a list of events to eval invocations.""" - # Group Events by invocation id. Events that share the same invocation id - # belong to the same invocation. - events_by_invocation_id: dict[str, list[Event]] = {} - - for event in events: - invocation_id = event.invocation_id - - if invocation_id not in events_by_invocation_id: - events_by_invocation_id[invocation_id] = [] - - events_by_invocation_id[invocation_id].append(event) + events_by_invocation_id = ( + EvaluationGenerator._collect_events_by_invocation_id(events) + ) invocations = [] for invocation_id, events in events_by_invocation_id.items(): final_response = None user_content = "" invocation_timestamp = 0 + app_details = None + if ( + app_details_per_invocation + and invocation_id in app_details_per_invocation + ): + app_details = app_details_per_invocation[invocation_id] events_to_add = [] @@ -271,11 +283,61 @@ class EvaluationGenerator: invocation_events=invocation_events ), creation_timestamp=invocation_timestamp, + app_details=app_details, ) ) return invocations + @staticmethod + def _get_app_details_by_invocation_id( + events: list[Event], request_intercepter: _RequestIntercepterPlugin + ) -> dict[str, AppDetails]: + """Creates an AppDetails object from the list of events.""" + events_by_invocation_id = ( + EvaluationGenerator._collect_events_by_invocation_id(events) + ) + app_details_by_invocation_id = {} + + for invocation_id, events in events_by_invocation_id.items(): + app_details = AppDetails(agent_details={}) + app_details_by_invocation_id[invocation_id] = app_details + + for event in events: + if event.author == _USER_AUTHOR: + continue + + llm_request = request_intercepter.get_model_request(event) + + if not llm_request: + continue + + if event.author not in app_details.agent_details: + agent_name = event.author + app_details.agent_details[agent_name] = AgentDetails( + name=agent_name, + instructions=llm_request.config.system_instruction, + tool_declarations=llm_request.config.tools or [], + ) + + return app_details_by_invocation_id + + @staticmethod + def _collect_events_by_invocation_id(events: list[Event]) -> dict[str, Event]: + # Group Events by invocation id. Events that share the same invocation id + # belong to the same invocation. + events_by_invocation_id: dict[str, list[Event]] = {} + + for event in events: + invocation_id = event.invocation_id + + if invocation_id not in events_by_invocation_id: + events_by_invocation_id[invocation_id] = [] + + events_by_invocation_id[invocation_id].append(event) + + return events_by_invocation_id + @staticmethod def _process_query_with_session(session_data, data): """Process the queries using the existing session data without invoking the runner.""" diff --git a/src/google/adk/evaluation/request_intercepter_plugin.py b/src/google/adk/evaluation/request_intercepter_plugin.py new file mode 100644 index 00000000..374a30f6 --- /dev/null +++ b/src/google/adk/evaluation/request_intercepter_plugin.py @@ -0,0 +1,94 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import logging +from typing import Optional +import uuid + +from typing_extensions import override + +from ..agents.callback_context import CallbackContext +from ..models.llm_request import LlmRequest +from ..models.llm_response import LlmResponse +from ..plugins.base_plugin import BasePlugin + +logger = logging.getLogger("google_adk." + __name__) + +_LLM_REQUEST_ID_KEY = "__llm_request_key__" + + +class _RequestIntercepterPlugin(BasePlugin): + """A plugin that intercepts requests that are made to the model and couples them with the model response. + + NOTE: This implementation is intended for eval systems internal usage. Do not + take direct depdency on it. + + Context behind the creation of this intercepter: + Some of the newer AutoRater backed metrics need access the pieces of + information that were presented to the model like instructions and the list + of available tools. + + We intercept the llm_request using this intercepter and make it available to + eval system. + + How is it done? + The class maintains a cache of llm_requests that pass through it. Each request + is given a unique id. The id is put in custom_metadata field of the response. + Eval systems have access to the response and can use the request id to + get the llm_request. + """ + + def __init__(self, name: str): + super().__init__(name=name) + self._llm_requests_cache: dict[str, LlmRequest] = {} + + @override + async def before_model_callback( + self, *, callback_context: CallbackContext, llm_request: LlmRequest + ) -> Optional[LlmResponse]: + # We add the llm_request to the call back context so that we can fetch + # it later. + request_id = str(uuid.uuid4()) + self._llm_requests_cache[request_id] = llm_request + callback_context.state[_LLM_REQUEST_ID_KEY] = request_id + + @override + async def after_model_callback( + self, *, callback_context: CallbackContext, llm_response: LlmResponse + ) -> Optional[LlmResponse]: + # Fetch the request_id from the callback_context + if callback_context and _LLM_REQUEST_ID_KEY in callback_context.state: + if llm_response.custom_metadata is None: + llm_response.custom_metadata = {} + + llm_response.custom_metadata[_LLM_REQUEST_ID_KEY] = ( + callback_context.state[_LLM_REQUEST_ID_KEY] + ) + + def get_model_request( + self, llm_response: LlmResponse + ) -> Optional[LlmRequest]: + """Fetches the request object, if found.""" + if ( + llm_response.custom_metadata + and _LLM_REQUEST_ID_KEY in llm_response.custom_metadata + ): + request_id = llm_response.custom_metadata[_LLM_REQUEST_ID_KEY] + + if request_id in self._llm_requests_cache: + return self._llm_requests_cache[request_id] + else: + logger.warning("`%s` not found in llm_request_cache.", request_id) diff --git a/tests/unittests/evaluation/test_evaluation_generator.py b/tests/unittests/evaluation/test_evaluation_generator.py index 2b649aa8..4d6d5509 100644 --- a/tests/unittests/evaluation/test_evaluation_generator.py +++ b/tests/unittests/evaluation/test_evaluation_generator.py @@ -14,8 +14,14 @@ from __future__ import annotations +from unittest import mock + +from google.adk.evaluation.app_details import AgentDetails +from google.adk.evaluation.app_details import AppDetails from google.adk.evaluation.evaluation_generator import EvaluationGenerator +from google.adk.evaluation.request_intercepter_plugin import _RequestIntercepterPlugin from google.adk.events.event import Event +from google.adk.models.llm_request import LlmRequest from google.genai import types @@ -195,3 +201,128 @@ class TestConvertEventsToEvalInvocation: assert events[1].author == "sub_agent_1" assert events[2].author == "sub_agent_1" assert events[3].author == "sub_agent_2" + + +class TestGetAppDetailsByInvocationId: + """Test cases for EvaluationGenerator._get_app_details_by_invocation_id method.""" + + def test_get_app_details_by_invocation_id_empty(self): + """Tests with an empty list of events.""" + mock_request_intercepter = mock.MagicMock(spec=_RequestIntercepterPlugin) + app_details = EvaluationGenerator._get_app_details_by_invocation_id( + [], mock_request_intercepter + ) + assert app_details == {} + + def test_get_app_details_by_invocation_id_no_model_requests(self): + """Tests when request_intercepter returns no model requests.""" + mock_request_intercepter = mock.MagicMock(spec=_RequestIntercepterPlugin) + mock_request_intercepter.get_model_request.return_value = None + events = [ + _build_event("user", [types.Part(text="Hello")], "inv1"), + _build_event("agent", [types.Part(text="Hi there!")], "inv1"), + ] + app_details = EvaluationGenerator._get_app_details_by_invocation_id( + events, mock_request_intercepter + ) + assert app_details == {"inv1": AppDetails(agent_details={})} + mock_request_intercepter.get_model_request.assert_called_once_with( + events[1] + ) + + def test_get_app_details_single_invocation_single_agent(self): + """Tests a single invocation with one agent.""" + mock_request_intercepter = mock.MagicMock(spec=_RequestIntercepterPlugin) + mock_llm_request = LlmRequest(model="test") + mock_llm_request.config.system_instruction = "instruction1" + mock_llm_request.config.tools = [types.Tool()] + mock_request_intercepter.get_model_request.return_value = mock_llm_request + + events = [ + _build_event("user", [types.Part(text="Hello")], "inv1"), + _build_event("agent", [types.Part(text="Hi there!")], "inv1"), + ] + app_details = EvaluationGenerator._get_app_details_by_invocation_id( + events, mock_request_intercepter + ) + + expected_app_details = { + "inv1": AppDetails( + agent_details={ + "agent": AgentDetails( + name="agent", + instructions="instruction1", + tool_declarations=[types.Tool()], + ) + } + ) + } + assert app_details == expected_app_details + mock_request_intercepter.get_model_request.assert_called_once_with( + events[1] + ) + + def test_get_app_details_multiple_invocations_multiple_agents(self): + """Tests multiple invocations with multiple agents.""" + mock_request_intercepter = mock.MagicMock(spec=_RequestIntercepterPlugin) + + def get_model_request_side_effect(event): + mock_llm_request = LlmRequest(model="test") + if event.invocation_id == "inv1" and event.author == "agent1": + mock_llm_request.config.system_instruction = "instruction1" + mock_llm_request.config.tools = [ + types.Tool( + function_declarations=[types.FunctionDeclaration(name="tool1")] + ) + ] + return mock_llm_request + if event.invocation_id == "inv2" and event.author == "agent2": + mock_llm_request.config.system_instruction = "instruction2" + return mock_llm_request + return None + + mock_request_intercepter.get_model_request.side_effect = ( + get_model_request_side_effect + ) + + events = [ + _build_event("user", [types.Part(text="Hello")], "inv1"), + _build_event("agent1", [types.Part(text="Hi there!")], "inv1"), + _build_event("user", [types.Part(text="Hello again")], "inv2"), + _build_event("agent2", [types.Part(text="Hi again!")], "inv2"), + _build_event( + "agent1", [types.Part(text="Hi again from agent1")], "inv2" + ), # no request + ] + app_details = EvaluationGenerator._get_app_details_by_invocation_id( + events, mock_request_intercepter + ) + + expected_app_details = { + "inv1": AppDetails( + agent_details={ + "agent1": AgentDetails( + name="agent1", + instructions="instruction1", + tool_declarations=[ + types.Tool( + function_declarations=[ + types.FunctionDeclaration(name="tool1") + ] + ) + ], + ) + } + ), + "inv2": AppDetails( + agent_details={ + "agent2": AgentDetails( + name="agent2", + instructions="instruction2", + tool_declarations=[], + ) + } + ), + } + assert app_details == expected_app_details + assert mock_request_intercepter.get_model_request.call_count == 3 diff --git a/tests/unittests/evaluation/test_request_intercepter_plugin.py b/tests/unittests/evaluation/test_request_intercepter_plugin.py new file mode 100644 index 00000000..1381e001 --- /dev/null +++ b/tests/unittests/evaluation/test_request_intercepter_plugin.py @@ -0,0 +1,73 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from unittest import mock + +from google.adk.agents.callback_context import CallbackContext +from google.adk.evaluation.request_intercepter_plugin import _LLM_REQUEST_ID_KEY +from google.adk.evaluation.request_intercepter_plugin import _RequestIntercepterPlugin +from google.adk.models.llm_request import LlmRequest +from google.adk.models.llm_response import LlmResponse +from google.genai import types + + +class TestRequestIntercepterPlugin: + + async def test_intercept_request_and_response(self): + plugin = _RequestIntercepterPlugin(name="test_plugin") + llm_request = LlmRequest( + model="test_model", + contents=[ + types.Content( + role="user", + parts=[types.Part(text="hello")], + ) + ], + ) + mock_invocation_context = mock.MagicMock() + mock_invocation_context.session.state = {} + callback_context = CallbackContext(mock_invocation_context) + llm_response = LlmResponse() + + # Test before_model_callback + await plugin.before_model_callback( + callback_context=callback_context, llm_request=llm_request + ) + assert _LLM_REQUEST_ID_KEY in callback_context.state + request_id = callback_context.state[_LLM_REQUEST_ID_KEY] + assert isinstance(request_id, str) + + # Test after_model_callback + await plugin.after_model_callback( + callback_context=callback_context, llm_response=llm_response + ) + assert llm_response.custom_metadata is not None + assert _LLM_REQUEST_ID_KEY in llm_response.custom_metadata + assert llm_response.custom_metadata[_LLM_REQUEST_ID_KEY] == request_id + + # Test get_model_request + retrieved_request = plugin.get_model_request(llm_response) + assert retrieved_request == llm_request + + def test_get_model_request_not_found(self): + plugin = _RequestIntercepterPlugin(name="test_plugin") + llm_response = LlmResponse() + assert plugin.get_model_request(llm_response) is None + + llm_response_with_metadata = LlmResponse( + custom_metadata={_LLM_REQUEST_ID_KEY: "non_existent_id"} + ) + assert plugin.get_model_request(llm_response_with_metadata) is None