feat: Data model for storing App Details and data model for steps

Details:
1. Data model for storing App Details (the agentic system)
As we move towards LLM as Judge metrics, we see that some of these metrics need information about the Agentic system that was used for inferencing. We add a data model to capture that.

2. Data model for Steps
We refine the concept of intermediate data. Previously it stored data in the form of a multiple lists, thereby losing out on the chronological information. This information is needed for some of the metrics. So we refine the concept of intermediate data as series of logical steps that an Agent Take.

PiperOrigin-RevId: 811122784

src/google/adk/evaluation/agent_evaluator.py

@@ -34,7 +34,8 @@ from pydantic import ValidationError
 from ..agents.base_agent import BaseAgent
 from ..utils.context_utils import Aclosing
 from .constants import MISSING_EVAL_DEPENDENCIES_MESSAGE
-from .eval_case import IntermediateData
+from .eval_case import get_all_tool_calls
+from .eval_case import IntermediateDataType
 from .eval_case import Invocation
 from .eval_metrics import EvalMetric
 from .eval_metrics import EvalMetricResult
@@ -457,12 +458,11 @@ class AgentEvaluator:
 
   @staticmethod
   def _convert_tool_calls_to_text(
-      intermediate_data: Optional[IntermediateData],
+      intermediate_data: Optional[IntermediateDataType],
   ) -> str:
-    if intermediate_data and intermediate_data.tool_uses:
-      return "\n".join([str(t) for t in intermediate_data.tool_uses])
+    tool_calls = get_all_tool_calls(intermediate_data)
 
-    return ""
+    return "\n".join([str(t) for t in tool_calls])
 
   @staticmethod
   def _get_agent_for_eval(

src/google/adk/evaluation/app_details.py

@@ -0,0 +1,49 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.genai import types as genai_types
+from pydantic import Field
+
+from .common import EvalBaseModel
+
+
+class AgentDetails(EvalBaseModel):
+  """Details about the individual agent in the App.
+
+  This could be a root agent or the sub-agents in the Agent Tree.
+  """
+
+  name: str
+  """The name of the Agent that uniquely identifies it in the App."""
+
+  instructions: str = Field(default="")
+  """The instructions set on the Agent."""
+
+  tool_declarations: genai_types.ToolListUnion = Field(default_factory=list)
+  """A list of tools available to the Agent."""
+
+
+class AppDetails(EvalBaseModel):
+  """Contains details about the App (the agentic system).
+
+  This structure is only a projection of the acutal app. Only details
+  that are relevant to the Eval System are captured here.
+  """
+
+  agent_details: dict[str, AgentDetails] = Field(
+      default_factory=dict,
+  )
+  """A mapping from the agent name to the details of that agent."""

src/google/adk/evaluation/common.py

@@ -23,4 +23,5 @@ class EvalBaseModel(pydantic.BaseModel):
       alias_generator=alias_generators.to_camel,
       populate_by_name=True,
       extra='forbid',
+      arbitrary_types_allowed=True,
   )

src/google/adk/evaluation/eval_case.py

@@ -16,23 +16,17 @@ from __future__ import annotations
 
 from typing import Any
 from typing import Optional
+from typing import Union
 
 from google.genai import types as genai_types
-from pydantic import alias_generators
-from pydantic import BaseModel
-from pydantic import ConfigDict
 from pydantic import Field
+from typing_extensions import TypeAlias
 
+from .app_details import AppDetails
+from .common import EvalBaseModel
 from .eval_rubrics import Rubric
 
 
-class EvalBaseModel(BaseModel):
-  model_config = ConfigDict(
-      alias_generator=alias_generators.to_camel,
-      populate_by_name=True,
-  )
-
-
 class IntermediateData(EvalBaseModel):
   """Container for intermediate data that an agent would generate as it responds with a final answer."""
 
@@ -54,6 +48,33 @@ class IntermediateData(EvalBaseModel):
   """
 
 
+class InvocationEvent(EvalBaseModel):
+  """An immutable record representing a specific point in the agent's invocation.
+
+  It captures agent's replies, requests to use tools (function calls), and tool
+  results.
+
+  This structure is a simple projection of the actual `Event` datamodel that
+  is intended for the Eval System.
+  """
+
+  author: str
+  """The name of the agent that authored/owned this event."""
+
+  content: Optional[genai_types.Content]
+  """The content of the event."""
+
+
+class InvocationEvents(EvalBaseModel):
+  """A container for events that occur during the course of an invocation."""
+
+  invocation_events: list[InvocationEvent] = Field(default_factory=list)
+  """A list of invocation events."""
+
+
+IntermediateDataType: TypeAlias = Union[IntermediateData, InvocationEvents]
+
+
 class Invocation(EvalBaseModel):
   """Represents a single invocation."""
 
@@ -66,7 +87,7 @@ class Invocation(EvalBaseModel):
   final_response: Optional[genai_types.Content] = None
   """Final response from the agent."""
 
-  intermediate_data: Optional[IntermediateData] = None
+  intermediate_data: Optional[IntermediateDataType] = None
   """Intermediate steps generated as a part of Agent execution.
 
   For a multi-agent system, it is also helpful to inspect the route that
@@ -81,6 +102,9 @@ class Invocation(EvalBaseModel):
   )
   """A list of rubrics that are applicable to only this invocation."""
 
+  app_details: Optional[AppDetails] = Field(default=None)
+  """Details about the App that was used for this invocation."""
+
 
 class SessionInput(EvalBaseModel):
   """Values that help initialize a Session."""
@@ -117,3 +141,30 @@ class EvalCase(EvalBaseModel):
       default=None,
   )
   """A list of rubrics that are applicable to all the invocations in the conversation of this eval case."""
+
+
+def get_all_tool_calls(
+    intermediate_data: Optional[IntermediateDataType],
+) -> list[genai_types.FunctionCall]:
+  """A utility method to retrieve tools calls from intermediate data."""
+  if not intermediate_data:
+    return []
+
+  tool_calls = []
+  if isinstance(intermediate_data, IntermediateData):
+    tool_calls = intermediate_data.tool_uses
+  elif isinstance(intermediate_data, InvocationEvents):
+    # Go over each event in the list of events
+    for invocation_event in intermediate_data.invocation_events:
+      # Check if the event has content and some parts.
+      if invocation_event.content and invocation_event.content.parts:
+        for p in invocation_event.content.parts:
+          # For each part, we check if any of those part is a function call.
+          if p.function_call:
+            tool_calls.append(p.function_call)
+  else:
+    raise ValueError(
+        f"Unsupported type for intermediate_data `{intermediate_data}`"
+    )
+
+  return tool_calls

src/google/adk/evaluation/evaluation_generator.py

@@ -24,6 +24,7 @@ from pydantic import BaseModel
 from ..agents.llm_agent import Agent
 from ..artifacts.base_artifact_service import BaseArtifactService
 from ..artifacts.in_memory_artifact_service import InMemoryArtifactService
+from ..events.event import Event
 from ..memory.base_memory_service import BaseMemoryService
 from ..memory.in_memory_memory_service import InMemoryMemoryService
 from ..runners import Runner
@@ -31,12 +32,17 @@ from ..sessions.base_session_service import BaseSessionService
 from ..sessions.in_memory_session_service import InMemorySessionService
 from ..sessions.session import Session
 from ..utils.context_utils import Aclosing
+from .app_details import AppDetails
 from .eval_case import EvalCase
-from .eval_case import IntermediateData
 from .eval_case import Invocation
+from .eval_case import InvocationEvent
+from .eval_case import InvocationEvents
 from .eval_case import SessionInput
 from .eval_set import EvalSet
 
+_USER_AUTHOR = "user"
+_DEFAULT_AUTHOR = "agent"
+
 
 class EvalCaseResponses(BaseModel):
   """Contains multiple responses associated with an EvalCase.
@@ -174,8 +180,6 @@ class EvaluationGenerator:
     if callable(reset_func):
       reset_func()
 
-    response_invocations = []
-
     async with Runner(
         app_name=app_name,
         agent=root_agent,
@@ -183,42 +187,94 @@ class EvaluationGenerator:
         session_service=session_service,
         memory_service=memory_service,
     ) as runner:
+      events = []
+
       for invocation in invocations:
-        final_response = None
         user_content = invocation.user_content
-        tool_uses = []
-        invocation_id = ""
+        invocation_id = None
 
         async with Aclosing(
             runner.run_async(
                 user_id=user_id, session_id=session_id, new_message=user_content
             )
         ) as agen:
+
           async for event in agen:
-            invocation_id = (
-                event.invocation_id if not invocation_id else invocation_id
-            )
+            if not invocation_id:
+              invocation_id = event.invocation_id
+              events.append(
+                  Event(
+                      content=user_content,
+                      author=_USER_AUTHOR,
+                      invocation_id=invocation_id,
+                  )
+              )
 
-            if (
-                event.is_final_response()
-                and event.content
-                and event.content.parts
-            ):
-              final_response = event.content
-            elif event.get_function_calls():
-              for call in event.get_function_calls():
-                tool_uses.append(call)
+            events.append(event)
 
-        response_invocations.append(
-            Invocation(
-                invocation_id=invocation_id,
-                user_content=user_content,
-                final_response=final_response,
-                intermediate_data=IntermediateData(tool_uses=tool_uses),
-            )
-        )
+      return EvaluationGenerator.convert_events_to_eval_invocations(events)
 
-    return response_invocations
+  @staticmethod
+  def convert_events_to_eval_invocations(
+      events: list[Event],
+  ) -> list[Invocation]:
+    """Converts a list of events to eval invocations."""
+    # Group Events by invocation id. Events that share the same invocation id
+    # belong to the same invocation.
+    events_by_invocation_id: dict[str, list[Event]] = {}
+
+    for event in events:
+      invocation_id = event.invocation_id
+
+      if invocation_id not in events_by_invocation_id:
+        events_by_invocation_id[invocation_id] = []
+
+      events_by_invocation_id[invocation_id].append(event)
+
+    invocations = []
+    for invocation_id, events in events_by_invocation_id.items():
+      final_response = None
+      user_content = ""
+      invocation_timestamp = 0
+
+      events_to_add = []
+
+      for event in events:
+        current_author = (event.author or _DEFAULT_AUTHOR).lower()
+
+        if current_author == _USER_AUTHOR:
+          # If the author is the user, then we just identify it and move on
+          # to the next event.
+          user_content = event.content
+          invocation_timestamp = event.timestamp
+          continue
+
+        if event.content and event.content.parts:
+          if event.is_final_response():
+            final_response = event.content
+          else:
+            for p in event.content.parts:
+              if p.function_call or p.function_response or p.text:
+                events_to_add.append(event)
+                break
+
+      invocation_events = [
+          InvocationEvent(author=e.author, content=e.content)
+          for e in events_to_add
+      ]
+      invocations.append(
+          Invocation(
+              invocation_id=invocation_id,
+              user_content=user_content,
+              final_response=final_response,
+              intermediate_data=InvocationEvents(
+                  invocation_events=invocation_events
+              ),
+              creation_timestamp=invocation_timestamp,
+          )
+      )
+
+    return invocations
 
   @staticmethod
   def _process_query_with_session(session_data, data):

src/google/adk/evaluation/gcs_eval_sets_manager.py

@@ -84,7 +84,12 @@ class GcsEvalSetsManager(EvalSetsManager):
     """Writes an EvalSet to GCS."""
     blob = self.bucket.blob(blob_name)
     blob.upload_from_string(
-        eval_set.model_dump_json(indent=2),
+        eval_set.model_dump_json(
+            indent=2,
+            exclude_unset=True,
+            exclude_defaults=True,
+            exclude_none=True,
+        ),
         content_type="application/json",
     )
 

src/google/adk/evaluation/local_eval_sets_manager.py

@@ -325,7 +325,14 @@ class LocalEvalSetsManager(EvalSetsManager):
 
   def _write_eval_set_to_path(self, eval_set_path: str, eval_set: EvalSet):
     with open(eval_set_path, "w", encoding="utf-8") as f:
-      f.write(eval_set.model_dump_json(indent=2))
+      f.write(
+          eval_set.model_dump_json(
+              indent=2,
+              exclude_unset=True,
+              exclude_defaults=True,
+              exclude_none=True,
+          )
+      )
 
   def _save_eval_set(self, app_name: str, eval_set_id: str, eval_set: EvalSet):
     eval_set_file_path = self._get_eval_set_file_path(app_name, eval_set_id)

src/google/adk/evaluation/trajectory_evaluator.py

@@ -23,6 +23,7 @@ from tabulate import tabulate
 from typing_extensions import deprecated
 from typing_extensions import override
 
+from .eval_case import get_all_tool_calls
 from .eval_case import Invocation
 from .eval_metrics import EvalMetric
 from .eval_metrics import Interval
@@ -83,14 +84,9 @@ class TrajectoryEvaluator(Evaluator):
     per_invocation_results = []
 
     for actual, expected in zip(actual_invocations, expected_invocations):
-      actual_tool_uses = (
-          actual.intermediate_data.tool_uses if actual.intermediate_data else []
-      )
-      expected_tool_uses = (
-          expected.intermediate_data.tool_uses
-          if expected.intermediate_data
-          else []
-      )
+      actual_tool_uses = get_all_tool_calls(actual.intermediate_data)
+      expected_tool_uses = get_all_tool_calls(expected.intermediate_data)
+
       tool_use_accuracy = (
           1.0
           if self._are_tool_calls_equal(actual_tool_uses, expected_tool_uses)