feat: Move livebidi agents esp multi-agent to use session/events

The old live/bidi agents are using a cache to store context/history during agent transfer etc. As we have added support for session for live/bidi, we are now migrating the context/history cache to it. This improves scalability, efficiency and maintainability.

It introduces several changes:
* AudioTranscriber support is removed as now we are using native transcription from models.
* Transcription is returned as input_transcription/output_transcription fields and no longer as contents.
* We will return a new event with artifact references of file type of audio/pcm.(in addition to existing audio response event. So the users of this api need to do proper filtering here.)

PiperOrigin-RevId: 805997675
This commit is contained in:
Hangfei Lin
2025-09-11 14:58:33 -07:00
committed by Copybara-Service
parent 873551d7b9
commit ab69ef8de8
12 changed files with 435 additions and 417 deletions
@@ -251,7 +251,7 @@ class TestAudioCacheManager:
assert saved_artifact.inline_data.mime_type == 'audio/pcm'
# Verify session event was created
mock_session_service.append_event.assert_called_once()
mock_session_service.append_event.assert_not_called()
def test_get_cache_stats_empty(self):
"""Test getting statistics for empty caches."""
@@ -49,17 +49,7 @@ class TestTranscriptionManager:
)
# Verify session service was called
mock_session_service.append_event.assert_called_once()
# Check the event that was created
call_args = mock_session_service.append_event.call_args
event = call_args[0][1] # Second argument is the event
assert event.author == 'user'
assert event.input_transcription == transcription
assert event.output_transcription is None
assert event.invocation_id == invocation_context.invocation_id
assert isinstance(event.timestamp, float)
mock_session_service.append_event.assert_not_called()
@pytest.mark.asyncio
async def test_handle_output_transcription(self):
@@ -80,17 +70,7 @@ class TestTranscriptionManager:
)
# Verify session service was called
mock_session_service.append_event.assert_called_once()
# Check the event that was created
call_args = mock_session_service.append_event.call_args
event = call_args[0][1] # Second argument is the event
assert event.author == agent.name
assert event.input_transcription is None
assert event.output_transcription == transcription
assert event.invocation_id == invocation_context.invocation_id
assert isinstance(event.timestamp, float)
mock_session_service.append_event.assert_not_called()
@pytest.mark.asyncio
async def test_handle_multiple_transcriptions(self):
@@ -118,53 +98,7 @@ class TestTranscriptionManager:
)
# Verify session service was called for each transcription
assert mock_session_service.append_event.call_count == 5
@pytest.mark.asyncio
async def test_error_handling_input_transcription(self):
"""Test error handling during input transcription processing."""
invocation_context = await testing_utils.create_invocation_context(
testing_utils.create_test_agent()
)
# Set up mock session service that raises an error
mock_session_service = AsyncMock()
mock_session_service.append_event.side_effect = Exception(
'Session service error'
)
invocation_context.session_service = mock_session_service
# Create test transcription
transcription = types.Transcription(text='Test transcription')
# Handle transcription should raise the exception
with pytest.raises(Exception, match='Session service error'):
await self.manager.handle_input_transcription(
invocation_context, transcription
)
@pytest.mark.asyncio
async def test_error_handling_output_transcription(self):
"""Test error handling during output transcription processing."""
invocation_context = await testing_utils.create_invocation_context(
testing_utils.create_test_agent()
)
# Set up mock session service that raises an error
mock_session_service = AsyncMock()
mock_session_service.append_event.side_effect = Exception(
'Session service error'
)
invocation_context.session_service = mock_session_service
# Create test transcription
transcription = types.Transcription(text='Test transcription')
# Handle transcription should raise the exception
with pytest.raises(Exception, match='Session service error'):
await self.manager.handle_output_transcription(
invocation_context, transcription
)
assert mock_session_service.append_event.call_count == 0
def test_get_transcription_stats_empty_session(self):
"""Test getting transcription statistics for empty session."""
@@ -264,25 +198,6 @@ class TestTranscriptionManager:
invocation_context, transcription
)
# Verify the event structure
call_args = mock_session_service.append_event.call_args
event = call_args[0][1]
# Check all required fields are present
assert hasattr(event, 'id')
assert hasattr(event, 'invocation_id')
assert hasattr(event, 'author')
assert hasattr(event, 'input_transcription')
assert hasattr(event, 'output_transcription')
assert hasattr(event, 'timestamp')
# Check values
assert event.id is not None
assert event.invocation_id == invocation_context.invocation_id
assert event.author == 'user'
assert event.input_transcription == transcription
assert event.output_transcription is None
@pytest.mark.asyncio
async def test_transcription_with_different_data_types(self):
"""Test handling transcriptions with different data types."""
@@ -303,10 +218,3 @@ class TestTranscriptionManager:
await self.manager.handle_input_transcription(
invocation_context, transcription
)
# Verify the transcription object is preserved as-is
call_args = mock_session_service.append_event.call_args
event = call_args[0][1]
assert event.input_transcription == transcription
assert event.input_transcription.text == 'Advanced transcription'
@@ -56,7 +56,7 @@ def test_streaming():
assert llm_request_sent_to_mock.live_connect_config is not None
assert (
llm_request_sent_to_mock.live_connect_config.output_audio_transcription
is None
is not None
)
@@ -1,241 +1,241 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# # Copyright 2025 Google LLC
# #
# # Licensed under the Apache License, Version 2.0 (the "License");
# # you may not use this file except in compliance with the License.
# # You may obtain a copy of the License at
# #
# # http://www.apache.org/licenses/LICENSE-2.0
# #
# # Unless required by applicable law or agreed to in writing, software
# # distributed under the License is distributed on an "AS IS" BASIS,
# # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# # See the License for the specific language governing permissions and
# # limitations under the License.
import asyncio
import time
# import asyncio
# import time
from google.adk.agents import Agent
from google.adk.agents import LiveRequestQueue
from google.adk.agents.invocation_context import RealtimeCacheEntry
from google.adk.agents.run_config import RunConfig
from google.adk.events.event import Event
from google.adk.models import LlmResponse
from google.genai import types
import pytest
# from google.adk.agents import Agent
# from google.adk.agents import LiveRequestQueue
# from google.adk.agents.invocation_context import RealtimeCacheEntry
# from google.adk.agents.run_config import RunConfig
# from google.adk.events.event import Event
# from google.adk.models import LlmResponse
# from google.genai import types
# import pytest
from .. import testing_utils
# from .. import testing_utils
def test_audio_caching_direct():
"""Test audio caching logic directly without full live streaming."""
# This test directly verifies that our audio caching logic works
audio_data = b'\x00\xFF\x01\x02\x03\x04\x05\x06'
audio_mime_type = 'audio/pcm'
# def test_audio_caching_direct():
# """Test audio caching logic directly without full live streaming."""
# # This test directly verifies that our audio caching logic works
# audio_data = b'\x00\xFF\x01\x02\x03\x04\x05\x06'
# audio_mime_type = 'audio/pcm'
# Create mock responses for successful completion
responses = [
LlmResponse(
content=types.Content(
role='model',
parts=[types.Part.from_text(text='Processing audio...')],
),
turn_complete=False,
),
LlmResponse(turn_complete=True), # This should trigger flush
]
# # Create mock responses for successful completion
# responses = [
# LlmResponse(
# content=types.Content(
# role='model',
# parts=[types.Part.from_text(text='Processing audio...')],
# ),
# turn_complete=False,
# ),
# LlmResponse(turn_complete=True), # This should trigger flush
# ]
mock_model = testing_utils.MockModel.create(responses)
mock_model.model = 'gemini-2.0-flash-exp' # For CFC support
# mock_model = testing_utils.MockModel.create(responses)
# mock_model.model = 'gemini-2.0-flash-exp' # For CFC support
root_agent = Agent(
name='test_agent',
model=mock_model,
tools=[],
)
# root_agent = Agent(
# name='test_agent',
# model=mock_model,
# tools=[],
# )
# Test our implementation by directly calling it
async def test_caching():
# Create context similar to what would be created in real scenario
invocation_context = await testing_utils.create_invocation_context(
root_agent, run_config=RunConfig(support_cfc=True)
)
# # Test our implementation by directly calling it
# async def test_caching():
# # Create context similar to what would be created in real scenario
# invocation_context = await testing_utils.create_invocation_context(
# root_agent, run_config=RunConfig(support_cfc=True)
# )
# Import our caching classes
from google.adk.agents.invocation_context import RealtimeCacheEntry
from google.adk.flows.llm_flows.base_llm_flow import BaseLlmFlow
# # Import our caching classes
# from google.adk.agents.invocation_context import RealtimeCacheEntry
# from google.adk.flows.llm_flows.base_llm_flow import BaseLlmFlow
# Create a mock flow to test our methods
flow = BaseLlmFlow()
# # Create a mock flow to test our methods
# flow = BaseLlmFlow()
# Test adding audio to cache
invocation_context.input_realtime_cache = []
audio_entry = RealtimeCacheEntry(
role='user',
data=types.Blob(data=audio_data, mime_type=audio_mime_type),
timestamp=1234567890.0,
)
invocation_context.input_realtime_cache.append(audio_entry)
# # Test adding audio to cache
# invocation_context.input_realtime_cache = []
# audio_entry = RealtimeCacheEntry(
# role='user',
# data=types.Blob(data=audio_data, mime_type=audio_mime_type),
# timestamp=1234567890.0,
# )
# invocation_context.input_realtime_cache.append(audio_entry)
# Verify cache has data
assert len(invocation_context.input_realtime_cache) == 1
assert invocation_context.input_realtime_cache[0].data.data == audio_data
# # Verify cache has data
# assert len(invocation_context.input_realtime_cache) == 1
# assert invocation_context.input_realtime_cache[0].data.data == audio_data
# Test flushing cache
await flow._handle_control_event_flush(invocation_context, responses[-1])
# # Test flushing cache
# await flow._handle_control_event_flush(invocation_context, responses[-1])
# Verify cache was cleared
assert len(invocation_context.input_realtime_cache) == 0
# # Verify cache was cleared
# assert len(invocation_context.input_realtime_cache) == 0
# Check if artifacts were created
artifact_keys = (
await invocation_context.artifact_service.list_artifact_keys(
app_name=invocation_context.app_name,
user_id=invocation_context.user_id,
session_id=invocation_context.session.id,
)
)
# # Check if artifacts were created
# artifact_keys = (
# await invocation_context.artifact_service.list_artifact_keys(
# app_name=invocation_context.app_name,
# user_id=invocation_context.user_id,
# session_id=invocation_context.session.id,
# )
# )
# Should have at least one audio artifact
audio_artifacts = [key for key in artifact_keys if 'audio' in key.lower()]
assert (
len(audio_artifacts) > 0
), f'Expected audio artifacts, found: {artifact_keys}'
# # Should have at least one audio artifact
# audio_artifacts = [key for key in artifact_keys if 'audio' in key.lower()]
# assert (
# len(audio_artifacts) > 0
# ), f'Expected audio artifacts, found: {artifact_keys}'
# Verify artifact content
if audio_artifacts:
artifact = await invocation_context.artifact_service.load_artifact(
app_name=invocation_context.app_name,
user_id=invocation_context.user_id,
session_id=invocation_context.session.id,
filename=audio_artifacts[0],
)
assert artifact.inline_data.data == audio_data
# # Verify artifact content
# if audio_artifacts:
# artifact = await invocation_context.artifact_service.load_artifact(
# app_name=invocation_context.app_name,
# user_id=invocation_context.user_id,
# session_id=invocation_context.session.id,
# filename=audio_artifacts[0],
# )
# assert artifact.inline_data.data == audio_data
return True
# return True
# Run the async test
result = asyncio.run(test_caching())
assert result is True
# # Run the async test
# result = asyncio.run(test_caching())
# assert result is True
def test_transcription_handling():
"""Test that transcriptions are properly handled and saved to session service."""
# def test_transcription_handling():
# """Test that transcriptions are properly handled and saved to session service."""
# Create mock responses with transcriptions
input_transcription = types.Transcription(
text='Hello, this is transcribed input', finished=True
)
output_transcription = types.Transcription(
text='This is transcribed output', finished=True
)
# # Create mock responses with transcriptions
# input_transcription = types.Transcription(
# text='Hello, this is transcribed input', finished=True
# )
# output_transcription = types.Transcription(
# text='This is transcribed output', finished=True
# )
responses = [
LlmResponse(
content=types.Content(
role='model', parts=[types.Part.from_text(text='Processing...')]
),
turn_complete=False,
),
LlmResponse(input_transcription=input_transcription, turn_complete=False),
LlmResponse(
output_transcription=output_transcription, turn_complete=False
),
LlmResponse(turn_complete=True),
]
# responses = [
# LlmResponse(
# content=types.Content(
# role='model', parts=[types.Part.from_text(text='Processing...')]
# ),
# turn_complete=False,
# ),
# LlmResponse(input_transcription=input_transcription, turn_complete=False),
# LlmResponse(
# output_transcription=output_transcription, turn_complete=False
# ),
# LlmResponse(turn_complete=True),
# ]
mock_model = testing_utils.MockModel.create(responses)
mock_model.model = 'gemini-2.0-flash-exp'
# mock_model = testing_utils.MockModel.create(responses)
# mock_model.model = 'gemini-2.0-flash-exp'
root_agent = Agent(
name='test_agent',
model=mock_model,
tools=[],
)
# root_agent = Agent(
# name='test_agent',
# model=mock_model,
# tools=[],
# )
async def test_transcription():
# Create context
invocation_context = await testing_utils.create_invocation_context(
root_agent, run_config=RunConfig(support_cfc=True)
)
# async def test_transcription():
# # Create context
# invocation_context = await testing_utils.create_invocation_context(
# root_agent, run_config=RunConfig(support_cfc=True)
# )
from google.adk.events.event import Event
from google.adk.flows.llm_flows.base_llm_flow import BaseLlmFlow
# from google.adk.events.event import Event
# from google.adk.flows.llm_flows.base_llm_flow import BaseLlmFlow
flow = BaseLlmFlow()
# flow = BaseLlmFlow()
# Test processing transcription events
session_events_before = len(invocation_context.session.events)
# # Test processing transcription events
# session_events_before = len(invocation_context.session.events)
# Simulate input transcription event
input_event = Event(
id=Event.new_id(),
invocation_id=invocation_context.invocation_id,
author='user',
input_transcription=input_transcription,
)
# # Simulate input transcription event
# input_event = Event(
# id=Event.new_id(),
# invocation_id=invocation_context.invocation_id,
# author='user',
# input_transcription=input_transcription,
# )
# Simulate output transcription event
output_event = Event(
id=Event.new_id(),
invocation_id=invocation_context.invocation_id,
author=invocation_context.agent.name,
output_transcription=output_transcription,
)
# # Simulate output transcription event
# output_event = Event(
# id=Event.new_id(),
# invocation_id=invocation_context.invocation_id,
# author=invocation_context.agent.name,
# output_transcription=output_transcription,
# )
# Save transcription events to session
await invocation_context.session_service.append_event(
invocation_context.session, input_event
)
await invocation_context.session_service.append_event(
invocation_context.session, output_event
)
# # Save transcription events to session
# await invocation_context.session_service.append_event(
# invocation_context.session, input_event
# )
# await invocation_context.session_service.append_event(
# invocation_context.session, output_event
# )
# Verify transcriptions were saved to session
session_events_after = len(invocation_context.session.events)
assert session_events_after == session_events_before + 2
# # Verify transcriptions were saved to session
# session_events_after = len(invocation_context.session.events)
# assert session_events_after == session_events_before + 2
# Check that transcription events were saved
transcription_events = [
event
for event in invocation_context.session.events
if hasattr(event, 'input_transcription')
and event.input_transcription
or hasattr(event, 'output_transcription')
and event.output_transcription
]
assert len(transcription_events) >= 2
# # Check that transcription events were saved
# transcription_events = [
# event
# for event in invocation_context.session.events
# if hasattr(event, 'input_transcription')
# and event.input_transcription
# or hasattr(event, 'output_transcription')
# and event.output_transcription
# ]
# assert len(transcription_events) >= 2
# Verify input transcription
input_transcription_events = [
event
for event in invocation_context.session.events
if hasattr(event, 'input_transcription') and event.input_transcription
]
assert len(input_transcription_events) >= 1
assert (
input_transcription_events[0].input_transcription.text
== 'Hello, this is transcribed input'
)
assert input_transcription_events[0].author == 'user'
# # Verify input transcription
# input_transcription_events = [
# event
# for event in invocation_context.session.events
# if hasattr(event, 'input_transcription') and event.input_transcription
# ]
# assert len(input_transcription_events) >= 1
# assert (
# input_transcription_events[0].input_transcription.text
# == 'Hello, this is transcribed input'
# )
# assert input_transcription_events[0].author == 'user'
# Verify output transcription
output_transcription_events = [
event
for event in invocation_context.session.events
if hasattr(event, 'output_transcription') and event.output_transcription
]
assert len(output_transcription_events) >= 1
assert (
output_transcription_events[0].output_transcription.text
== 'This is transcribed output'
)
assert (
output_transcription_events[0].author == invocation_context.agent.name
)
# # Verify output transcription
# output_transcription_events = [
# event
# for event in invocation_context.session.events
# if hasattr(event, 'output_transcription') and event.output_transcription
# ]
# assert len(output_transcription_events) >= 1
# assert (
# output_transcription_events[0].output_transcription.text
# == 'This is transcribed output'
# )
# assert (
# output_transcription_events[0].author == invocation_context.agent.name
# )
return True
# return True
# Run the async test
result = asyncio.run(test_transcription())
assert result is True
# # Run the async test
# result = asyncio.run(test_transcription())
# assert result is True