mirror of
https://github.com/encounter/adk-python.git
synced 2026-07-09 18:19:28 -07:00
fix: Improve handling of partial and complete transcriptions in live calls
In `gemini_llm_connection.py`, accumulate partial transcription texts and emit `LlmResponse` with `partial=True` for each chunk. When the transcription is marked as `finished`, emit a final `LlmResponse` with the full accumulated text and `partial=False`. In `runners.py`, modify `_should_append_to_history` to only add transcription events to the history when they are fully finished, preventing partial transcriptions from being added. Co-authored-by: Hangfei Lin <hangfei@google.com> PiperOrigin-RevId: 829029715
This commit is contained in:
committed by
Copybara-Service
parent
44d45fe9cd
commit
1819ecb4b8
@@ -587,21 +587,19 @@ class BaseLlmFlow(ABC):
|
||||
|
||||
# Handle transcription events ONCE per llm_response, outside the event loop
|
||||
if llm_response.input_transcription:
|
||||
input_transcription_event = (
|
||||
await self.transcription_manager.handle_input_transcription(
|
||||
invocation_context, llm_response.input_transcription
|
||||
)
|
||||
model_response_event.input_transcription = (
|
||||
llm_response.input_transcription
|
||||
)
|
||||
yield input_transcription_event
|
||||
model_response_event.partial = llm_response.partial
|
||||
yield model_response_event
|
||||
return
|
||||
|
||||
if llm_response.output_transcription:
|
||||
output_transcription_event = (
|
||||
await self.transcription_manager.handle_output_transcription(
|
||||
invocation_context, llm_response.output_transcription
|
||||
)
|
||||
model_response_event.output_transcription = (
|
||||
llm_response.output_transcription
|
||||
)
|
||||
yield output_transcription_event
|
||||
model_response_event.partial = llm_response.partial
|
||||
yield model_response_event
|
||||
return
|
||||
|
||||
# Flush audio caches based on control events using configurable settings
|
||||
|
||||
@@ -35,6 +35,8 @@ class GeminiLlmConnection(BaseLlmConnection):
|
||||
|
||||
def __init__(self, gemini_session: live.AsyncSession):
|
||||
self._gemini_session = gemini_session
|
||||
self._input_transcription_text: str = ''
|
||||
self._output_transcription_text: str = ''
|
||||
|
||||
async def send_history(self, history: list[types.Content]):
|
||||
"""Sends the conversation history to the gemini model.
|
||||
@@ -166,15 +168,49 @@ class GeminiLlmConnection(BaseLlmConnection):
|
||||
text = ''
|
||||
yield llm_response
|
||||
if message.server_content.input_transcription:
|
||||
llm_response = LlmResponse(
|
||||
input_transcription=message.server_content.input_transcription,
|
||||
)
|
||||
yield llm_response
|
||||
if message.server_content.input_transcription.text:
|
||||
self._input_transcription_text += (
|
||||
message.server_content.input_transcription.text
|
||||
)
|
||||
yield LlmResponse(
|
||||
input_transcription=types.Transcription(
|
||||
text=message.server_content.input_transcription.text,
|
||||
finished=False,
|
||||
),
|
||||
partial=True,
|
||||
)
|
||||
# finished=True and partial transcription may happen in the same
|
||||
# message.
|
||||
if message.server_content.input_transcription.finished:
|
||||
yield LlmResponse(
|
||||
input_transcription=types.Transcription(
|
||||
text=self._input_transcription_text,
|
||||
finished=True,
|
||||
),
|
||||
partial=False,
|
||||
)
|
||||
self._input_transcription_text = ''
|
||||
if message.server_content.output_transcription:
|
||||
llm_response = LlmResponse(
|
||||
output_transcription=message.server_content.output_transcription
|
||||
)
|
||||
yield llm_response
|
||||
if message.server_content.output_transcription.text:
|
||||
self._output_transcription_text += (
|
||||
message.server_content.output_transcription.text
|
||||
)
|
||||
yield LlmResponse(
|
||||
output_transcription=types.Transcription(
|
||||
text=message.server_content.output_transcription.text,
|
||||
finished=False,
|
||||
),
|
||||
partial=True,
|
||||
)
|
||||
if message.server_content.output_transcription.finished:
|
||||
yield LlmResponse(
|
||||
output_transcription=types.Transcription(
|
||||
text=self._output_transcription_text,
|
||||
finished=True,
|
||||
),
|
||||
partial=False,
|
||||
)
|
||||
self._output_transcription_text = ''
|
||||
if message.server_content.turn_complete:
|
||||
if text:
|
||||
yield self.__build_full_text_response(text)
|
||||
@@ -188,10 +224,12 @@ class GeminiLlmConnection(BaseLlmConnection):
|
||||
# in case it's an interrupted message, we merge the previous partial
|
||||
# text. Other we don't merge. because content can be none when model
|
||||
# safety threshold is triggered
|
||||
if message.server_content.interrupted and text:
|
||||
yield self.__build_full_text_response(text)
|
||||
text = ''
|
||||
yield LlmResponse(interrupted=message.server_content.interrupted)
|
||||
if message.server_content.interrupted:
|
||||
if text:
|
||||
yield self.__build_full_text_response(text)
|
||||
text = ''
|
||||
else:
|
||||
yield LlmResponse(interrupted=message.server_content.interrupted)
|
||||
if message.tool_call:
|
||||
if text:
|
||||
yield self.__build_full_text_response(text)
|
||||
|
||||
@@ -588,6 +588,10 @@ class Runner:
|
||||
# Don't append audio response from model in live mode to session.
|
||||
# The data is appended to artifacts with a reference in file_data in the
|
||||
# event.
|
||||
# We should append non-partial events only.For example, non-finished(partial)
|
||||
# transcription events should not be appended.
|
||||
# Function call and function response events should be appended.
|
||||
# Other control events should be appended.
|
||||
if is_live_call and contents._is_live_model_audio_event(event):
|
||||
return False
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user