fix: Improve handling of partial and complete transcriptions in live calls

In `gemini_llm_connection.py`, accumulate partial transcription texts and emit `LlmResponse` with `partial=True` for each chunk. When the transcription is marked as `finished`, emit a final `LlmResponse` with the full accumulated text and `partial=False`.

In `runners.py`, modify `_should_append_to_history` to only add transcription events to the history when they are fully finished, preventing partial transcriptions from being added.

Co-authored-by: Hangfei Lin <hangfei@google.com>
PiperOrigin-RevId: 829029715
This commit is contained in:
Hangfei Lin
2025-11-06 11:09:21 -08:00
committed by Copybara-Service
parent 44d45fe9cd
commit 1819ecb4b8
5 changed files with 278 additions and 22 deletions
@@ -587,21 +587,19 @@ class BaseLlmFlow(ABC):
# Handle transcription events ONCE per llm_response, outside the event loop
if llm_response.input_transcription:
input_transcription_event = (
await self.transcription_manager.handle_input_transcription(
invocation_context, llm_response.input_transcription
)
model_response_event.input_transcription = (
llm_response.input_transcription
)
yield input_transcription_event
model_response_event.partial = llm_response.partial
yield model_response_event
return
if llm_response.output_transcription:
output_transcription_event = (
await self.transcription_manager.handle_output_transcription(
invocation_context, llm_response.output_transcription
)
model_response_event.output_transcription = (
llm_response.output_transcription
)
yield output_transcription_event
model_response_event.partial = llm_response.partial
yield model_response_event
return
# Flush audio caches based on control events using configurable settings
+50 -12
View File
@@ -35,6 +35,8 @@ class GeminiLlmConnection(BaseLlmConnection):
def __init__(self, gemini_session: live.AsyncSession):
self._gemini_session = gemini_session
self._input_transcription_text: str = ''
self._output_transcription_text: str = ''
async def send_history(self, history: list[types.Content]):
"""Sends the conversation history to the gemini model.
@@ -166,15 +168,49 @@ class GeminiLlmConnection(BaseLlmConnection):
text = ''
yield llm_response
if message.server_content.input_transcription:
llm_response = LlmResponse(
input_transcription=message.server_content.input_transcription,
)
yield llm_response
if message.server_content.input_transcription.text:
self._input_transcription_text += (
message.server_content.input_transcription.text
)
yield LlmResponse(
input_transcription=types.Transcription(
text=message.server_content.input_transcription.text,
finished=False,
),
partial=True,
)
# finished=True and partial transcription may happen in the same
# message.
if message.server_content.input_transcription.finished:
yield LlmResponse(
input_transcription=types.Transcription(
text=self._input_transcription_text,
finished=True,
),
partial=False,
)
self._input_transcription_text = ''
if message.server_content.output_transcription:
llm_response = LlmResponse(
output_transcription=message.server_content.output_transcription
)
yield llm_response
if message.server_content.output_transcription.text:
self._output_transcription_text += (
message.server_content.output_transcription.text
)
yield LlmResponse(
output_transcription=types.Transcription(
text=message.server_content.output_transcription.text,
finished=False,
),
partial=True,
)
if message.server_content.output_transcription.finished:
yield LlmResponse(
output_transcription=types.Transcription(
text=self._output_transcription_text,
finished=True,
),
partial=False,
)
self._output_transcription_text = ''
if message.server_content.turn_complete:
if text:
yield self.__build_full_text_response(text)
@@ -188,10 +224,12 @@ class GeminiLlmConnection(BaseLlmConnection):
# in case it's an interrupted message, we merge the previous partial
# text. Other we don't merge. because content can be none when model
# safety threshold is triggered
if message.server_content.interrupted and text:
yield self.__build_full_text_response(text)
text = ''
yield LlmResponse(interrupted=message.server_content.interrupted)
if message.server_content.interrupted:
if text:
yield self.__build_full_text_response(text)
text = ''
else:
yield LlmResponse(interrupted=message.server_content.interrupted)
if message.tool_call:
if text:
yield self.__build_full_text_response(text)
+4
View File
@@ -588,6 +588,10 @@ class Runner:
# Don't append audio response from model in live mode to session.
# The data is appended to artifacts with a reference in file_data in the
# event.
# We should append non-partial events only.For example, non-finished(partial)
# transcription events should not be appended.
# Function call and function response events should be appended.
# Other control events should be appended.
if is_live_call and contents._is_live_model_audio_event(event):
return False
return True