fix: change LiteLLM content and tool parameter handling

This changes how content parts are converted for LiteLLM, treating all "text/" mime types as plain text and only "application/pdf" as a file

Close #1940

Co-authored-by: George Weale <gweale@google.com>
PiperOrigin-RevId: 831418696
This commit is contained in:
George Weale
2025-11-12 09:23:00 -08:00
committed by Copybara-Service
parent e2d3b2d862
commit a19be12c1f
2 changed files with 55 additions and 8 deletions
+23 -2
View File
@@ -81,10 +81,19 @@ _FINISH_REASON_MAPPING = {
}
_SUPPORTED_FILE_CONTENT_MIME_TYPES = set(
["application/pdf", "application/json", "text/plain"]
["application/pdf", "application/json"]
)
def _decode_inline_text_data(raw_bytes: bytes) -> str:
"""Decodes inline file bytes that represent textual content."""
try:
return raw_bytes.decode("utf-8")
except UnicodeDecodeError:
logger.debug("Falling back to latin-1 decoding for inline file bytes.")
return raw_bytes.decode("latin-1", errors="replace")
class ChatCompletionFileUrlObject(TypedDict, total=False):
file_data: str
file_id: str
@@ -371,6 +380,15 @@ def _get_content(
and part.inline_data.data
and part.inline_data.mime_type
):
if part.inline_data.mime_type.startswith("text/"):
decoded_text = _decode_inline_text_data(part.inline_data.data)
if len(parts) == 1:
return decoded_text
content_objects.append({
"type": "text",
"text": decoded_text,
})
continue
base64_string = base64.b64encode(part.inline_data.data).decode("utf-8")
data_uri = f"data:{part.inline_data.mime_type};base64,{base64_string}"
# LiteLLM providers extract the MIME type from the data URI; avoid
@@ -397,7 +415,10 @@ def _get_content(
"file": {"file_data": data_uri},
})
else:
raise ValueError("LiteLlm(BaseLlm) does not support this content part.")
raise ValueError(
"LiteLlm(BaseLlm) does not support content part with MIME type "
f"{part.inline_data.mime_type}."
)
elif part.file_data and part.file_data.file_uri:
file_object: ChatCompletionFileUrlObject = {
"file_id": part.file_data.file_uri,