feat: add file-backed artifact service

- add FileArtifactService that persists artifacts to the local filesystem
- adjust BaseArtifactService and exports so callers can wire in the filebacked implementation

Co-authored-by: George Weale <gweale@google.com>
PiperOrigin-RevId: 828629298
This commit is contained in:
George Weale
2025-11-05 14:26:51 -08:00
committed by Copybara-Service
parent d9ec07d39b
commit 99ca6aa6e6
4 changed files with 1040 additions and 41 deletions
+2
View File
@@ -13,11 +13,13 @@
# limitations under the License.
from .base_artifact_service import BaseArtifactService
from .file_artifact_service import FileArtifactService
from .gcs_artifact_service import GcsArtifactService
from .in_memory_artifact_service import InMemoryArtifactService
__all__ = [
'BaseArtifactService',
'FileArtifactService',
'GcsArtifactService',
'InMemoryArtifactService',
]
@@ -20,23 +20,44 @@ from typing import Any
from typing import Optional
from google.genai import types
from pydantic import alias_generators
from pydantic import BaseModel
from pydantic import ConfigDict
from pydantic import Field
class ArtifactVersion(BaseModel):
"""Represents the metadata of a specific version of an artifact."""
"""Metadata describing a specific version of an artifact."""
version: int
"""The version number of the artifact."""
canonical_uri: str
"""The canonical URI of the artifact version."""
custom_metadata: dict[str, Any] = Field(default_factory=dict)
"""A dictionary of custom metadata associated with the artifact version."""
create_time: float = Field(default_factory=lambda: datetime.now().timestamp())
"""The creation time of the artifact version."""
mime_type: Optional[str] = None
"""The MIME type of the artifact version."""
model_config = ConfigDict(
alias_generator=alias_generators.to_camel,
populate_by_name=True,
)
version: int = Field(
description=(
"Monotonically increasing identifier for the artifact version."
)
)
canonical_uri: str = Field(
description="Canonical URI referencing the persisted artifact payload."
)
custom_metadata: dict[str, Any] = Field(
default_factory=dict,
description="Optional user-supplied metadata stored with the artifact.",
)
create_time: float = Field(
default_factory=lambda: datetime.now().timestamp(),
description=(
"Unix timestamp (seconds) when the version record was created."
),
)
mime_type: Optional[str] = Field(
default=None,
description=(
"MIME type when the artifact payload is stored as binary data."
),
)
class BaseArtifactService(ABC):
File diff suppressed because it is too large Load Diff
@@ -12,17 +12,24 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# pylint: disable=missing-class-docstring,missing-function-docstring
"""Tests for the artifact service."""
from datetime import datetime
import enum
import json
from pathlib import Path
from typing import Any
from typing import Optional
from typing import Union
from unittest import mock
from unittest.mock import patch
from urllib.parse import unquote
from urllib.parse import urlparse
from google.adk.artifacts.base_artifact_service import ArtifactVersion
from google.adk.artifacts.file_artifact_service import FileArtifactService
from google.adk.artifacts.gcs_artifact_service import GcsArtifactService
from google.adk.artifacts.in_memory_artifact_service import InMemoryArtifactService
from google.genai import types
@@ -35,6 +42,7 @@ FIXED_DATETIME = datetime(2025, 1, 1, 12, 0, 0)
class ArtifactServiceType(Enum):
FILE = "FILE"
IN_MEMORY = "IN_MEMORY"
GCS = "GCS"
@@ -159,22 +167,34 @@ def mock_gcs_artifact_service():
return GcsArtifactService(bucket_name="test_bucket")
def get_artifact_service(
service_type: ArtifactServiceType = ArtifactServiceType.IN_MEMORY,
):
"""Creates an artifact service for testing."""
if service_type == ArtifactServiceType.GCS:
return mock_gcs_artifact_service()
return InMemoryArtifactService()
@pytest.fixture
def artifact_service_factory(tmp_path: Path):
"""Provides an artifact service constructor bound to the test tmp path."""
def factory(
service_type: ArtifactServiceType = ArtifactServiceType.IN_MEMORY,
):
if service_type == ArtifactServiceType.GCS:
return mock_gcs_artifact_service()
if service_type == ArtifactServiceType.FILE:
return FileArtifactService(root_dir=tmp_path / "artifacts")
return InMemoryArtifactService()
return factory
@pytest.mark.asyncio
@pytest.mark.parametrize(
"service_type", [ArtifactServiceType.IN_MEMORY, ArtifactServiceType.GCS]
"service_type",
[
ArtifactServiceType.IN_MEMORY,
ArtifactServiceType.GCS,
ArtifactServiceType.FILE,
],
)
async def test_load_empty(service_type):
async def test_load_empty(service_type, artifact_service_factory):
"""Tests loading an artifact when none exists."""
artifact_service = get_artifact_service(service_type)
artifact_service = artifact_service_factory(service_type)
assert not await artifact_service.load_artifact(
app_name="test_app",
user_id="test_user",
@@ -185,11 +205,16 @@ async def test_load_empty(service_type):
@pytest.mark.asyncio
@pytest.mark.parametrize(
"service_type", [ArtifactServiceType.IN_MEMORY, ArtifactServiceType.GCS]
"service_type",
[
ArtifactServiceType.IN_MEMORY,
ArtifactServiceType.GCS,
ArtifactServiceType.FILE,
],
)
async def test_save_load_delete(service_type):
async def test_save_load_delete(service_type, artifact_service_factory):
"""Tests saving, loading, and deleting an artifact."""
artifact_service = get_artifact_service(service_type)
artifact_service = artifact_service_factory(service_type)
artifact = types.Part.from_bytes(data=b"test_data", mime_type="text/plain")
app_name = "app0"
user_id = "user0"
@@ -238,11 +263,16 @@ async def test_save_load_delete(service_type):
@pytest.mark.asyncio
@pytest.mark.parametrize(
"service_type", [ArtifactServiceType.IN_MEMORY, ArtifactServiceType.GCS]
"service_type",
[
ArtifactServiceType.IN_MEMORY,
ArtifactServiceType.GCS,
ArtifactServiceType.FILE,
],
)
async def test_list_keys(service_type):
async def test_list_keys(service_type, artifact_service_factory):
"""Tests listing keys in the artifact service."""
artifact_service = get_artifact_service(service_type)
artifact_service = artifact_service_factory(service_type)
artifact = types.Part.from_bytes(data=b"test_data", mime_type="text/plain")
app_name = "app0"
user_id = "user0"
@@ -269,11 +299,16 @@ async def test_list_keys(service_type):
@pytest.mark.asyncio
@pytest.mark.parametrize(
"service_type", [ArtifactServiceType.IN_MEMORY, ArtifactServiceType.GCS]
"service_type",
[
ArtifactServiceType.IN_MEMORY,
ArtifactServiceType.GCS,
ArtifactServiceType.FILE,
],
)
async def test_list_versions(service_type):
async def test_list_versions(service_type, artifact_service_factory):
"""Tests listing versions of an artifact."""
artifact_service = get_artifact_service(service_type)
artifact_service = artifact_service_factory(service_type)
app_name = "app0"
user_id = "user0"
@@ -308,11 +343,18 @@ async def test_list_versions(service_type):
@pytest.mark.asyncio
@pytest.mark.parametrize(
"service_type", [ArtifactServiceType.IN_MEMORY, ArtifactServiceType.GCS]
"service_type",
[
ArtifactServiceType.IN_MEMORY,
ArtifactServiceType.GCS,
ArtifactServiceType.FILE,
],
)
async def test_list_keys_preserves_user_prefix(service_type):
async def test_list_keys_preserves_user_prefix(
service_type, artifact_service_factory
):
"""Tests that list_artifact_keys preserves 'user:' prefix in returned names."""
artifact_service = get_artifact_service(service_type)
artifact_service = artifact_service_factory(service_type)
artifact = types.Part.from_bytes(data=b"test_data", mime_type="text/plain")
app_name = "app0"
user_id = "user0"
@@ -358,9 +400,11 @@ async def test_list_keys_preserves_user_prefix(service_type):
@pytest.mark.parametrize(
"service_type", [ArtifactServiceType.IN_MEMORY, ArtifactServiceType.GCS]
)
async def test_list_artifact_versions_and_get_artifact_version(service_type):
async def test_list_artifact_versions_and_get_artifact_version(
service_type, artifact_service_factory
):
"""Tests listing artifact versions and getting a specific version."""
artifact_service = get_artifact_service(service_type)
artifact_service = artifact_service_factory(service_type)
app_name = "app0"
user_id = "user0"
session_id = "123"
@@ -443,9 +487,11 @@ async def test_list_artifact_versions_and_get_artifact_version(service_type):
@pytest.mark.parametrize(
"service_type", [ArtifactServiceType.IN_MEMORY, ArtifactServiceType.GCS]
)
async def test_list_artifact_versions_with_user_prefix(service_type):
async def test_list_artifact_versions_with_user_prefix(
service_type, artifact_service_factory
):
"""Tests listing artifact versions with user prefix."""
artifact_service = get_artifact_service(service_type)
artifact_service = artifact_service_factory(service_type)
app_name = "app0"
user_id = "user0"
session_id = "123"
@@ -504,9 +550,11 @@ async def test_list_artifact_versions_with_user_prefix(service_type):
@pytest.mark.parametrize(
"service_type", [ArtifactServiceType.IN_MEMORY, ArtifactServiceType.GCS]
)
async def test_get_artifact_version_artifact_does_not_exist(service_type):
async def test_get_artifact_version_artifact_does_not_exist(
service_type, artifact_service_factory
):
"""Tests getting an artifact version when artifact does not exist."""
artifact_service = get_artifact_service(service_type)
artifact_service = artifact_service_factory(service_type)
assert not await artifact_service.get_artifact_version(
app_name="test_app",
user_id="test_user",
@@ -519,9 +567,11 @@ async def test_get_artifact_version_artifact_does_not_exist(service_type):
@pytest.mark.parametrize(
"service_type", [ArtifactServiceType.IN_MEMORY, ArtifactServiceType.GCS]
)
async def test_get_artifact_version_out_of_index(service_type):
async def test_get_artifact_version_out_of_index(
service_type, artifact_service_factory
):
"""Tests loading an artifact with an out-of-index version."""
artifact_service = get_artifact_service(service_type)
artifact_service = artifact_service_factory(service_type)
app_name = "app0"
user_id = "user0"
session_id = "123"
@@ -544,3 +594,178 @@ async def test_get_artifact_version_out_of_index(service_type):
filename=filename,
version=3,
)
@pytest.mark.asyncio
async def test_file_metadata_camelcase(tmp_path, artifact_service_factory):
"""Ensures FileArtifactService writes camelCase metadata without newlines."""
artifact_service = artifact_service_factory(ArtifactServiceType.FILE)
artifact = types.Part.from_bytes(
data=b"binary-content", mime_type="application/octet-stream"
)
await artifact_service.save_artifact(
app_name="myapp",
user_id="user123",
session_id="sess789",
filename="docs/report.txt",
artifact=artifact,
)
metadata_path = (
tmp_path
/ "artifacts"
/ "apps"
/ "myapp"
/ "users"
/ "user123"
/ "sessions"
/ "sess789"
/ "artifacts"
/ "docs"
/ "report.txt"
/ "versions"
/ "0"
/ "metadata.json"
)
raw_metadata = metadata_path.read_text(encoding="utf-8")
assert "\n" not in raw_metadata
metadata = json.loads(raw_metadata)
payload_path = (metadata_path.parent / "report.txt").resolve()
expected_canonical_uri = payload_path.as_uri()
create_time = metadata.pop("createTime", None)
assert create_time is not None
assert metadata == {
"fileName": "docs/report.txt",
"mimeType": "application/octet-stream",
"canonicalUri": expected_canonical_uri,
"version": 0,
"customMetadata": {},
}
parsed_canonical = urlparse(metadata["canonicalUri"])
canonical_path = Path(unquote(parsed_canonical.path))
assert canonical_path.name == "report.txt"
assert canonical_path.read_bytes() == b"binary-content"
@pytest.mark.asyncio
async def test_file_list_artifact_versions(tmp_path, artifact_service_factory):
"""FileArtifactService exposes canonical URIs and metadata for each version."""
artifact_service = artifact_service_factory(ArtifactServiceType.FILE)
artifact = types.Part.from_bytes(
data=b"binary-content", mime_type="application/octet-stream"
)
custom_metadata = {"origin": "unit-test"}
await artifact_service.save_artifact(
app_name="myapp",
user_id="user123",
session_id="sess789",
filename="docs/report.txt",
artifact=artifact,
custom_metadata=custom_metadata,
)
versions = await artifact_service.list_artifact_versions(
app_name="myapp",
user_id="user123",
session_id="sess789",
filename="docs/report.txt",
)
assert len(versions) == 1
version_meta = versions[0]
assert version_meta.version == 0
version_payload_path = (
tmp_path
/ "artifacts"
/ "apps"
/ "myapp"
/ "users"
/ "user123"
/ "sessions"
/ "sess789"
/ "artifacts"
/ "docs"
/ "report.txt"
/ "versions"
/ "0"
/ "report.txt"
).resolve()
assert version_meta.canonical_uri == version_payload_path.as_uri()
assert version_meta.custom_metadata == custom_metadata
parsed_version_uri = urlparse(version_meta.canonical_uri)
version_uri_path = Path(unquote(parsed_version_uri.path))
assert version_uri_path.read_bytes() == b"binary-content"
fetched = await artifact_service.get_artifact_version(
app_name="myapp",
user_id="user123",
session_id="sess789",
filename="docs/report.txt",
version=0,
)
assert fetched is not None
assert fetched.version == version_meta.version
assert fetched.canonical_uri == version_meta.canonical_uri
assert fetched.custom_metadata == version_meta.custom_metadata
latest = await artifact_service.get_artifact_version(
app_name="myapp",
user_id="user123",
session_id="sess789",
filename="docs/report.txt",
)
assert latest is not None
assert latest.version == version_meta.version
assert latest.canonical_uri == version_meta.canonical_uri
assert latest.custom_metadata == version_meta.custom_metadata
@pytest.mark.asyncio
@pytest.mark.parametrize(
("filename", "session_id"),
[
("../escape.txt", "sess123"),
("user:../escape.txt", "sess123"),
("/absolute/path.txt", "sess123"),
("user:/absolute/path.txt", None),
],
)
async def test_file_save_artifact_rejects_out_of_scope_paths(
tmp_path, filename, session_id
):
"""FileArtifactService prevents path traversal outside of its storage roots."""
artifact_service = FileArtifactService(root_dir=tmp_path / "artifacts")
part = types.Part(text="content")
with pytest.raises(ValueError):
await artifact_service.save_artifact(
app_name="myapp",
user_id="user123",
session_id=session_id,
filename=filename,
artifact=part,
)
@pytest.mark.asyncio
async def test_file_save_artifact_rejects_absolute_path_within_scope(tmp_path):
"""Absolute filenames are rejected even when they point inside the scope."""
artifact_service = FileArtifactService(root_dir=tmp_path / "artifacts")
absolute_in_scope = (
tmp_path
/ "artifacts"
/ "apps"
/ "myapp"
/ "users"
/ "user123"
/ "artifacts"
/ "diagram.png"
)
part = types.Part(text="content")
with pytest.raises(ValueError):
await artifact_service.save_artifact(
app_name="myapp",
user_id="user123",
session_id=None,
filename=str(absolute_in_scope),
artifact=part,
)