feat: Re-factor some eval sets manager logic, and implement GcsEvalSetsManager to handle storage of eval sets on GCS

Eval sets will be stored as json files under `gs://{bucket_name}/{app_name}/evals/eval_sets/`

PiperOrigin-RevId: 770487129

src/google/adk/evaluation/_eval_sets_manager_utils.py

@@ -0,0 +1,108 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+from ..errors.not_found_error import NotFoundError
+from .eval_case import EvalCase
+from .eval_set import EvalSet
+from .eval_sets_manager import EvalSetsManager
+
+logger = logging.getLogger("google_adk." + __name__)
+
+
+def get_eval_set_from_app_and_id(
+    eval_sets_manager: EvalSetsManager, app_name: str, eval_set_id: str
+) -> EvalSet:
+  """Returns an EvalSet if found, otherwise raises NotFoundError."""
+  eval_set = eval_sets_manager.get_eval_set(app_name, eval_set_id)
+  if not eval_set:
+    raise NotFoundError(f"Eval set `{eval_set_id}` not found.")
+  return eval_set
+
+
+def get_eval_case_from_eval_set(
+    eval_set: EvalSet, eval_case_id: str
+) -> Optional[EvalCase]:
+  """Returns an EvalCase if found, otherwise None."""
+  eval_case_to_find = None
+
+  # Look up the eval case by eval_case_id
+  for eval_case in eval_set.eval_cases:
+    if eval_case.eval_id == eval_case_id:
+      eval_case_to_find = eval_case
+      break
+
+  return eval_case_to_find
+
+
+def add_eval_case_to_eval_set(
+    eval_set: EvalSet, eval_case: EvalCase
+) -> EvalSet:
+  """Adds an eval case to an eval set and returns the updated eval set."""
+  eval_case_id = eval_case.eval_id
+
+  if [x for x in eval_set.eval_cases if x.eval_id == eval_case_id]:
+    raise ValueError(
+        f"Eval id `{eval_case_id}` already exists in `{eval_set.eval_set_id}`"
+        " eval set.",
+    )
+
+  eval_set.eval_cases.append(eval_case)
+  return eval_set
+
+
+def update_eval_case_in_eval_set(
+    eval_set: EvalSet, updated_eval_case: EvalCase
+) -> EvalSet:
+  """Updates an eval case in an eval set and returns the updated eval set."""
+  # Find the eval case to be updated.
+  eval_case_id = updated_eval_case.eval_id
+  eval_case_to_update = get_eval_case_from_eval_set(eval_set, eval_case_id)
+
+  if not eval_case_to_update:
+    raise NotFoundError(
+        f"Eval case `{eval_case_id}` not found in eval set"
+        f" `{eval_set.eval_set_id}`."
+    )
+
+  # Remove the existing eval case and add the updated eval case.
+  eval_set.eval_cases.remove(eval_case_to_update)
+  eval_set.eval_cases.append(updated_eval_case)
+  return eval_set
+
+
+def delete_eval_case_from_eval_set(
+    eval_set: EvalSet, eval_case_id: str
+) -> EvalSet:
+  """Deletes an eval case from an eval set and returns the updated eval set."""
+  # Find the eval case to be deleted.
+  eval_case_to_delete = get_eval_case_from_eval_set(eval_set, eval_case_id)
+
+  if not eval_case_to_delete:
+    raise NotFoundError(
+        f"Eval case `{eval_case_id}` not found in eval set"
+        f" `{eval_set.eval_set_id}`."
+    )
+
+  # Remove the existing eval case.
+  logger.info(
+      "EvalCase`%s` was found in the eval set. It will be removed permanently.",
+      eval_case_id,
+  )
+  eval_set.eval_cases.remove(eval_case_to_delete)
+  return eval_set

src/google/adk/evaluation/gcs_eval_sets_manager.py

@@ -0,0 +1,196 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import logging
+import re
+import time
+from typing import Optional
+
+from google.cloud import exceptions as cloud_exceptions
+from google.cloud import storage
+from typing_extensions import override
+
+from ._eval_sets_manager_utils import add_eval_case_to_eval_set
+from ._eval_sets_manager_utils import delete_eval_case_from_eval_set
+from ._eval_sets_manager_utils import get_eval_case_from_eval_set
+from ._eval_sets_manager_utils import get_eval_set_from_app_and_id
+from ._eval_sets_manager_utils import update_eval_case_in_eval_set
+from .eval_case import EvalCase
+from .eval_set import EvalSet
+from .eval_sets_manager import EvalSetsManager
+
+logger = logging.getLogger("google_adk." + __name__)
+
+_EVAL_SETS_DIR = "evals/eval_sets"
+_EVAL_SET_FILE_EXTENSION = ".evalset.json"
+
+
+class GcsEvalSetsManager(EvalSetsManager):
+  """An EvalSetsManager that stores eval sets in a GCS bucket."""
+
+  def __init__(self, bucket_name: str, **kwargs):
+    """Initializes the GcsEvalSetsManager.
+
+    Args:
+      bucket_name: The name of the bucket to use.
+      **kwargs: Keyword arguments to pass to the Google Cloud Storage client.
+    """
+    self.bucket_name = bucket_name
+    self.storage_client = storage.Client(**kwargs)
+    self.bucket = self.storage_client.bucket(self.bucket_name)
+    # Check if the bucket exists.
+    if not self.bucket.exists():
+      raise ValueError(
+          f"Bucket `{self.bucket_name}` does not exist. Please create it "
+          "before using the GcsEvalSetsManager."
+      )
+
+  def _get_eval_sets_dir(self, app_name: str) -> str:
+    return f"{app_name}/{_EVAL_SETS_DIR}"
+
+  def _get_eval_set_blob_name(self, app_name: str, eval_set_id: str) -> str:
+    eval_sets_dir = self._get_eval_sets_dir(app_name)
+    return f"{eval_sets_dir}/{eval_set_id}{_EVAL_SET_FILE_EXTENSION}"
+
+  def _validate_id(self, id_name: str, id_value: str):
+    pattern = r"^[a-zA-Z0-9_]+$"
+    if not bool(re.fullmatch(pattern, id_value)):
+      raise ValueError(
+          f"Invalid {id_name}. {id_name} should have the `{pattern}` format",
+      )
+
+  def _write_eval_set_to_blob(self, blob_name: str, eval_set: EvalSet):
+    """Writes an EvalSet to GCS."""
+    blob = self.bucket.blob(blob_name)
+    blob.upload_from_string(
+        eval_set.model_dump_json(indent=2),
+        content_type="application/json",
+    )
+
+  def _save_eval_set(self, app_name: str, eval_set_id: str, eval_set: EvalSet):
+    eval_set_blob_name = self._get_eval_set_blob_name(app_name, eval_set_id)
+    self._write_eval_set_to_blob(eval_set_blob_name, eval_set)
+
+  @override
+  def get_eval_set(self, app_name: str, eval_set_id: str) -> Optional[EvalSet]:
+    """Returns an EvalSet identified by an app_name and eval_set_id."""
+    eval_set_blob_name = self._get_eval_set_blob_name(app_name, eval_set_id)
+    blob = self.bucket.blob(eval_set_blob_name)
+    if not blob.exists():
+      return None
+    eval_set_data = blob.download_as_text()
+    return EvalSet.model_validate_json(eval_set_data)
+
+  @override
+  def create_eval_set(self, app_name: str, eval_set_id: str):
+    """Creates an empty EvalSet and saves it to GCS."""
+    self._validate_id(id_name="Eval Set Id", id_value=eval_set_id)
+    new_eval_set_blob_name = self._get_eval_set_blob_name(app_name, eval_set_id)
+    if self.bucket.blob(new_eval_set_blob_name).exists():
+      raise ValueError(
+          f"Eval set `{eval_set_id}` already exists for app `{app_name}`."
+      )
+    logger.info("Creating eval set blob: `%s`", new_eval_set_blob_name)
+    new_eval_set = EvalSet(
+        eval_set_id=eval_set_id,
+        name=eval_set_id,
+        eval_cases=[],
+        creation_timestamp=time.time(),
+    )
+    self._write_eval_set_to_blob(new_eval_set_blob_name, new_eval_set)
+
+  @override
+  def list_eval_sets(self, app_name: str) -> list[str]:
+    """Returns a list of EvalSet ids that belong to the given app_name."""
+    eval_sets_dir = self._get_eval_sets_dir(app_name)
+    eval_sets = []
+    try:
+      for blob in self.bucket.list_blobs(prefix=eval_sets_dir):
+        if not blob.name.endswith(_EVAL_SET_FILE_EXTENSION):
+          continue
+        eval_set_id = blob.name.split("/")[-1].removesuffix(
+            _EVAL_SET_FILE_EXTENSION
+        )
+        eval_sets.append(eval_set_id)
+      return sorted(eval_sets)
+    except cloud_exceptions.NotFound as e:
+      raise ValueError(
+          f"App `{app_name}` not found in GCS bucket `{self.bucket_name}`."
+      ) from e
+
+  @override
+  def get_eval_case(
+      self, app_name: str, eval_set_id: str, eval_case_id: str
+  ) -> Optional[EvalCase]:
+    """Returns an EvalCase identified by an app_name, eval_set_id and eval_case_id."""
+    eval_set = self.get_eval_set(app_name, eval_set_id)
+    if not eval_set:
+      return None
+    return get_eval_case_from_eval_set(eval_set, eval_case_id)
+
+  @override
+  def add_eval_case(self, app_name: str, eval_set_id: str, eval_case: EvalCase):
+    """Adds the given EvalCase to an existing EvalSet.
+
+    Args:
+      app_name: The name of the app.
+      eval_set_id: The id of the eval set containing the eval case to update.
+      eval_case: The EvalCase to add.
+
+    Raises:
+      NotFoundError: If the eval set is not found.
+      ValueError: If the eval case already exists in the eval set.
+    """
+    eval_set = get_eval_set_from_app_and_id(self, app_name, eval_set_id)
+    updated_eval_set = add_eval_case_to_eval_set(eval_set, eval_case)
+    self._save_eval_set(app_name, eval_set_id, updated_eval_set)
+
+  @override
+  def update_eval_case(
+      self, app_name: str, eval_set_id: str, updated_eval_case: EvalCase
+  ):
+    """Updates an existing EvalCase.
+
+    Args:
+      app_name: The name of the app.
+      eval_set_id: The id of the eval set containing the eval case to update.
+      updated_eval_case: The updated EvalCase. Overwrites the existing EvalCase
+        using the eval_id field.
+
+    Raises:
+      NotFoundError: If the eval set or the eval case is not found.
+    """
+    eval_set = get_eval_set_from_app_and_id(self, app_name, eval_set_id)
+    updated_eval_set = update_eval_case_in_eval_set(eval_set, updated_eval_case)
+    self._save_eval_set(app_name, eval_set_id, updated_eval_set)
+
+  @override
+  def delete_eval_case(
+      self, app_name: str, eval_set_id: str, eval_case_id: str
+  ):
+    """Deletes the EvalCase with the given eval_case_id from the given EvalSet.
+
+    Args:
+      app_name: The name of the app.
+      eval_set_id: The id of the eval set containing the eval case to delete.
+      eval_case_id: The id of the eval case to delete.
+
+    Raises:
+      NotFoundError: If the eval set or the eval case to delete is not found.
+    """
+    eval_set = get_eval_set_from_app_and_id(self, app_name, eval_set_id)
+    updated_eval_set = delete_eval_case_from_eval_set(eval_set, eval_case_id)
+    self._save_eval_set(app_name, eval_set_id, updated_eval_set)

src/google/adk/evaluation/local_eval_sets_manager.py

@@ -27,7 +27,11 @@ from google.genai import types as genai_types
 from pydantic import ValidationError
 from typing_extensions import override
 
-from ..errors.not_found_error import NotFoundError
+from ._eval_sets_manager_utils import add_eval_case_to_eval_set
+from ._eval_sets_manager_utils import delete_eval_case_from_eval_set
+from ._eval_sets_manager_utils import get_eval_case_from_eval_set
+from ._eval_sets_manager_utils import get_eval_set_from_app_and_id
+from ._eval_sets_manager_utils import update_eval_case_in_eval_set
 from .eval_case import EvalCase
 from .eval_case import IntermediateData
 from .eval_case import Invocation
@@ -218,7 +222,7 @@ class LocalEvalSetsManager(EvalSetsManager):
           eval_cases=[],
           creation_timestamp=time.time(),
       )
-      self._write_eval_set(new_eval_set_path, new_eval_set)
+      self._write_eval_set_to_path(new_eval_set_path, new_eval_set)
 
   @override
   def list_eval_sets(self, app_name: str) -> list[str]:
@@ -233,6 +237,16 @@ class LocalEvalSetsManager(EvalSetsManager):
 
     return sorted(eval_sets)
 
+  @override
+  def get_eval_case(
+      self, app_name: str, eval_set_id: str, eval_case_id: str
+  ) -> Optional[EvalCase]:
+    """Returns an EvalCase if found, otherwise None."""
+    eval_set = self.get_eval_set(app_name, eval_set_id)
+    if not eval_set:
+      return None
+    return get_eval_case_from_eval_set(eval_set, eval_case_id)
+
   @override
   def add_eval_case(self, app_name: str, eval_set_id: str, eval_case: EvalCase):
     """Adds the given EvalCase to an existing EvalSet identified by app_name and eval_set_id.
@@ -240,44 +254,10 @@ class LocalEvalSetsManager(EvalSetsManager):
     Raises:
       NotFoundError: If the eval set is not found.
     """
-    eval_case_id = eval_case.eval_id
-    self._validate_id(id_name="Eval Case Id", id_value=eval_case_id)
+    eval_set = get_eval_set_from_app_and_id(self, app_name, eval_set_id)
+    updated_eval_set = add_eval_case_to_eval_set(eval_set, eval_case)
 
-    eval_set = self.get_eval_set(app_name, eval_set_id)
-
-    if not eval_set:
-      raise NotFoundError(f"Eval set `{eval_set_id}` not found.")
-
-    if [x for x in eval_set.eval_cases if x.eval_id == eval_case_id]:
-      raise ValueError(
-          f"Eval id `{eval_case_id}` already exists in `{eval_set_id}`"
-          " eval set.",
-      )
-
-    eval_set.eval_cases.append(eval_case)
-
-    eval_set_file_path = self._get_eval_set_file_path(app_name, eval_set_id)
-    self._write_eval_set(eval_set_file_path, eval_set)
-
-  @override
-  def get_eval_case(
-      self, app_name: str, eval_set_id: str, eval_case_id: str
-  ) -> Optional[EvalCase]:
-    """Returns an EvalCase if found, otherwise None."""
-    eval_set = self.get_eval_set(app_name, eval_set_id)
-
-    if not eval_set:
-      return None
-
-    eval_case_to_find = None
-
-    # Look up the eval case by eval_case_id
-    for eval_case in eval_set.eval_cases:
-      if eval_case.eval_id == eval_case_id:
-        eval_case_to_find = eval_case
-        break
-
-    return eval_case_to_find
+    self._save_eval_set(app_name, eval_set_id, updated_eval_set)
 
   @override
   def update_eval_case(
@@ -288,28 +268,9 @@ class LocalEvalSetsManager(EvalSetsManager):
     Raises:
       NotFoundError: If the eval set or the eval case is not found.
     """
-    eval_case_id = updated_eval_case.eval_id
-
-    # Find the eval case to be updated.
-    eval_case_to_update = self.get_eval_case(
-        app_name, eval_set_id, eval_case_id
-    )
-
-    if eval_case_to_update:
-      # Remove the eval case from the existing eval set.
-      eval_set = self.get_eval_set(app_name, eval_set_id)
-      eval_set.eval_cases.remove(eval_case_to_update)
-
-      # Add the updated eval case to the existing eval set.
-      eval_set.eval_cases.append(updated_eval_case)
-
-      # Persit the eval set.
-      eval_set_file_path = self._get_eval_set_file_path(app_name, eval_set_id)
-      self._write_eval_set(eval_set_file_path, eval_set)
-    else:
-      raise NotFoundError(
-          f"Eval Set `{eval_set_id}` or Eval id `{eval_case_id}` not found.",
-      )
+    eval_set = get_eval_set_from_app_and_id(self, app_name, eval_set_id)
+    updated_eval_set = update_eval_case_in_eval_set(eval_set, updated_eval_case)
+    self._save_eval_set(app_name, eval_set_id, updated_eval_set)
 
   @override
   def delete_eval_case(
@@ -320,25 +281,9 @@ class LocalEvalSetsManager(EvalSetsManager):
     Raises:
       NotFoundError: If the eval set or the eval case to delete is not found.
     """
-    # Find the eval case that needs to be deleted.
-    eval_case_to_remove = self.get_eval_case(
-        app_name, eval_set_id, eval_case_id
-    )
-
-    if eval_case_to_remove:
-      logger.info(
-          "EvalCase`%s` was found in the eval set. It will be removed"
-          " permanently.",
-          eval_case_id,
-      )
-      eval_set = self.get_eval_set(app_name, eval_set_id)
-      eval_set.eval_cases.remove(eval_case_to_remove)
-      eval_set_file_path = self._get_eval_set_file_path(app_name, eval_set_id)
-      self._write_eval_set(eval_set_file_path, eval_set)
-    else:
-      raise NotFoundError(
-          f"Eval Set `{eval_set_id}` or Eval id `{eval_case_id}` not found.",
-      )
+    eval_set = get_eval_set_from_app_and_id(self, app_name, eval_set_id)
+    updated_eval_set = delete_eval_case_from_eval_set(eval_set, eval_case_id)
+    self._save_eval_set(app_name, eval_set_id, updated_eval_set)
 
   def _get_eval_set_file_path(self, app_name: str, eval_set_id: str) -> str:
     return os.path.join(
@@ -354,6 +299,10 @@ class LocalEvalSetsManager(EvalSetsManager):
           f"Invalid {id_name}. {id_name} should have the `{pattern}` format",
       )
 
-  def _write_eval_set(self, eval_set_path: str, eval_set: EvalSet):
+  def _write_eval_set_to_path(self, eval_set_path: str, eval_set: EvalSet):
     with open(eval_set_path, "w") as f:
       f.write(eval_set.model_dump_json(indent=2))
+
+  def _save_eval_set(self, app_name: str, eval_set_id: str, eval_set: EvalSet):
+    eval_set_file_path = self._get_eval_set_file_path(app_name, eval_set_id)
+    self._write_eval_set_to_path(eval_set_file_path, eval_set)

tests/unittests/evaluation/test_local_eval_sets_manager.py

@@ -361,8 +361,8 @@ class TestLocalEvalSetsManager:
     app_name = "test_app"
     eval_set_id = "test_eval_set"
     mocker.patch("os.path.exists", return_value=False)
-    mock_write_eval_set = mocker.patch(
-        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set"
+    mock_write_eval_set_to_path = mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set_to_path"
     )
     eval_set_file_path = os.path.join(
         local_eval_sets_manager._agents_dir,
@@ -371,7 +371,7 @@ class TestLocalEvalSetsManager:
     )
 
     local_eval_sets_manager.create_eval_set(app_name, eval_set_id)
-    mock_write_eval_set.assert_called_once_with(
+    mock_write_eval_set_to_path.assert_called_once_with(
         eval_set_file_path,
         EvalSet(
             eval_set_id=eval_set_id,
@@ -420,8 +420,8 @@ class TestLocalEvalSetsManager:
         "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_set",
         return_value=mock_eval_set,
     )
-    mock_write_eval_set = mocker.patch(
-        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set"
+    mock_write_eval_set_to_path = mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set_to_path"
     )
 
     local_eval_sets_manager.add_eval_case(app_name, eval_set_id, mock_eval_case)
@@ -434,7 +434,7 @@ class TestLocalEvalSetsManager:
         eval_set_id + _EVAL_SET_FILE_EXTENSION,
     )
     mock_eval_set.eval_cases.append(mock_eval_case)
-    mock_write_eval_set.assert_called_once_with(
+    mock_write_eval_set_to_path.assert_called_once_with(
         expected_eval_set_file_path, mock_eval_set
     )
 
@@ -568,8 +568,8 @@ class TestLocalEvalSetsManager:
         "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_case",
         return_value=mock_eval_case,
     )
-    mock_write_eval_set = mocker.patch(
-        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set"
+    mock_write_eval_set_to_path = mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set_to_path"
     )
 
     local_eval_sets_manager.update_eval_case(
@@ -583,12 +583,12 @@ class TestLocalEvalSetsManager:
         app_name,
         eval_set_id + _EVAL_SET_FILE_EXTENSION,
     )
-    mock_write_eval_set.assert_called_once_with(
+    mock_write_eval_set_to_path.assert_called_once_with(
         expected_eval_set_file_path,
         EvalSet(eval_set_id=eval_set_id, eval_cases=[updated_eval_case]),
     )
 
-  def test_local_eval_sets_manager_update_eval_case_eval_case_not_found(
+  def test_local_eval_sets_manager_update_eval_case_eval_set_not_found(
       self, local_eval_sets_manager, mocker
   ):
     app_name = "test_app"
@@ -601,10 +601,34 @@ class TestLocalEvalSetsManager:
         return_value=None,
     )
 
+    with pytest.raises(
+        NotFoundError,
+        match=f"Eval set `{eval_set_id}` not found.",
+    ):
+      local_eval_sets_manager.update_eval_case(
+          app_name, eval_set_id, updated_eval_case
+      )
+
+  def test_local_eval_sets_manager_update_eval_case_eval_case_not_found(
+      self, local_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    updated_eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+    mock_eval_set = EvalSet(eval_set_id=eval_set_id, eval_cases=[])
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_set",
+        return_value=mock_eval_set,
+    )
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_case",
+        return_value=None,
+    )
     with pytest.raises(
         NotFoundError,
         match=(
-            f"Eval Set `{eval_set_id}` or Eval id `{eval_case_id}` not found."
+            f"Eval case `{eval_case_id}` not found in eval set `{eval_set_id}`."
         ),
     ):
       local_eval_sets_manager.update_eval_case(
@@ -630,8 +654,8 @@ class TestLocalEvalSetsManager:
         "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_case",
         return_value=mock_eval_case,
     )
-    mock_write_eval_set = mocker.patch(
-        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set"
+    mock_write_eval_set_to_path = mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set_to_path"
     )
 
     local_eval_sets_manager.delete_eval_case(
@@ -644,12 +668,12 @@ class TestLocalEvalSetsManager:
         app_name,
         eval_set_id + _EVAL_SET_FILE_EXTENSION,
     )
-    mock_write_eval_set.assert_called_once_with(
+    mock_write_eval_set_to_path.assert_called_once_with(
         expected_eval_set_file_path,
         EvalSet(eval_set_id=eval_set_id, eval_cases=[]),
     )
 
-  def test_local_eval_sets_manager_delete_eval_case_eval_case_not_found(
+  def test_local_eval_sets_manager_delete_eval_case_eval_set_not_found(
       self, local_eval_sets_manager, mocker
   ):
     app_name = "test_app"
@@ -660,18 +684,41 @@ class TestLocalEvalSetsManager:
         "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_case",
         return_value=None,
     )
-    mock_write_eval_set = mocker.patch(
-        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set"
+    mock_write_eval_set_to_path = mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set_to_path"
     )
 
     with pytest.raises(
         NotFoundError,
-        match=(
-            f"Eval Set `{eval_set_id}` or Eval id `{eval_case_id}` not found."
-        ),
+        match=f"Eval set `{eval_set_id}` not found.",
     ):
       local_eval_sets_manager.delete_eval_case(
           app_name, eval_set_id, eval_case_id
       )
 
-    mock_write_eval_set.assert_not_called()
+    mock_write_eval_set_to_path.assert_not_called()
+
+  def test_local_eval_sets_manager_delete_eval_case_eval_case_not_found(
+      self, local_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    mock_eval_set = EvalSet(eval_set_id=eval_set_id, eval_cases=[])
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_set",
+        return_value=mock_eval_set,
+    )
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_case",
+        return_value=None,
+    )
+    with pytest.raises(
+        NotFoundError,
+        match=(
+            f"Eval case `{eval_case_id}` not found in eval set `{eval_set_id}`."
+        ),
+    ):
+      local_eval_sets_manager.delete_eval_case(
+          app_name, eval_set_id, eval_case_id
+      )