diff --git a/src/google/adk/evaluation/eval_sets_manager.py b/src/google/adk/evaluation/eval_sets_manager.py
index 82f72bab..0c062af5 100644
--- a/src/google/adk/evaluation/eval_sets_manager.py
+++ b/src/google/adk/evaluation/eval_sets_manager.py
@@ -36,7 +36,11 @@ class EvalSetsManager(ABC):
 
   @abstractmethod
   def list_eval_sets(self, app_name: str) -> list[str]:
-    """Returns a list of EvalSets that belong to the given app_name."""
+    """Returns a list of EvalSets that belong to the given app_name.
+
+    Raises:
+      NotFoundError: If the app_name doesn't exist.
+    """
 
   @abstractmethod
   def get_eval_case(
diff --git a/src/google/adk/evaluation/gcs_eval_sets_manager.py b/src/google/adk/evaluation/gcs_eval_sets_manager.py
index c253e4cd..ba7871ce 100644
--- a/src/google/adk/evaluation/gcs_eval_sets_manager.py
+++ b/src/google/adk/evaluation/gcs_eval_sets_manager.py
@@ -23,6 +23,7 @@ from google.cloud import exceptions as cloud_exceptions
 from google.cloud import storage
 from typing_extensions import override
 
+from ..errors.not_found_error import NotFoundError
 from ._eval_sets_manager_utils import add_eval_case_to_eval_set
 from ._eval_sets_manager_utils import delete_eval_case_from_eval_set
 from ._eval_sets_manager_utils import get_eval_case_from_eval_set
@@ -130,7 +131,7 @@ class GcsEvalSetsManager(EvalSetsManager):
         eval_sets.append(eval_set_id)
       return sorted(eval_sets)
     except cloud_exceptions.NotFound as e:
-      raise ValueError(
+      raise NotFoundError(
           f"App `{app_name}` not found in GCS bucket `{self.bucket_name}`."
       ) from e
 
diff --git a/tests/unittests/evaluation/test_gcs_eval_sets_manager.py b/tests/unittests/evaluation/test_gcs_eval_sets_manager.py
index bb8e3bd3..b410fd59 100644
--- a/tests/unittests/evaluation/test_gcs_eval_sets_manager.py
+++ b/tests/unittests/evaluation/test_gcs_eval_sets_manager.py
@@ -17,6 +17,7 @@ from google.adk.evaluation.eval_case import EvalCase
 from google.adk.evaluation.eval_set import EvalSet
 from google.adk.evaluation.gcs_eval_sets_manager import _EVAL_SET_FILE_EXTENSION
 from google.adk.evaluation.gcs_eval_sets_manager import GcsEvalSetsManager
+from google.cloud import exceptions as cloud_exceptions
 import pytest
 
 from .mock_gcs_utils import MockBlob
@@ -121,6 +122,18 @@ class TestGcsEvalSetsManager:
 
     assert eval_sets == ["eval_set_1", "eval_set_2"]
 
+  def test_gcs_eval_sets_manager_list_eval_sets_fails(
+      self, gcs_eval_sets_manager, mocker
+  ):
+    mocker.patch.object(
+        gcs_eval_sets_manager.bucket,
+        "list_blobs",
+        side_effect=cloud_exceptions.NotFound("not found"),
+    )
+
+    with pytest.raises(NotFoundError):
+      gcs_eval_sets_manager.list_eval_sets("test_app")
+
   def test_gcs_eval_sets_manager_add_eval_case_success(
       self, gcs_eval_sets_manager, mocker
   ):