fix: Re-adding eval related changes

Due to reasons that are being investigated, some of the recent changes got unintentionally reverted. We are adding those back in this PR.

PiperOrigin-RevId: 789384063

src/google/adk/cli/cli_tools_click.py

@@ -33,8 +33,6 @@ from . import cli_create
 from . import cli_deploy
 from .. import version
 from ..evaluation.constants import MISSING_EVAL_DEPENDENCIES_MESSAGE
-from ..evaluation.local_eval_set_results_manager import LocalEvalSetResultsManager
-from ..sessions.in_memory_session_service import InMemorySessionService
 from .cli import run_cli
 from .fast_api import get_fast_api_app
 from .utils import envs
@@ -289,7 +287,7 @@ def cli_run(
         exists=True, dir_okay=True, file_okay=False, resolve_path=True
     ),
 )
-@click.argument("eval_set_file_path", nargs=-1)
+@click.argument("eval_set_file_path_or_id", nargs=-1)
 @click.option("--config_file_path", help="Optional. The path to config file.")
 @click.option(
     "--print_detailed_results",
@@ -309,7 +307,7 @@ def cli_run(
 )
 def cli_eval(
     agent_module_file_path: str,
-    eval_set_file_path: list[str],
+    eval_set_file_path_or_id: list[str],
     config_file_path: str,
     print_detailed_results: bool,
     eval_storage_uri: Optional[str] = None,
@@ -319,20 +317,51 @@ def cli_eval(
   AGENT_MODULE_FILE_PATH: The path to the __init__.py file that contains a
   module by the name "agent". "agent" module contains a root_agent.
 
-  EVAL_SET_FILE_PATH: You can specify one or more eval set file paths.
+  EVAL_SET_FILE_PATH_OR_ID: You can specify one or more eval set file paths or
+  eval set id.
 
+  Mixing of eval set file paths with eval set ids is not allowed.
+
+  *Eval Set File Path*
   For each file, all evals will be run by default.
 
   If you want to run only specific evals from a eval set, first create a comma
   separated list of eval names and then add that as a suffix to the eval set
   file name, demarcated by a `:`.
 
-  For example,
+  For example, we have `sample_eval_set_file.json` file that has following the
+  eval cases:
+  sample_eval_set_file.json:
+    |....... eval_1
+    |....... eval_2
+    |....... eval_3
+    |....... eval_4
+    |....... eval_5
 
   sample_eval_set_file.json:eval_1,eval_2,eval_3
 
   This will only run eval_1, eval_2 and eval_3 from sample_eval_set_file.json.
 
+  *Eval Set Id*
+  For each eval set, all evals will be run by default.
+
+  If you want to run only specific evals from a eval set, first create a comma
+  separated list of eval names and then add that as a suffix to the eval set
+  file name, demarcated by a `:`.
+
+  For example, we have `sample_eval_set_id` that has following the eval cases:
+  sample_eval_set_id:
+    |....... eval_1
+    |....... eval_2
+    |....... eval_3
+    |....... eval_4
+    |....... eval_5
+
+  If we did:
+      sample_eval_set_id:eval_1,eval_2,eval_3
+
+  This will only run eval_1, eval_2 and eval_3 from sample_eval_set_id.
+
   CONFIG_FILE_PATH: The path to config file.
 
   PRINT_DETAILED_RESULTS: Prints detailed results on the console.
@@ -340,103 +369,137 @@ def cli_eval(
   envs.load_dotenv_for_agent(agent_module_file_path, ".")
 
   try:
+    from ..evaluation.base_eval_service import InferenceConfig
+    from ..evaluation.base_eval_service import InferenceRequest
+    from ..evaluation.eval_metrics import EvalMetric
+    from ..evaluation.eval_metrics import JudgeModelOptions
+    from ..evaluation.eval_result import EvalCaseResult
+    from ..evaluation.evaluator import EvalStatus
+    from ..evaluation.in_memory_eval_sets_manager import InMemoryEvalSetsManager
+    from ..evaluation.local_eval_service import LocalEvalService
+    from ..evaluation.local_eval_set_results_manager import LocalEvalSetResultsManager
     from ..evaluation.local_eval_sets_manager import load_eval_set_from_file
-    from .cli_eval import EvalCaseResult
+    from ..evaluation.local_eval_sets_manager import LocalEvalSetsManager
-    from .cli_eval import EvalMetric
+    from .cli_eval import _collect_eval_results
-    from .cli_eval import EvalStatus
+    from .cli_eval import _collect_inferences
     from .cli_eval import get_evaluation_criteria_or_default
     from .cli_eval import get_root_agent
     from .cli_eval import parse_and_get_evals_to_run
-    from .cli_eval import run_evals
+  except ModuleNotFoundError as mnf:
-    from .cli_eval import try_get_reset_func
+    raise click.ClickException(MISSING_EVAL_DEPENDENCIES_MESSAGE) from mnf
-  except ModuleNotFoundError:
-    raise click.ClickException(MISSING_EVAL_DEPENDENCIES_MESSAGE)
 
   evaluation_criteria = get_evaluation_criteria_or_default(config_file_path)
   eval_metrics = []
   for metric_name, threshold in evaluation_criteria.items():
     eval_metrics.append(
-        EvalMetric(metric_name=metric_name, threshold=threshold)
+        EvalMetric(
+            metric_name=metric_name,
+            threshold=threshold,
+            judge_model_options=JudgeModelOptions(),
+        )
     )
 
   print(f"Using evaluation criteria: {evaluation_criteria}")
 
   root_agent = get_root_agent(agent_module_file_path)
-  reset_func = try_get_reset_func(agent_module_file_path)
+  app_name = os.path.basename(agent_module_file_path)
-
+  agents_dir = os.path.dirname(agent_module_file_path)
-  gcs_eval_sets_manager = None
+  eval_sets_manager = None
   eval_set_results_manager = None
+
   if eval_storage_uri:
     gcs_eval_managers = evals.create_gcs_eval_managers_from_uri(
         eval_storage_uri
     )
-    gcs_eval_sets_manager = gcs_eval_managers.eval_sets_manager
+    eval_sets_manager = gcs_eval_managers.eval_sets_manager
     eval_set_results_manager = gcs_eval_managers.eval_set_results_manager
   else:
-    eval_set_results_manager = LocalEvalSetResultsManager(
+    eval_set_results_manager = LocalEvalSetResultsManager(agents_dir=agents_dir)
-        agents_dir=os.path.dirname(agent_module_file_path)
-    )
-  eval_set_file_path_to_evals = parse_and_get_evals_to_run(eval_set_file_path)
-  eval_set_id_to_eval_cases = {}
 
-  # Read the eval_set files and get the cases.
+  inference_requests = []
-  for eval_set_file_path, eval_case_ids in eval_set_file_path_to_evals.items():
+  eval_set_file_or_id_to_evals = parse_and_get_evals_to_run(
-    if gcs_eval_sets_manager:
+      eval_set_file_path_or_id
-      eval_set = gcs_eval_sets_manager._load_eval_set_from_blob(
+  )
-          eval_set_file_path
+
-      )
+  # Check if the first entry is a file that exists, if it does then we assume
-      if not eval_set:
+  # rest of the entries are also files. We enforce this assumption in the if
-        raise click.ClickException(
+  # block.
-            f"Eval set {eval_set_file_path} not found in GCS."
+  if eval_set_file_or_id_to_evals and os.path.exists(
+      list(eval_set_file_or_id_to_evals.keys())[0]
+  ):
+    eval_sets_manager = InMemoryEvalSetsManager()
+
+    # Read the eval_set files and get the cases.
+    for (
+        eval_set_file_path,
+        eval_case_ids,
+    ) in eval_set_file_or_id_to_evals.items():
+      try:
+        eval_set = load_eval_set_from_file(
+            eval_set_file_path, eval_set_file_path
         )
-    else:
+      except FileNotFoundError as fne:
-      eval_set = load_eval_set_from_file(eval_set_file_path, eval_set_file_path)
+        raise click.ClickException(
-    eval_cases = eval_set.eval_cases
+            f"`{eval_set_file_path}` should be a valid eval set file."
+        ) from fne
 
-    if eval_case_ids:
+      eval_sets_manager.create_eval_set(
-      # There are eval_ids that we should select.
+          app_name=app_name, eval_set_id=eval_set.eval_set_id
-      eval_cases = [
+      )
-          e for e in eval_set.eval_cases if e.eval_id in eval_case_ids
+      for eval_case in eval_set.eval_cases:
-      ]
+        eval_sets_manager.add_eval_case(
+            app_name=app_name,
+            eval_set_id=eval_set.eval_set_id,
+            eval_case=eval_case,
+        )
+      inference_requests.append(
+          InferenceRequest(
+              app_name=app_name,
+              eval_set_id=eval_set.eval_set_id,
+              eval_case_ids=eval_case_ids,
+              inference_config=InferenceConfig(),
+          )
+      )
+  else:
+    # We assume that what we have are eval set ids instead.
+    eval_sets_manager = (
+        eval_sets_manager
+        if eval_storage_uri
+        else LocalEvalSetsManager(agents_dir=agents_dir)
+    )
 
-    eval_set_id_to_eval_cases[eval_set.eval_set_id] = eval_cases
+    for eval_set_id_key, eval_case_ids in eval_set_file_or_id_to_evals.items():
-
+      inference_requests.append(
-  async def _collect_eval_results() -> list[EvalCaseResult]:
+          InferenceRequest(
-    session_service = InMemorySessionService()
+              app_name=app_name,
-    eval_case_results = []
+              eval_set_id=eval_set_id_key,
-    async for eval_case_result in run_evals(
+              eval_case_ids=eval_case_ids,
-        eval_set_id_to_eval_cases,
+              inference_config=InferenceConfig(),
-        root_agent,
+          )
-        reset_func,
-        eval_metrics,
-        session_service=session_service,
-    ):
-      eval_case_result.session_details = await session_service.get_session(
-          app_name=os.path.basename(agent_module_file_path),
-          user_id=eval_case_result.user_id,
-          session_id=eval_case_result.session_id,
       )
-      eval_case_results.append(eval_case_result)
-    return eval_case_results
 
   try:
-    eval_results = asyncio.run(_collect_eval_results())
+    eval_service = LocalEvalService(
-  except ModuleNotFoundError:
+        root_agent=root_agent,
-    raise click.ClickException(MISSING_EVAL_DEPENDENCIES_MESSAGE)
+        eval_sets_manager=eval_sets_manager,
-
+        eval_set_results_manager=eval_set_results_manager,
-  # Write eval set results.
-  eval_set_id_to_eval_results = collections.defaultdict(list)
-  for eval_case_result in eval_results:
-    eval_set_id = eval_case_result.eval_set_id
-    eval_set_id_to_eval_results[eval_set_id].append(eval_case_result)
-
-  for eval_set_id, eval_case_results in eval_set_id_to_eval_results.items():
-    eval_set_results_manager.save_eval_set_result(
-        app_name=os.path.basename(agent_module_file_path),
-        eval_set_id=eval_set_id,
-        eval_case_results=eval_case_results,
     )
 
+    inference_results = asyncio.run(
+        _collect_inferences(
+            inference_requests=inference_requests, eval_service=eval_service
+        )
+    )
+    eval_results = asyncio.run(
+        _collect_eval_results(
+            inference_results=inference_results,
+            eval_service=eval_service,
+            eval_metrics=eval_metrics,
+        )
+    )
+  except ModuleNotFoundError as mnf:
+    raise click.ClickException(MISSING_EVAL_DEPENDENCIES_MESSAGE) from mnf
+
   print("*********************************************************************")
   eval_run_summary = {}
 
@@ -890,8 +953,10 @@ def cli_deploy_cloud_run(
     port: int,
     trace_to_cloud: bool,
     with_ui: bool,
-    verbosity: str,
     adk_version: str,
+    verbosity: str = "WARNING",
+    reload: bool = True,
+    allow_origins: Optional[list[str]] = None,
     log_level: Optional[str] = None,
     session_service_uri: Optional[str] = None,
     artifact_service_uri: Optional[str] = None,
@@ -923,6 +988,7 @@ def cli_deploy_cloud_run(
         temp_folder=temp_folder,
         port=port,
         trace_to_cloud=trace_to_cloud,
+        allow_origins=allow_origins,
         with_ui=with_ui,
         log_level=log_level,
         verbosity=verbosity,

tests/unittests/cli/utils/test_cli_tools_click.py

@@ -23,16 +23,44 @@ from types import SimpleNamespace
 from typing import Any
 from typing import Dict
 from typing import List
-from typing import Optional
 from typing import Tuple
+from unittest import mock
 
 import click
 from click.testing import CliRunner
-import google.adk.evaluation.local_eval_sets_manager as managerModule
+from google.adk.agents.base_agent import BaseAgent
+from google.adk.cli import cli_tools_click
+from google.adk.evaluation.eval_case import EvalCase
+from google.adk.evaluation.eval_set import EvalSet
+from google.adk.evaluation.local_eval_set_results_manager import LocalEvalSetResultsManager
+from google.adk.evaluation.local_eval_sets_manager import LocalEvalSetsManager
 from pydantic import BaseModel
 import pytest
 
-from src.google.adk.cli import cli_tools_click
+
+class DummyAgent(BaseAgent):
+
+  def __init__(self, name):
+    super().__init__(name=name)
+    self.sub_agents = []
+
+
+root_agent = DummyAgent(name="dummy_agent")
+
+
+@pytest.fixture
+def mock_load_eval_set_from_file():
+  with mock.patch(
+      "google.adk.evaluation.local_eval_sets_manager.load_eval_set_from_file"
+  ) as mock_func:
+    yield mock_func
+
+
+@pytest.fixture
+def mock_get_root_agent():
+  with mock.patch("google.adk.cli.cli_eval.get_root_agent") as mock_func:
+    mock_func.return_value = root_agent
+    yield mock_func
 
 
 # Helpers
@@ -376,3 +404,75 @@ def test_cli_web_passes_deprecated_uris(
   called_kwargs = mock_get_app.calls[0][1]
   assert called_kwargs.get("session_service_uri") == "sqlite:///deprecated.db"
   assert called_kwargs.get("artifact_service_uri") == "gs://deprecated"
+
+
+def test_cli_eval_with_eval_set_file_path(
+    mock_load_eval_set_from_file,
+    mock_get_root_agent,
+    tmp_path,
+):
+  agent_path = tmp_path / "my_agent"
+  agent_path.mkdir()
+  (agent_path / "__init__.py").touch()
+
+  eval_set_file = tmp_path / "my_evals.json"
+  eval_set_file.write_text("{}")
+
+  mock_load_eval_set_from_file.return_value = EvalSet(
+      eval_set_id="my_evals",
+      eval_cases=[EvalCase(eval_id="case1", conversation=[])],
+  )
+
+  result = CliRunner().invoke(
+      cli_tools_click.cli_eval,
+      [str(agent_path), str(eval_set_file)],
+  )
+
+  assert result.exit_code == 0
+  # Assert that we wrote eval set results
+  eval_set_results_manager = LocalEvalSetResultsManager(
+      agents_dir=str(tmp_path)
+  )
+  eval_set_results = eval_set_results_manager.list_eval_set_results(
+      app_name="my_agent"
+  )
+  assert len(eval_set_results) == 1
+
+
+def test_cli_eval_with_eval_set_id(
+    mock_get_root_agent,
+    tmp_path,
+):
+  app_name = "test_app"
+  eval_set_id = "test_eval_set_id"
+  agent_path = tmp_path / app_name
+  agent_path.mkdir()
+  (agent_path / "__init__.py").touch()
+
+  eval_sets_manager = LocalEvalSetsManager(agents_dir=str(tmp_path))
+  eval_sets_manager.create_eval_set(app_name=app_name, eval_set_id=eval_set_id)
+  eval_sets_manager.add_eval_case(
+      app_name=app_name,
+      eval_set_id=eval_set_id,
+      eval_case=EvalCase(eval_id="case1", conversation=[]),
+  )
+  eval_sets_manager.add_eval_case(
+      app_name=app_name,
+      eval_set_id=eval_set_id,
+      eval_case=EvalCase(eval_id="case2", conversation=[]),
+  )
+
+  result = CliRunner().invoke(
+      cli_tools_click.cli_eval,
+      [str(agent_path), "test_eval_set_id:case1,case2"],
+  )
+
+  assert result.exit_code == 0
+  # Assert that we wrote eval set results
+  eval_set_results_manager = LocalEvalSetResultsManager(
+      agents_dir=str(tmp_path)
+  )
+  eval_set_results = eval_set_results_manager.list_eval_set_results(
+      app_name=app_name
+  )
+  assert len(eval_set_results) == 2