mirror of
https://github.com/encounter/adk-python.git
synced 2026-07-09 18:19:28 -07:00
feat: Re-factor some eval sets manager logic, and implement GcsEvalSetsManager to handle storage of eval sets on GCS
Eval sets will be stored as json files under `gs://{bucket_name}/{app_name}/evals/eval_sets/`
PiperOrigin-RevId: 770487129
This commit is contained in:
Google Team Member
committed by
Copybara-Service
Copybara-Service
parent
bbceb4f2e8
commit
1551bd4f4d
@@ -361,8 +361,8 @@ class TestLocalEvalSetsManager:
|
||||
app_name = "test_app"
|
||||
eval_set_id = "test_eval_set"
|
||||
mocker.patch("os.path.exists", return_value=False)
|
||||
mock_write_eval_set = mocker.patch(
|
||||
"google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set"
|
||||
mock_write_eval_set_to_path = mocker.patch(
|
||||
"google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set_to_path"
|
||||
)
|
||||
eval_set_file_path = os.path.join(
|
||||
local_eval_sets_manager._agents_dir,
|
||||
@@ -371,7 +371,7 @@ class TestLocalEvalSetsManager:
|
||||
)
|
||||
|
||||
local_eval_sets_manager.create_eval_set(app_name, eval_set_id)
|
||||
mock_write_eval_set.assert_called_once_with(
|
||||
mock_write_eval_set_to_path.assert_called_once_with(
|
||||
eval_set_file_path,
|
||||
EvalSet(
|
||||
eval_set_id=eval_set_id,
|
||||
@@ -420,8 +420,8 @@ class TestLocalEvalSetsManager:
|
||||
"google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_set",
|
||||
return_value=mock_eval_set,
|
||||
)
|
||||
mock_write_eval_set = mocker.patch(
|
||||
"google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set"
|
||||
mock_write_eval_set_to_path = mocker.patch(
|
||||
"google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set_to_path"
|
||||
)
|
||||
|
||||
local_eval_sets_manager.add_eval_case(app_name, eval_set_id, mock_eval_case)
|
||||
@@ -434,7 +434,7 @@ class TestLocalEvalSetsManager:
|
||||
eval_set_id + _EVAL_SET_FILE_EXTENSION,
|
||||
)
|
||||
mock_eval_set.eval_cases.append(mock_eval_case)
|
||||
mock_write_eval_set.assert_called_once_with(
|
||||
mock_write_eval_set_to_path.assert_called_once_with(
|
||||
expected_eval_set_file_path, mock_eval_set
|
||||
)
|
||||
|
||||
@@ -568,8 +568,8 @@ class TestLocalEvalSetsManager:
|
||||
"google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_case",
|
||||
return_value=mock_eval_case,
|
||||
)
|
||||
mock_write_eval_set = mocker.patch(
|
||||
"google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set"
|
||||
mock_write_eval_set_to_path = mocker.patch(
|
||||
"google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set_to_path"
|
||||
)
|
||||
|
||||
local_eval_sets_manager.update_eval_case(
|
||||
@@ -583,12 +583,12 @@ class TestLocalEvalSetsManager:
|
||||
app_name,
|
||||
eval_set_id + _EVAL_SET_FILE_EXTENSION,
|
||||
)
|
||||
mock_write_eval_set.assert_called_once_with(
|
||||
mock_write_eval_set_to_path.assert_called_once_with(
|
||||
expected_eval_set_file_path,
|
||||
EvalSet(eval_set_id=eval_set_id, eval_cases=[updated_eval_case]),
|
||||
)
|
||||
|
||||
def test_local_eval_sets_manager_update_eval_case_eval_case_not_found(
|
||||
def test_local_eval_sets_manager_update_eval_case_eval_set_not_found(
|
||||
self, local_eval_sets_manager, mocker
|
||||
):
|
||||
app_name = "test_app"
|
||||
@@ -601,10 +601,34 @@ class TestLocalEvalSetsManager:
|
||||
return_value=None,
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
NotFoundError,
|
||||
match=f"Eval set `{eval_set_id}` not found.",
|
||||
):
|
||||
local_eval_sets_manager.update_eval_case(
|
||||
app_name, eval_set_id, updated_eval_case
|
||||
)
|
||||
|
||||
def test_local_eval_sets_manager_update_eval_case_eval_case_not_found(
|
||||
self, local_eval_sets_manager, mocker
|
||||
):
|
||||
app_name = "test_app"
|
||||
eval_set_id = "test_eval_set"
|
||||
eval_case_id = "test_eval_case"
|
||||
updated_eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
|
||||
mock_eval_set = EvalSet(eval_set_id=eval_set_id, eval_cases=[])
|
||||
mocker.patch(
|
||||
"google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_set",
|
||||
return_value=mock_eval_set,
|
||||
)
|
||||
mocker.patch(
|
||||
"google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_case",
|
||||
return_value=None,
|
||||
)
|
||||
with pytest.raises(
|
||||
NotFoundError,
|
||||
match=(
|
||||
f"Eval Set `{eval_set_id}` or Eval id `{eval_case_id}` not found."
|
||||
f"Eval case `{eval_case_id}` not found in eval set `{eval_set_id}`."
|
||||
),
|
||||
):
|
||||
local_eval_sets_manager.update_eval_case(
|
||||
@@ -630,8 +654,8 @@ class TestLocalEvalSetsManager:
|
||||
"google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_case",
|
||||
return_value=mock_eval_case,
|
||||
)
|
||||
mock_write_eval_set = mocker.patch(
|
||||
"google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set"
|
||||
mock_write_eval_set_to_path = mocker.patch(
|
||||
"google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set_to_path"
|
||||
)
|
||||
|
||||
local_eval_sets_manager.delete_eval_case(
|
||||
@@ -644,12 +668,12 @@ class TestLocalEvalSetsManager:
|
||||
app_name,
|
||||
eval_set_id + _EVAL_SET_FILE_EXTENSION,
|
||||
)
|
||||
mock_write_eval_set.assert_called_once_with(
|
||||
mock_write_eval_set_to_path.assert_called_once_with(
|
||||
expected_eval_set_file_path,
|
||||
EvalSet(eval_set_id=eval_set_id, eval_cases=[]),
|
||||
)
|
||||
|
||||
def test_local_eval_sets_manager_delete_eval_case_eval_case_not_found(
|
||||
def test_local_eval_sets_manager_delete_eval_case_eval_set_not_found(
|
||||
self, local_eval_sets_manager, mocker
|
||||
):
|
||||
app_name = "test_app"
|
||||
@@ -660,18 +684,41 @@ class TestLocalEvalSetsManager:
|
||||
"google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_case",
|
||||
return_value=None,
|
||||
)
|
||||
mock_write_eval_set = mocker.patch(
|
||||
"google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set"
|
||||
mock_write_eval_set_to_path = mocker.patch(
|
||||
"google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set_to_path"
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
NotFoundError,
|
||||
match=(
|
||||
f"Eval Set `{eval_set_id}` or Eval id `{eval_case_id}` not found."
|
||||
),
|
||||
match=f"Eval set `{eval_set_id}` not found.",
|
||||
):
|
||||
local_eval_sets_manager.delete_eval_case(
|
||||
app_name, eval_set_id, eval_case_id
|
||||
)
|
||||
|
||||
mock_write_eval_set.assert_not_called()
|
||||
mock_write_eval_set_to_path.assert_not_called()
|
||||
|
||||
def test_local_eval_sets_manager_delete_eval_case_eval_case_not_found(
|
||||
self, local_eval_sets_manager, mocker
|
||||
):
|
||||
app_name = "test_app"
|
||||
eval_set_id = "test_eval_set"
|
||||
eval_case_id = "test_eval_case"
|
||||
mock_eval_set = EvalSet(eval_set_id=eval_set_id, eval_cases=[])
|
||||
mocker.patch(
|
||||
"google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_set",
|
||||
return_value=mock_eval_set,
|
||||
)
|
||||
mocker.patch(
|
||||
"google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_case",
|
||||
return_value=None,
|
||||
)
|
||||
with pytest.raises(
|
||||
NotFoundError,
|
||||
match=(
|
||||
f"Eval case `{eval_case_id}` not found in eval set `{eval_set_id}`."
|
||||
),
|
||||
):
|
||||
local_eval_sets_manager.delete_eval_case(
|
||||
app_name, eval_set_id, eval_case_id
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user