mirror of
https://github.com/encounter/adk-python.git
synced 2026-07-09 18:19:28 -07:00
feat: Add support for persisting eval run results
If the EvalRunResultsManager is provided to LocalEvalService, then we want to persist the eval run results using it. PiperOrigin-RevId: 782196848
This commit is contained in:
committed by
Copybara-Service
parent
33eec34577
commit
bab3be2cf3
@@ -159,12 +159,22 @@ class LocalEvalService(BaseEvalService):
|
|||||||
run_evaluation(inference_result)
|
run_evaluation(inference_result)
|
||||||
for inference_result in evaluate_request.inference_results
|
for inference_result in evaluate_request.inference_results
|
||||||
]
|
]
|
||||||
|
|
||||||
for evaluation_task in asyncio.as_completed(evaluation_tasks):
|
for evaluation_task in asyncio.as_completed(evaluation_tasks):
|
||||||
yield await evaluation_task
|
inference_result, eval_case_result = await evaluation_task
|
||||||
|
|
||||||
|
if self._eval_set_results_manager:
|
||||||
|
self._eval_set_results_manager.save_eval_set_result(
|
||||||
|
app_name=inference_result.app_name,
|
||||||
|
eval_set_id=inference_result.eval_set_id,
|
||||||
|
eval_case_results=[eval_case_result],
|
||||||
|
)
|
||||||
|
|
||||||
|
yield eval_case_result
|
||||||
|
|
||||||
async def _evaluate_single_inference_result(
|
async def _evaluate_single_inference_result(
|
||||||
self, inference_result: InferenceResult, evaluate_config: EvaluateConfig
|
self, inference_result: InferenceResult, evaluate_config: EvaluateConfig
|
||||||
) -> EvalCaseResult:
|
) -> tuple[InferenceResult, EvalCaseResult]:
|
||||||
"""Returns EvalCaseResult for the given inference result.
|
"""Returns EvalCaseResult for the given inference result.
|
||||||
|
|
||||||
A single inference result can have multiple invocations. For each
|
A single inference result can have multiple invocations. For each
|
||||||
@@ -267,7 +277,7 @@ class LocalEvalService(BaseEvalService):
|
|||||||
else 'test_user_id'
|
else 'test_user_id'
|
||||||
)
|
)
|
||||||
|
|
||||||
return EvalCaseResult(
|
eval_case_result = EvalCaseResult(
|
||||||
eval_set_file=inference_result.eval_set_id,
|
eval_set_file=inference_result.eval_set_id,
|
||||||
eval_set_id=inference_result.eval_set_id,
|
eval_set_id=inference_result.eval_set_id,
|
||||||
eval_id=inference_result.eval_case_id,
|
eval_id=inference_result.eval_case_id,
|
||||||
@@ -275,9 +285,16 @@ class LocalEvalService(BaseEvalService):
|
|||||||
overall_eval_metric_results=overall_eval_metric_results,
|
overall_eval_metric_results=overall_eval_metric_results,
|
||||||
eval_metric_result_per_invocation=eval_metric_result_per_invocation,
|
eval_metric_result_per_invocation=eval_metric_result_per_invocation,
|
||||||
session_id=inference_result.session_id,
|
session_id=inference_result.session_id,
|
||||||
|
session_details=await self._session_service.get_session(
|
||||||
|
app_name=inference_result.app_name,
|
||||||
|
user_id=user_id,
|
||||||
|
session_id=inference_result.session_id,
|
||||||
|
),
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
return (inference_result, eval_case_result)
|
||||||
|
|
||||||
async def _evaluate_metric(
|
async def _evaluate_metric(
|
||||||
self,
|
self,
|
||||||
eval_metric: EvalMetric,
|
eval_metric: EvalMetric,
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ from google.adk.evaluation.eval_metrics import EvalMetricResult
|
|||||||
from google.adk.evaluation.eval_result import EvalCaseResult
|
from google.adk.evaluation.eval_result import EvalCaseResult
|
||||||
from google.adk.evaluation.eval_set import EvalCase
|
from google.adk.evaluation.eval_set import EvalCase
|
||||||
from google.adk.evaluation.eval_set import EvalSet
|
from google.adk.evaluation.eval_set import EvalSet
|
||||||
|
from google.adk.evaluation.eval_set_results_manager import EvalSetResultsManager
|
||||||
from google.adk.evaluation.eval_sets_manager import EvalSetsManager
|
from google.adk.evaluation.eval_sets_manager import EvalSetsManager
|
||||||
from google.adk.evaluation.evaluator import EvalStatus
|
from google.adk.evaluation.evaluator import EvalStatus
|
||||||
from google.adk.evaluation.evaluator import EvaluationResult
|
from google.adk.evaluation.evaluator import EvaluationResult
|
||||||
@@ -51,13 +52,21 @@ def dummy_agent():
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def eval_service(dummy_agent, mock_eval_sets_manager):
|
def mock_eval_set_results_manager():
|
||||||
|
return mock.create_autospec(EvalSetResultsManager)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def eval_service(
|
||||||
|
dummy_agent, mock_eval_sets_manager, mock_eval_set_results_manager
|
||||||
|
):
|
||||||
DEFAULT_METRIC_EVALUATOR_REGISTRY.register_evaluator(
|
DEFAULT_METRIC_EVALUATOR_REGISTRY.register_evaluator(
|
||||||
metric_name="fake_metric", evaluator=FakeEvaluator
|
metric_name="fake_metric", evaluator=FakeEvaluator
|
||||||
)
|
)
|
||||||
return LocalEvalService(
|
return LocalEvalService(
|
||||||
root_agent=dummy_agent,
|
root_agent=dummy_agent,
|
||||||
eval_sets_manager=mock_eval_sets_manager,
|
eval_sets_manager=mock_eval_sets_manager,
|
||||||
|
eval_set_results_manager=mock_eval_set_results_manager,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -90,7 +99,9 @@ class FakeEvaluator(Evaluator):
|
|||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_perform_inference_success(
|
async def test_perform_inference_success(
|
||||||
eval_service, dummy_agent, mock_eval_sets_manager
|
eval_service,
|
||||||
|
dummy_agent,
|
||||||
|
mock_eval_sets_manager,
|
||||||
):
|
):
|
||||||
eval_set = EvalSet(
|
eval_set = EvalSet(
|
||||||
eval_set_id="test_eval_set",
|
eval_set_id="test_eval_set",
|
||||||
@@ -127,7 +138,9 @@ async def test_perform_inference_success(
|
|||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_perform_inference_with_case_ids(
|
async def test_perform_inference_with_case_ids(
|
||||||
eval_service, dummy_agent, mock_eval_sets_manager
|
eval_service,
|
||||||
|
dummy_agent,
|
||||||
|
mock_eval_sets_manager,
|
||||||
):
|
):
|
||||||
eval_set = EvalSet(
|
eval_set = EvalSet(
|
||||||
eval_set_id="test_eval_set",
|
eval_set_id="test_eval_set",
|
||||||
@@ -172,7 +185,8 @@ async def test_perform_inference_with_case_ids(
|
|||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_perform_inference_eval_set_not_found(
|
async def test_perform_inference_eval_set_not_found(
|
||||||
eval_service, mock_eval_sets_manager
|
eval_service,
|
||||||
|
mock_eval_sets_manager,
|
||||||
):
|
):
|
||||||
mock_eval_sets_manager.get_eval_set.return_value = None
|
mock_eval_sets_manager.get_eval_set.return_value = None
|
||||||
|
|
||||||
@@ -188,7 +202,9 @@ async def test_perform_inference_eval_set_not_found(
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_evaluate_success(eval_service, mock_eval_sets_manager):
|
async def test_evaluate_success(
|
||||||
|
eval_service, mock_eval_sets_manager, mock_eval_set_results_manager
|
||||||
|
):
|
||||||
inference_results = [
|
inference_results = [
|
||||||
InferenceResult(
|
InferenceResult(
|
||||||
app_name="test_app",
|
app_name="test_app",
|
||||||
@@ -224,11 +240,13 @@ async def test_evaluate_success(eval_service, mock_eval_sets_manager):
|
|||||||
assert isinstance(results[0], EvalCaseResult)
|
assert isinstance(results[0], EvalCaseResult)
|
||||||
assert isinstance(results[1], EvalCaseResult)
|
assert isinstance(results[1], EvalCaseResult)
|
||||||
assert mock_eval_sets_manager.get_eval_case.call_count == 2
|
assert mock_eval_sets_manager.get_eval_case.call_count == 2
|
||||||
|
assert mock_eval_set_results_manager.save_eval_set_result.call_count == 2
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_evaluate_eval_case_not_found(
|
async def test_evaluate_eval_case_not_found(
|
||||||
eval_service, mock_eval_sets_manager
|
eval_service,
|
||||||
|
mock_eval_sets_manager,
|
||||||
):
|
):
|
||||||
inference_results = [
|
inference_results = [
|
||||||
InferenceResult(
|
InferenceResult(
|
||||||
@@ -256,7 +274,7 @@ async def test_evaluate_eval_case_not_found(
|
|||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_evaluate_single_inference_result(
|
async def test_evaluate_single_inference_result(
|
||||||
eval_service, mock_eval_sets_manager
|
eval_service, mock_eval_sets_manager, mock_eval_set_results_manager
|
||||||
):
|
):
|
||||||
invocation = Invocation(
|
invocation = Invocation(
|
||||||
user_content=genai_types.Content(
|
user_content=genai_types.Content(
|
||||||
@@ -289,7 +307,7 @@ async def test_evaluate_single_inference_result(
|
|||||||
mock_eval_case.session_input = None
|
mock_eval_case.session_input = None
|
||||||
mock_eval_sets_manager.get_eval_case.return_value = mock_eval_case
|
mock_eval_sets_manager.get_eval_case.return_value = mock_eval_case
|
||||||
|
|
||||||
result = await eval_service._evaluate_single_inference_result(
|
_, result = await eval_service._evaluate_single_inference_result(
|
||||||
inference_result=inference_result, evaluate_config=evaluate_config
|
inference_result=inference_result, evaluate_config=evaluate_config
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user