encounter/adk-python
0
0
Fork 0
mirror of https://github.com/encounter/adk-python.git synced 2026-07-09 18:19:28 -07:00
Code Issues Packages Projects Releases Wiki Activity

feat: Support per-eval case and per-invocation rubrics in rubric-based evaluators

Browse Source 
Co-authored-by: Joseph Pagadora <jcpagadora@google.com>
PiperOrigin-RevId: 853820099
This commit is contained in:
Joseph Pagadora
2026-01-08 11:26:06 -08:00
committed by Copybara-Service
parent 688791396a
commit 8afb99a078
8 changed files with 402 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unittests/evaluation/test_llm_as_judge.py
+8 -2
View File
@@ -20,6 +20,7 @@ from google.adk.evaluation.eval_case import Invocation
from google.adk.evaluation.eval_metrics import EvalMetric
from google.adk.evaluation.eval_metrics import JudgeModelOptions
from google.adk.evaluation.eval_metrics import LlmAsAJudgeCriterion
from google.adk.evaluation.eval_rubrics import Rubric
from google.adk.evaluation.evaluator import EvalStatus
from google.adk.evaluation.evaluator import EvaluationResult
from google.adk.evaluation.evaluator import PerInvocationResult
@@ -35,12 +36,17 @@ import pytest
class MockLlmAsJudge(LlmAsJudge):
def format_auto_rater_prompt(
self, actual_invocation: Invocation, expected_invocation: Invocation
self,
actual_invocation: Invocation,
expected_invocation: Optional[Invocation],
rubrics: Optional[list[Rubric]] = None,
) -> str:
return "formatted prompt"
def convert_auto_rater_response_to_score(
self, llm_response: LlmResponse
self,
llm_response: LlmResponse,
rubrics: Optional[list[Rubric]] = None,
) -> AutoRaterScore:
return AutoRaterScore(score=1.0)
tests/unittests/evaluation/test_local_eval_service.py
+113
View File
@@ -34,6 +34,8 @@ from google.adk.evaluation.eval_metrics import Interval
from google.adk.evaluation.eval_metrics import MetricInfo
from google.adk.evaluation.eval_metrics import MetricValueInfo
from google.adk.evaluation.eval_result import EvalCaseResult
from google.adk.evaluation.eval_rubrics import Rubric
from google.adk.evaluation.eval_rubrics import RubricContent
from google.adk.evaluation.eval_set import EvalCase
from google.adk.evaluation.eval_set import EvalSet
from google.adk.evaluation.eval_set_results_manager import EvalSetResultsManager
@@ -42,6 +44,9 @@ from google.adk.evaluation.evaluator import EvalStatus
from google.adk.evaluation.evaluator import EvaluationResult
from google.adk.evaluation.evaluator import Evaluator
from google.adk.evaluation.evaluator import PerInvocationResult
from google.adk.evaluation.local_eval_service import _add_rubrics_to_invocation
from google.adk.evaluation.local_eval_service import _copy_eval_case_rubrics_to_actual_invocations
from google.adk.evaluation.local_eval_service import _copy_invocation_rubrics_to_actual_invocations
from google.adk.evaluation.local_eval_service import LocalEvalService
from google.adk.evaluation.metric_evaluator_registry import DEFAULT_METRIC_EVALUATOR_REGISTRY
from google.adk.models.registry import LLMRegistry
@@ -678,3 +683,111 @@ async def test_mcp_stdio_agent_no_runtime_error(mocker):
import shutil
shutil.rmtree(test_dir, ignore_errors=True)
def test_add_rubrics_to_invocation_initializes_rubrics_list():
invocation = Invocation(user_content=genai_types.Content())
rubric = Rubric(
rubric_id="r1", rubric_content=RubricContent(text_property="p1")
)
_add_rubrics_to_invocation(invocation, [rubric])
assert invocation.rubrics == [rubric]
def test_add_rubrics_to_invocation_adds_to_existing_list():
rubric1 = Rubric(
rubric_id="r1", rubric_content=RubricContent(text_property="p1")
)
rubric2 = Rubric(
rubric_id="r2", rubric_content=RubricContent(text_property="p2")
)
invocation = Invocation(user_content=genai_types.Content(), rubrics=[rubric1])
_add_rubrics_to_invocation(invocation, [rubric2])
assert invocation.rubrics == [rubric1, rubric2]
def test_add_rubrics_to_invocation_errors_on_duplicate_id():
rubric1 = Rubric(
rubric_id="r1", rubric_content=RubricContent(text_property="p1")
)
rubric2 = Rubric(
rubric_id="r1", rubric_content=RubricContent(text_property="p2")
)
invocation = Invocation(user_content=genai_types.Content(), rubrics=[rubric1])
with pytest.raises(ValueError):
_add_rubrics_to_invocation(invocation, [rubric2])
def test_copy_eval_case_rubrics_to_actual_invocations():
rubric1 = Rubric(
rubric_id="r1", rubric_content=RubricContent(text_property="p1")
)
eval_case = EvalCase(
eval_id="case1",
conversation=[
Invocation(
user_content=genai_types.Content(
parts=[genai_types.Part(text="expected invocation 1.")]
)
),
Invocation(
user_content=genai_types.Content(
parts=[genai_types.Part(text="expected invocation 2.")]
)
),
],
rubrics=[rubric1],
)
invocations = [
Invocation(
user_content=genai_types.Content(
parts=[genai_types.Part(text="actual invocation 1.")]
)
),
Invocation(
user_content=genai_types.Content(
parts=[genai_types.Part(text="actual invocation 2.")]
)
),
]
_copy_eval_case_rubrics_to_actual_invocations(eval_case, invocations)
assert invocations[0].rubrics == [rubric1]
assert invocations[1].rubrics == [rubric1]
def test_copy_invocation_rubrics_to_actual_invocations():
rubric1 = Rubric(
rubric_id="r1", rubric_content=RubricContent(text_property="p1")
)
rubric2 = Rubric(
rubric_id="r2", rubric_content=RubricContent(text_property="p2")
)
expected = [
Invocation(
user_content=genai_types.Content(
parts=[genai_types.Part(text="expected invocation 1.")]
),
rubrics=[rubric1],
),
Invocation(
user_content=genai_types.Content(
parts=[genai_types.Part(text="expected invocation 2.")]
),
rubrics=[rubric2],
),
]
actual = [
Invocation(
user_content=genai_types.Content(
parts=[genai_types.Part(text="actual invocation 1.")]
)
),
Invocation(
user_content=genai_types.Content(
parts=[genai_types.Part(text="actual invocation 2.")]
)
),
]
_copy_invocation_rubrics_to_actual_invocations(expected, actual)
assert actual[0].rubrics == [rubric1]
assert actual[1].rubrics == [rubric2]
tests/unittests/evaluation/test_rubric_based_evaluator.py
+73 -1
View File
@@ -465,6 +465,7 @@ class TestRubricBasedEvaluator:
evaluator: RubricBasedEvaluator,
):
"""Tests convert_auto_rater_response_to_score with an empty response."""
evaluator.create_effective_rubrics_list(None)
response = LlmResponse(
content=genai_types.Content(parts=[genai_types.Part(text="")])
)
@@ -477,6 +478,7 @@ class TestRubricBasedEvaluator:
evaluator: RubricBasedEvaluator,
):
"""Tests convert_auto_rater_response_to_score with a malformed response."""
evaluator.create_effective_rubrics_list(None)
response = LlmResponse(
content=genai_types.Content(
parts=[genai_types.Part(text="This is not a valid format.")]
@@ -491,6 +493,7 @@ class TestRubricBasedEvaluator:
evaluator: RubricBasedEvaluator,
):
"""Tests convert_auto_rater_response_to_score with mixed verdicts."""
evaluator.create_effective_rubrics_list(None)
response_text = """
Property: Is the response good?
Rationale: It was good.
@@ -515,6 +518,7 @@ class TestRubricBasedEvaluator:
evaluator: RubricBasedEvaluator,
):
"""Tests convert_auto_rater_response_to_score with an invalid verdict."""
evaluator.create_effective_rubrics_list(None)
response_text = """
Property: Is the response good?
Rationale: It was good.
@@ -539,6 +543,7 @@ class TestRubricBasedEvaluator:
evaluator: RubricBasedEvaluator,
):
"""Tests convert_auto_rater_response_to_score with an unknown property."""
evaluator.create_effective_rubrics_list(None)
response_text = """
Property: Is the response amazing?
Rationale: It was amazing.
@@ -551,4 +556,71 @@ class TestRubricBasedEvaluator:
)
auto_rater_score = evaluator.convert_auto_rater_response_to_score(response)
assert auto_rater_score.score is None
assert len(auto_rater_score.rubric_scores) == 0
assert not auto_rater_score.rubric_scores
def test_create_effective_rubrics_list_with_invocation_rubrics(
self, evaluator: RubricBasedEvaluator
):
invocation_rubrics = [
Rubric(
rubric_id="3",
rubric_content=RubricContent(text_property="Invocation rubric"),
)
]
evaluator.create_effective_rubrics_list(invocation_rubrics)
effective_rubrics = evaluator.get_effective_rubrics_list()
assert len(effective_rubrics) == 3
assert {r.rubric_id for r in effective_rubrics} == {"1", "2", "3"}
def test_create_effective_rubrics_list_with_duplicate_invocation_rubric_id(
self, evaluator: RubricBasedEvaluator
):
invocation_rubrics = [
Rubric(
rubric_id="1",
rubric_content=RubricContent(text_property="Invocation rubric"),
)
]
with pytest.raises(ValueError):
evaluator.create_effective_rubrics_list(invocation_rubrics)
def test_create_effective_rubrics_list_with_no_invocation_rubrics(
self, evaluator: RubricBasedEvaluator
):
evaluator.create_effective_rubrics_list(None)
effective_rubrics = evaluator.get_effective_rubrics_list()
assert len(effective_rubrics) == 2
assert {r.rubric_id for r in effective_rubrics} == {"1", "2"}
def test_get_effective_rubrics_list_before_creation_raises_error(
self, evaluator: RubricBasedEvaluator
):
with pytest.raises(
ValueError, match="Effective rubrics list not initialized."
):
evaluator.get_effective_rubrics_list()
def test_create_effective_rubrics_list_multiple_calls(
self, evaluator: RubricBasedEvaluator
):
invocation_rubrics1 = [
Rubric(
rubric_id="3",
rubric_content=RubricContent(text_property="Invocation rubric 1"),
)
]
evaluator.create_effective_rubrics_list(invocation_rubrics1)
effective_rubrics1 = evaluator.get_effective_rubrics_list()
assert len(effective_rubrics1) == 3
assert {r.rubric_id for r in effective_rubrics1} == {"1", "2", "3"}
invocation_rubrics2 = [
Rubric(
rubric_id="4",
rubric_content=RubricContent(text_property="Invocation rubric 2"),
)
]
evaluator.create_effective_rubrics_list(invocation_rubrics2)
effective_rubrics2 = evaluator.get_effective_rubrics_list()
assert len(effective_rubrics2) == 3
assert {r.rubric_id for r in effective_rubrics2} == {"1", "2", "4"}