feat: Data model for Rubric based metric and eval config

Details:
- We plan on introducing Rubric based metrics in subsequent changes. This change introduces the data model needed that allows agent developer to provide rubrics.

- We also introduce a data model for the config that the eval system has been using for quite some time. It was loosely and informally described as a dictionary of metric names and expected thresholds. In this change, we actually formalize it using a pydantic data model, and extend it allow developers to specify rubrics as a part of their eval config.

What is a rubric based metric?
A rubric based metric is the assessment of a Agent's response (final or intermediate) along some rubric. This evaluation of agent's response significantly differs from the strategy where one has to provide a golden response.

PiperOrigin-RevId: 805488436
This commit is contained in:
Ankur Sharma
2025-09-10 13:20:07 -07:00
committed by Copybara-Service
parent 37228beddd
commit e88e667770
14 changed files with 484 additions and 98 deletions
@@ -15,6 +15,7 @@
from __future__ import annotations
from google.adk.evaluation.eval_case import Invocation
from google.adk.evaluation.eval_metrics import BaseCriterion
from google.adk.evaluation.eval_metrics import EvalMetric
from google.adk.evaluation.eval_metrics import JudgeModelOptions
from google.adk.evaluation.eval_metrics import PrebuiltMetrics
@@ -130,9 +131,8 @@ def _create_test_evaluator_gemini(
EvalMetric(
metric_name="final_response_match_v2",
threshold=threshold,
judge_model_options=JudgeModelOptions(
judge_model="gemini-2.5-flash",
num_samples=3,
criterion=BaseCriterion(
threshold=0.5,
),
),
)
@@ -20,6 +20,7 @@ from unittest.mock import MagicMock
from google.adk.evaluation.eval_case import Invocation
from google.adk.evaluation.eval_metrics import EvalMetric
from google.adk.evaluation.eval_metrics import JudgeModelOptions
from google.adk.evaluation.eval_metrics import LlmAsAJudgeCriterion
from google.adk.evaluation.evaluator import EvalStatus
from google.adk.evaluation.evaluator import EvaluationResult
from google.adk.evaluation.evaluator import PerInvocationResult
@@ -60,15 +61,19 @@ class MockLlmAsJudge(LlmAsJudge):
@pytest.fixture
def mock_llm_as_judge():
return MockLlmAsJudge(
EvalMetric(
eval_metric=EvalMetric(
metric_name="test_metric",
threshold=0.5,
judge_model_options=JudgeModelOptions(
judge_model="gemini-2.5-flash",
judge_model_config=genai_types.GenerateContentConfig(),
num_samples=3,
criterion=LlmAsAJudgeCriterion(
threshold=0.5,
judge_model_options=JudgeModelOptions(
judge_model="gemini-2.5-flash",
judge_model_config=genai_types.GenerateContentConfig(),
num_samples=3,
),
),
),
criterion_type=LlmAsAJudgeCriterion,
)
@@ -94,10 +99,11 @@ def test_get_eval_status():
assert get_eval_status(score=None, threshold=0.8) == EvalStatus.NOT_EVALUATED
def test_llm_as_judge_init_missing_judge_model_options():
def test_llm_as_judge_init_missing_criterion():
with pytest.raises(ValueError):
MockLlmAsJudge(
EvalMetric(metric_name="test_metric", threshold=0.8),
criterion_type=LlmAsAJudgeCriterion,
)
@@ -107,10 +113,16 @@ def test_llm_as_judge_init_unregistered_model():
EvalMetric(
metric_name="test_metric",
threshold=0.8,
judge_model_options=JudgeModelOptions(
judge_model="unregistered_model",
criterion=LlmAsAJudgeCriterion(
threshold=0.5,
judge_model_options=JudgeModelOptions(
judge_model="unregistered_model",
judge_model_config=genai_types.GenerateContentConfig(),
num_samples=3,
),
),
),
criterion_type=LlmAsAJudgeCriterion,
)