mirror of
https://github.com/encounter/adk-python.git
synced 2026-07-09 18:19:28 -07:00
feat: Data model for Rubric based metric and eval config
Details: - We plan on introducing Rubric based metrics in subsequent changes. This change introduces the data model needed that allows agent developer to provide rubrics. - We also introduce a data model for the config that the eval system has been using for quite some time. It was loosely and informally described as a dictionary of metric names and expected thresholds. In this change, we actually formalize it using a pydantic data model, and extend it allow developers to specify rubrics as a part of their eval config. What is a rubric based metric? A rubric based metric is the assessment of a Agent's response (final or intermediate) along some rubric. This evaluation of agent's response significantly differs from the strategy where one has to provide a golden response. PiperOrigin-RevId: 805488436
This commit is contained in:
committed by
Copybara-Service
parent
37228beddd
commit
e88e667770
@@ -15,6 +15,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from google.adk.evaluation.eval_case import Invocation
|
||||
from google.adk.evaluation.eval_metrics import BaseCriterion
|
||||
from google.adk.evaluation.eval_metrics import EvalMetric
|
||||
from google.adk.evaluation.eval_metrics import JudgeModelOptions
|
||||
from google.adk.evaluation.eval_metrics import PrebuiltMetrics
|
||||
@@ -130,9 +131,8 @@ def _create_test_evaluator_gemini(
|
||||
EvalMetric(
|
||||
metric_name="final_response_match_v2",
|
||||
threshold=threshold,
|
||||
judge_model_options=JudgeModelOptions(
|
||||
judge_model="gemini-2.5-flash",
|
||||
num_samples=3,
|
||||
criterion=BaseCriterion(
|
||||
threshold=0.5,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -20,6 +20,7 @@ from unittest.mock import MagicMock
|
||||
from google.adk.evaluation.eval_case import Invocation
|
||||
from google.adk.evaluation.eval_metrics import EvalMetric
|
||||
from google.adk.evaluation.eval_metrics import JudgeModelOptions
|
||||
from google.adk.evaluation.eval_metrics import LlmAsAJudgeCriterion
|
||||
from google.adk.evaluation.evaluator import EvalStatus
|
||||
from google.adk.evaluation.evaluator import EvaluationResult
|
||||
from google.adk.evaluation.evaluator import PerInvocationResult
|
||||
@@ -60,15 +61,19 @@ class MockLlmAsJudge(LlmAsJudge):
|
||||
@pytest.fixture
|
||||
def mock_llm_as_judge():
|
||||
return MockLlmAsJudge(
|
||||
EvalMetric(
|
||||
eval_metric=EvalMetric(
|
||||
metric_name="test_metric",
|
||||
threshold=0.5,
|
||||
judge_model_options=JudgeModelOptions(
|
||||
judge_model="gemini-2.5-flash",
|
||||
judge_model_config=genai_types.GenerateContentConfig(),
|
||||
num_samples=3,
|
||||
criterion=LlmAsAJudgeCriterion(
|
||||
threshold=0.5,
|
||||
judge_model_options=JudgeModelOptions(
|
||||
judge_model="gemini-2.5-flash",
|
||||
judge_model_config=genai_types.GenerateContentConfig(),
|
||||
num_samples=3,
|
||||
),
|
||||
),
|
||||
),
|
||||
criterion_type=LlmAsAJudgeCriterion,
|
||||
)
|
||||
|
||||
|
||||
@@ -94,10 +99,11 @@ def test_get_eval_status():
|
||||
assert get_eval_status(score=None, threshold=0.8) == EvalStatus.NOT_EVALUATED
|
||||
|
||||
|
||||
def test_llm_as_judge_init_missing_judge_model_options():
|
||||
def test_llm_as_judge_init_missing_criterion():
|
||||
with pytest.raises(ValueError):
|
||||
MockLlmAsJudge(
|
||||
EvalMetric(metric_name="test_metric", threshold=0.8),
|
||||
criterion_type=LlmAsAJudgeCriterion,
|
||||
)
|
||||
|
||||
|
||||
@@ -107,10 +113,16 @@ def test_llm_as_judge_init_unregistered_model():
|
||||
EvalMetric(
|
||||
metric_name="test_metric",
|
||||
threshold=0.8,
|
||||
judge_model_options=JudgeModelOptions(
|
||||
judge_model="unregistered_model",
|
||||
criterion=LlmAsAJudgeCriterion(
|
||||
threshold=0.5,
|
||||
judge_model_options=JudgeModelOptions(
|
||||
judge_model="unregistered_model",
|
||||
judge_model_config=genai_types.GenerateContentConfig(),
|
||||
num_samples=3,
|
||||
),
|
||||
),
|
||||
),
|
||||
criterion_type=LlmAsAJudgeCriterion,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user