mirror of
https://github.com/encounter/adk-python.git
synced 2026-07-09 18:19:28 -07:00
feat: Implement auto rater-based evaluator for responses
PiperOrigin-RevId: 780654576
This commit is contained in:
Joseph Pagadora
committed by
Copybara-Service
Copybara-Service
parent
45d60a1906
commit
75699fbeca
@@ -0,0 +1,478 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from google.adk.evaluation.eval_case import Invocation
|
||||
from google.adk.evaluation.eval_metrics import EvalMetric
|
||||
from google.adk.evaluation.eval_metrics import JudgeModelOptions
|
||||
from google.adk.evaluation.evaluator import EvalStatus
|
||||
from google.adk.evaluation.evaluator import PerInvocationResult
|
||||
from google.adk.evaluation.final_response_match_v2 import _parse_critique
|
||||
from google.adk.evaluation.final_response_match_v2 import FinalResponseMatchV2Evaluator
|
||||
from google.adk.evaluation.llm_as_judge_utils import Label
|
||||
from google.adk.models.llm_response import LlmResponse
|
||||
from google.genai import types as genai_types
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"response_text",
|
||||
[
|
||||
"""```json
|
||||
{
|
||||
"is_the_agent_response_valid_or_invalid": "valid",
|
||||
"reasoning": "The response is valid."
|
||||
}
|
||||
```""",
|
||||
"""```json
|
||||
{
|
||||
"is_the_agent_response_valid": "undefined label",
|
||||
}
|
||||
```""",
|
||||
],
|
||||
)
|
||||
def test_parse_critique_label_not_found(response_text):
|
||||
label = _parse_critique(response_text)
|
||||
assert label == Label.NOT_FOUND
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"response_text",
|
||||
[
|
||||
"""```json
|
||||
{
|
||||
"is_the_agent_response_valid": "valid",
|
||||
"reasoning": "The response is valid."
|
||||
}
|
||||
```""",
|
||||
"""```json
|
||||
{
|
||||
"is_the_agent_response_valid": ["valid"],
|
||||
"reasoning": "The response is valid."
|
||||
}
|
||||
```""",
|
||||
"""```json
|
||||
{
|
||||
"is_the_agent_response_valid":\n [ "valid\n"],
|
||||
"reasoning": "The response is valid."
|
||||
}
|
||||
```""",
|
||||
],
|
||||
)
|
||||
def test_parse_critique(response_text):
|
||||
label = _parse_critique(response_text)
|
||||
assert label == Label.VALID
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"response_text",
|
||||
[
|
||||
"""```json
|
||||
{
|
||||
"is_the_agent_response_invalid": "invalid",
|
||||
"reasoning": "The response is invalid."
|
||||
}
|
||||
```""",
|
||||
"""```json
|
||||
{
|
||||
"is_the_agent_response_invalid": ["invalid"],
|
||||
"reasoning": "The response is invalid."
|
||||
}
|
||||
```""",
|
||||
"""```json
|
||||
{
|
||||
"is_the_agent_response_invalid":\n [ "invalid\n"],
|
||||
"reasoning": "The response is invalid."
|
||||
}
|
||||
```""",
|
||||
],
|
||||
)
|
||||
def test_parse_critique_invalid(response_text):
|
||||
label = _parse_critique(response_text)
|
||||
assert label == Label.INVALID
|
||||
|
||||
|
||||
def create_test_template() -> str:
|
||||
return """
|
||||
This is a test template.
|
||||
|
||||
{{
|
||||
"User prompt": {prompt},
|
||||
"Agent response": {response},
|
||||
"Reference response": {golden_response},
|
||||
}}
|
||||
|
||||
The answer should be a json alone which follows the json structure below:
|
||||
{{
|
||||
"is_the_agent_response_valid": [valid or invalid],
|
||||
"reasoning":
|
||||
}}
|
||||
"""
|
||||
|
||||
|
||||
def _create_test_evaluator_gemini(
|
||||
threshold: float,
|
||||
) -> FinalResponseMatchV2Evaluator:
|
||||
evaluator = FinalResponseMatchV2Evaluator(
|
||||
EvalMetric(
|
||||
metric_name="final_response_match_v2",
|
||||
threshold=threshold,
|
||||
judge_model_options=JudgeModelOptions(
|
||||
judge_model="gemini-2.5-flash",
|
||||
num_samples=3,
|
||||
),
|
||||
),
|
||||
)
|
||||
evaluator._auto_rater_prompt_template = create_test_template()
|
||||
return evaluator
|
||||
|
||||
|
||||
def _create_test_invocations(
|
||||
candidate: str, reference: str
|
||||
) -> tuple[Invocation, Invocation]:
|
||||
"""Returns tuple of (actual_invocation, expected_invocation)."""
|
||||
actual_invocation = Invocation(
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="This is a test query.")],
|
||||
role="user",
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[genai_types.Part(text=candidate)],
|
||||
role="model",
|
||||
),
|
||||
)
|
||||
expected_invocation = Invocation(
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="This is a test query.")],
|
||||
role="user",
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[genai_types.Part(text=reference)],
|
||||
role="model",
|
||||
),
|
||||
)
|
||||
return actual_invocation, expected_invocation
|
||||
|
||||
|
||||
def test_format_auto_rater_prompt():
|
||||
evaluator = _create_test_evaluator_gemini(threshold=0.8)
|
||||
actual_invocation, expected_invocation = _create_test_invocations(
|
||||
"candidate text", "reference text"
|
||||
)
|
||||
prompt = evaluator.format_auto_rater_prompt(
|
||||
actual_invocation, expected_invocation
|
||||
)
|
||||
assert prompt == """
|
||||
This is a test template.
|
||||
|
||||
{
|
||||
"User prompt": This is a test query.,
|
||||
"Agent response": candidate text,
|
||||
"Reference response": reference text,
|
||||
}
|
||||
|
||||
The answer should be a json alone which follows the json structure below:
|
||||
{
|
||||
"is_the_agent_response_valid": [valid or invalid],
|
||||
"reasoning":
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def test_convert_auto_rater_response_to_score_valid():
|
||||
evaluator = _create_test_evaluator_gemini(threshold=0.8)
|
||||
auto_rater_response = """```json
|
||||
{
|
||||
"is_the_agent_response_valid": "valid",
|
||||
"reasoning": "The response is valid."
|
||||
}
|
||||
```"""
|
||||
llm_response = LlmResponse(
|
||||
content=genai_types.Content(
|
||||
parts=[genai_types.Part(text=auto_rater_response)],
|
||||
role="model",
|
||||
)
|
||||
)
|
||||
score = evaluator.convert_auto_rater_response_to_score(llm_response)
|
||||
assert score == 1.0
|
||||
|
||||
|
||||
def test_convert_auto_rater_response_to_score_invalid():
|
||||
evaluator = _create_test_evaluator_gemini(threshold=0.8)
|
||||
auto_rater_response = """```json
|
||||
{
|
||||
"is_the_agent_response_valid": "invalid",
|
||||
"reasoning": "The response is invalid."
|
||||
}
|
||||
```"""
|
||||
llm_response = LlmResponse(
|
||||
content=genai_types.Content(
|
||||
parts=[genai_types.Part(text=auto_rater_response)],
|
||||
role="model",
|
||||
)
|
||||
)
|
||||
score = evaluator.convert_auto_rater_response_to_score(llm_response)
|
||||
assert score == 0.0
|
||||
|
||||
|
||||
def test_convert_auto_rater_response_to_score_invalid_json():
|
||||
evaluator = _create_test_evaluator_gemini(threshold=0.8)
|
||||
llm_response = LlmResponse(
|
||||
content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="invalid json")],
|
||||
role="model",
|
||||
)
|
||||
)
|
||||
score = evaluator.convert_auto_rater_response_to_score(llm_response)
|
||||
assert score is None
|
||||
|
||||
|
||||
def test_convert_auto_rater_response_to_score_missing_key():
|
||||
evaluator = _create_test_evaluator_gemini(threshold=0.8)
|
||||
llm_response = LlmResponse(
|
||||
content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="{}")],
|
||||
role="model",
|
||||
)
|
||||
)
|
||||
score = evaluator.convert_auto_rater_response_to_score(llm_response)
|
||||
assert score is None
|
||||
|
||||
|
||||
def test_aggregate_per_invocation_samples_none_evaluated():
|
||||
evaluator = _create_test_evaluator_gemini(threshold=0.5)
|
||||
|
||||
actual_invocation, expected_invocation = _create_test_invocations(
|
||||
"candidate text", "reference text"
|
||||
)
|
||||
|
||||
per_invocation_result_samples = [
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=None,
|
||||
eval_status=EvalStatus.NOT_EVALUATED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=None,
|
||||
eval_status=EvalStatus.NOT_EVALUATED,
|
||||
),
|
||||
]
|
||||
|
||||
assert (
|
||||
evaluator.aggregate_per_invocation_samples(per_invocation_result_samples)
|
||||
== per_invocation_result_samples[0]
|
||||
)
|
||||
|
||||
|
||||
def test_aggregate_per_invocation_samples_valid():
|
||||
evaluator = _create_test_evaluator_gemini(threshold=0.5)
|
||||
|
||||
actual_invocation, expected_invocation = _create_test_invocations(
|
||||
"candidate text", "reference text"
|
||||
)
|
||||
|
||||
per_invocation_result_samples = [
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=1.0,
|
||||
eval_status=EvalStatus.PASSED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=1.0,
|
||||
eval_status=EvalStatus.PASSED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=0.0,
|
||||
eval_status=EvalStatus.FAILED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=0.0,
|
||||
eval_status=EvalStatus.FAILED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=1.0,
|
||||
eval_status=EvalStatus.PASSED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=1.0,
|
||||
eval_status=EvalStatus.NOT_EVALUATED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=None,
|
||||
eval_status=EvalStatus.NOT_EVALUATED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=0.0,
|
||||
eval_status=EvalStatus.NOT_EVALUATED,
|
||||
),
|
||||
]
|
||||
|
||||
per_invocation_result = evaluator.aggregate_per_invocation_samples(
|
||||
per_invocation_result_samples
|
||||
)
|
||||
|
||||
assert per_invocation_result.score == 1.0
|
||||
assert per_invocation_result.eval_status == EvalStatus.PASSED
|
||||
|
||||
|
||||
def test_aggregate_per_invocation_samples_invalid():
|
||||
evaluator = _create_test_evaluator_gemini(threshold=0.5)
|
||||
|
||||
actual_invocation, expected_invocation = _create_test_invocations(
|
||||
"candidate text", "reference text"
|
||||
)
|
||||
|
||||
per_invocation_result_samples = [
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=0.0,
|
||||
eval_status=EvalStatus.FAILED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=1.0,
|
||||
eval_status=EvalStatus.PASSED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=0.0,
|
||||
eval_status=EvalStatus.FAILED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=0.0,
|
||||
eval_status=EvalStatus.FAILED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=1.0,
|
||||
eval_status=EvalStatus.PASSED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=1.0,
|
||||
eval_status=EvalStatus.PASSED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=1.0,
|
||||
eval_status=EvalStatus.NOT_EVALUATED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=None,
|
||||
eval_status=EvalStatus.NOT_EVALUATED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=0.0,
|
||||
eval_status=EvalStatus.NOT_EVALUATED,
|
||||
),
|
||||
]
|
||||
|
||||
per_invocation_result = evaluator.aggregate_per_invocation_samples(
|
||||
per_invocation_result_samples
|
||||
)
|
||||
|
||||
assert per_invocation_result.score == 0.0
|
||||
assert per_invocation_result.eval_status == EvalStatus.FAILED
|
||||
|
||||
|
||||
def test_aggregate_invocation_results():
|
||||
evaluator = _create_test_evaluator_gemini(threshold=0.5)
|
||||
|
||||
actual_invocation, expected_invocation = _create_test_invocations(
|
||||
"candidate text", "reference text"
|
||||
)
|
||||
|
||||
per_invocation_results = [
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=1.0,
|
||||
eval_status=EvalStatus.PASSED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=1.0,
|
||||
eval_status=EvalStatus.PASSED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=0.0,
|
||||
eval_status=EvalStatus.FAILED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=0.0,
|
||||
eval_status=EvalStatus.FAILED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=None,
|
||||
eval_status=EvalStatus.PASSED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=100.0,
|
||||
eval_status=EvalStatus.NOT_EVALUATED,
|
||||
),
|
||||
PerInvocationResult(
|
||||
actual_invocation=actual_invocation,
|
||||
expected_invocation=expected_invocation,
|
||||
score=None,
|
||||
eval_status=EvalStatus.NOT_EVALUATED,
|
||||
),
|
||||
]
|
||||
|
||||
aggregated_result = evaluator.aggregate_invocation_results(
|
||||
per_invocation_results
|
||||
)
|
||||
|
||||
# Only 4 / 8 invocations are evaluated, and 2 / 4 are valid.
|
||||
assert aggregated_result.overall_score == 0.5
|
||||
assert aggregated_result.overall_eval_status == EvalStatus.PASSED
|
||||
@@ -0,0 +1,221 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from google.adk.evaluation.eval_case import Invocation
|
||||
from google.adk.evaluation.eval_metrics import EvalMetric
|
||||
from google.adk.evaluation.eval_metrics import JudgeModelOptions
|
||||
from google.adk.evaluation.evaluator import EvalStatus
|
||||
from google.adk.evaluation.evaluator import EvaluationResult
|
||||
from google.adk.evaluation.evaluator import PerInvocationResult
|
||||
from google.adk.evaluation.llm_as_judge import LlmAsJudge
|
||||
from google.adk.evaluation.llm_as_judge_utils import get_eval_status
|
||||
from google.adk.evaluation.llm_as_judge_utils import get_text_from_content
|
||||
from google.adk.models.llm_response import LlmResponse
|
||||
from google.genai import types as genai_types
|
||||
import pytest
|
||||
|
||||
|
||||
class MockLlmAsJudge(LlmAsJudge):
|
||||
|
||||
def format_auto_rater_prompt(
|
||||
self, actual_invocation: Invocation, expected_invocation: Invocation
|
||||
) -> str:
|
||||
return "formatted prompt"
|
||||
|
||||
def convert_auto_rater_response_to_score(
|
||||
self, llm_response: LlmResponse
|
||||
) -> Optional[float]:
|
||||
return 1.0
|
||||
|
||||
def aggregate_per_invocation_samples(
|
||||
self,
|
||||
per_invocation_samples: list[PerInvocationResult],
|
||||
) -> PerInvocationResult:
|
||||
return per_invocation_samples[0]
|
||||
|
||||
def aggregate_invocation_results(
|
||||
self, per_invocation_results: list[PerInvocationResult]
|
||||
) -> EvaluationResult:
|
||||
return EvaluationResult(
|
||||
overall_score=1.0, overall_eval_status=EvalStatus.PASSED
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_llm_as_judge():
|
||||
return MockLlmAsJudge(
|
||||
EvalMetric(
|
||||
metric_name="test_metric",
|
||||
threshold=0.5,
|
||||
judge_model_options=JudgeModelOptions(
|
||||
judge_model="gemini-2.5-flash",
|
||||
judge_model_config=genai_types.GenerateContentConfig(),
|
||||
num_samples=3,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_get_text_from_content():
|
||||
content = genai_types.Content(
|
||||
parts=[
|
||||
genai_types.Part(text="This is a test text."),
|
||||
genai_types.Part(text="This is another test text."),
|
||||
],
|
||||
role="model",
|
||||
)
|
||||
assert (
|
||||
get_text_from_content(content)
|
||||
== "This is a test text.\nThis is another test text."
|
||||
)
|
||||
|
||||
|
||||
def test_get_eval_status():
|
||||
assert get_eval_status(score=0.8, threshold=0.8) == EvalStatus.PASSED
|
||||
assert get_eval_status(score=0.7, threshold=0.8) == EvalStatus.FAILED
|
||||
assert get_eval_status(score=0.8, threshold=0.9) == EvalStatus.FAILED
|
||||
assert get_eval_status(score=0.9, threshold=0.8) == EvalStatus.PASSED
|
||||
assert get_eval_status(score=None, threshold=0.8) == EvalStatus.NOT_EVALUATED
|
||||
|
||||
|
||||
def test_llm_as_judge_init_missing_judge_model_options():
|
||||
with pytest.raises(ValueError):
|
||||
MockLlmAsJudge(
|
||||
EvalMetric(metric_name="test_metric", threshold=0.8),
|
||||
)
|
||||
|
||||
|
||||
def test_llm_as_judge_init_unregistered_model():
|
||||
with pytest.raises(ValueError):
|
||||
MockLlmAsJudge(
|
||||
EvalMetric(
|
||||
metric_name="test_metric",
|
||||
threshold=0.8,
|
||||
judge_model_options=JudgeModelOptions(
|
||||
judge_model="unregistered_model",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_judge_model():
|
||||
mock_judge_model = MagicMock()
|
||||
|
||||
async def mock_generate_content_async(llm_request):
|
||||
yield LlmResponse(
|
||||
content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="auto rater response")],
|
||||
)
|
||||
)
|
||||
|
||||
mock_judge_model.generate_content_async = mock_generate_content_async
|
||||
return mock_judge_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_evaluate_invocations_with_mock(
|
||||
mock_llm_as_judge, mock_judge_model
|
||||
):
|
||||
mock_llm_as_judge._judge_model = mock_judge_model
|
||||
|
||||
mock_format_auto_rater_prompt = MagicMock(
|
||||
wraps=mock_llm_as_judge.format_auto_rater_prompt
|
||||
)
|
||||
mock_llm_as_judge.format_auto_rater_prompt = mock_format_auto_rater_prompt
|
||||
|
||||
mock_convert_auto_rater_response_to_score = MagicMock(
|
||||
wraps=mock_llm_as_judge.convert_auto_rater_response_to_score
|
||||
)
|
||||
mock_llm_as_judge.convert_auto_rater_response_to_score = (
|
||||
mock_convert_auto_rater_response_to_score
|
||||
)
|
||||
|
||||
mock_aggregate_per_invocation_samples = MagicMock(
|
||||
wraps=mock_llm_as_judge.aggregate_per_invocation_samples
|
||||
)
|
||||
mock_llm_as_judge.aggregate_per_invocation_samples = (
|
||||
mock_aggregate_per_invocation_samples
|
||||
)
|
||||
|
||||
mock_aggregate_invocation_results = MagicMock(
|
||||
wraps=mock_llm_as_judge.aggregate_invocation_results
|
||||
)
|
||||
mock_llm_as_judge.aggregate_invocation_results = (
|
||||
mock_aggregate_invocation_results
|
||||
)
|
||||
|
||||
actual_invocations = [
|
||||
Invocation(
|
||||
invocation_id="id1",
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="user content 1")],
|
||||
role="user",
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[genai_types.Part(text="final response 1")],
|
||||
role="model",
|
||||
),
|
||||
),
|
||||
Invocation(
|
||||
invocation_id="id2",
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="user content 2")],
|
||||
role="user",
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[genai_types.Part(text="final response 2")],
|
||||
role="model",
|
||||
),
|
||||
),
|
||||
]
|
||||
expected_invocations = [
|
||||
Invocation(
|
||||
invocation_id="id1",
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="user content 1")],
|
||||
role="user",
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[genai_types.Part(text="expected response 1")],
|
||||
role="model",
|
||||
),
|
||||
),
|
||||
Invocation(
|
||||
invocation_id="id2",
|
||||
user_content=genai_types.Content(
|
||||
parts=[genai_types.Part(text="user content 2")],
|
||||
role="user",
|
||||
),
|
||||
final_response=genai_types.Content(
|
||||
parts=[genai_types.Part(text="expected response 2")],
|
||||
role="model",
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
result = await mock_llm_as_judge.evaluate_invocations(
|
||||
actual_invocations, expected_invocations
|
||||
)
|
||||
|
||||
# Assertions
|
||||
assert result.overall_score == 1.0
|
||||
assert mock_llm_as_judge.format_auto_rater_prompt.call_count == 2
|
||||
assert mock_llm_as_judge.convert_auto_rater_response_to_score.call_count == 6
|
||||
assert mock_llm_as_judge.aggregate_invocation_results.call_count == 1
|
||||
Reference in New Issue
Block a user