From 63353b2b74e23e97385892415c5a3f2a59c3504f Mon Sep 17 00:00:00 2001 From: Google Team Member Date: Tue, 4 Nov 2025 22:16:22 -0800 Subject: [PATCH] feat: Refactor gepa sample code and clean-up user demo colab PiperOrigin-RevId: 828293079 --- contributing/samples/gepa/adk_agent.py | 17 +- contributing/samples/gepa/experiment.py | 675 +++++ .../samples/gepa/gepa_tau_bench.ipynb | 2217 +++++++++++------ contributing/samples/gepa/rater_lib.py | 178 ++ .../gepa/rubric_validation_template.txt | 170 ++ contributing/samples/gepa/run_experiment.py | 116 + 6 files changed, 2558 insertions(+), 815 deletions(-) create mode 100644 contributing/samples/gepa/experiment.py create mode 100644 contributing/samples/gepa/rater_lib.py create mode 100644 contributing/samples/gepa/rubric_validation_template.txt create mode 100644 contributing/samples/gepa/run_experiment.py diff --git a/contributing/samples/gepa/adk_agent.py b/contributing/samples/gepa/adk_agent.py index ad3570dd..15e153da 100644 --- a/contributing/samples/gepa/adk_agent.py +++ b/contributing/samples/gepa/adk_agent.py @@ -31,8 +31,10 @@ from google.adk.agents import llm_agent from google.adk.agents import loop_agent from google.adk.events import event as event_lib from google.adk.models import google_llm +from google.adk.planners import built_in_planner from google.adk.tools import base_tool from google.genai import types +from retry import api as retry class EnvResponse(Protocol): @@ -134,6 +136,11 @@ def _adk_agent( model=model or 'gemini-2.5-flash', retry_options=_default_retry_options(), ), + planner=built_in_planner.BuiltInPlanner( + thinking_config=types.ThinkingConfig( + thinking_budget=-1, include_thoughts=False + ) + ), instruction=instruction, tools=tools, generate_content_config=types.GenerateContentConfig( @@ -173,11 +180,17 @@ class _UserAgent(base_agent.BaseAgent): return if last_event.content and last_event.content.parts: - next_message = last_event.content.parts[-1].text + next_message = '\n\n'.join([p.text for p in last_event.content.parts]) else: logging.warn('Empty content with event=%s', last_event) next_message = '' - env_response = self.env.step(types.Part(text=next_message)) + env_response = retry.retry_call( + self.env.step, + fargs=(types.Part(text=next_message),), + tries=3, + delay=2, + backoff=2, + ) output_event = event_lib.Event( content=types.Content( diff --git a/contributing/samples/gepa/experiment.py b/contributing/samples/gepa/experiment.py new file mode 100644 index 00000000..b7a79cae --- /dev/null +++ b/contributing/samples/gepa/experiment.py @@ -0,0 +1,675 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Runs Tau-bench.""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +import dataclasses +from datetime import datetime +import json +import logging +import multiprocessing +import os +import random +import traceback +from typing import Any +from typing import Callable +from typing import TypedDict + +import gepa +from gepa.core.adapter import EvaluationBatch +from gepa.core.adapter import GEPAAdapter +from google.genai import types +from litellm import provider_list +import rater_lib +from retry import retry +from tau_bench.envs import get_env +from tau_bench.envs.retail import tasks_dev +from tau_bench.envs.retail import tasks_test +from tau_bench.envs.retail import tasks_train +from tau_bench.envs.user import UserStrategy +from tau_bench.run import display_metrics +from tau_bench.types import EnvRunResult +from tau_bench.types import RunConfig +import tau_bench_agent as tau_bench_agent_lib + +from google import genai + + +class FilterInferenceWarnings(logging.Filter): + """Filters out Vertex inference warning about non-text parts in response.""" + + def filter(self, record: logging.LogRecord) -> bool: + """Filters out Vertex inference warning about non-text parts in response.""" + if record.levelname != 'WARNING': + return True + message_identifier = record.getMessage() + return not message_identifier.startswith( + 'Warning: there are non-text parts in the response:' + ) + + +def run_tau_bench_rollouts( + config: RunConfig, + print_results: bool = False, + system_instruction: str | None = None, + rater: rater_lib.Rater | None = None, +) -> list[EnvRunResult]: + """Runs a set of tau-bench tasks with a given agent configuration. + + This is a customized version of the standard tau-bench run function, adapted + for this experiment's needs. It handles environment setup, agent creation, + task execution in parallel, and result aggregation. + + Args: + config: A RunConfig object specifying the environment, models, and other + parameters for the run. + print_results: If True, prints the result of each task as it completes. + system_instruction: An optional system instruction to use for the agent, + overriding the default. + rater: An optional rater to evaluate the agent's performance. + + Returns: + A list of EnvRunResult objects, one for each completed task. + """ + if config.env not in ['retail', 'airline']: + raise ValueError('Only retail and airline envs are supported') + if config.model_provider not in provider_list: + raise ValueError('Invalid model provider') + if config.user_model_provider not in provider_list: + raise ValueError('Invalid user model provider') + if config.agent_strategy not in ['tool-calling', 'act', 'react', 'few-shot']: + raise ValueError('Invalid agent strategy') + if config.task_split not in ['train', 'test', 'dev']: + raise ValueError('Invalid task split') + if config.user_strategy not in [item.value for item in UserStrategy]: + raise ValueError('Invalid user strategy') + + random.seed(config.seed) + time_str = datetime.now().strftime('%m%d%H%M%S') + model_name = config.model.split('/')[-1] + ckpt_filename = ( + f'{config.agent_strategy}-{model_name}-{config.temperature}_range_' + f'{config.start_index}-{config.end_index}_user-{config.user_model}-' + f'{config.user_strategy}_{time_str}.json' + ) + ckpt_path = os.path.join(config.log_dir, ckpt_filename) + if not os.path.exists(config.log_dir): + os.makedirs(config.log_dir) + + print(f'Loading user with strategy: {config.user_strategy}') + env = get_env( + config.env, + user_strategy=config.user_strategy, + user_model=config.user_model, + user_provider=config.user_model_provider, + task_split=config.task_split, + ) + if system_instruction: + env.wiki = system_instruction + agent = tau_bench_agent_lib.adk_agent_factory( + tools_info=env.tools_info, + wiki=env.wiki, + config=config, + ) + if config.end_index == -1: + end_index = len(env.tasks) + else: + end_index = min(config.end_index, len(env.tasks)) + results: list[EnvRunResult] = [] + lock = multiprocessing.Lock() + if config.task_ids: + print(f'Running tasks {config.task_ids} (checkpoint path: {ckpt_path})') + else: + print( + f'Running tasks {config.start_index} to {end_index} ' + f'(checkpoint path: {ckpt_path})' + ) + for i in range(config.num_trials): + if config.task_ids: + idxs = config.task_ids + else: + idxs = list(range(config.start_index, end_index)) + if config.shuffle: + random.shuffle(idxs) + + @retry(tries=3, delay=10, backoff=2) + def _run_with_retry(idx: int) -> EnvRunResult: + isolated_env = get_env( + config.env, + user_strategy=config.user_strategy, + user_model=config.user_model, + task_split=config.task_split, + user_provider=config.user_model_provider, + task_index=idx, + ) + if print_results: + print(f'Running task {idx}') + res = agent.solve( + env=isolated_env, + task_index=idx, + ) + + rating = ( + rater(res.messages[1:] if len(res.messages) > 1 else res.messages) + if rater + else None + ) + info = dict(res.info) + info['metrics'] = dict(rating=rating, reward=res.reward) + + if rater: + score = rating['score'] + feedback = {k: v for k, v in rating.items() if k != 'score'} + else: + score = res.reward + feedback = ( + 'The agent successfully resolved all customer issues' + if score > 0 + else 'The agent failed to resolve all customer issues correctly' + ) + + info['feedback'] = feedback + return EnvRunResult( + task_id=idx, + reward=score, + info=info, + traj=res.messages, + trial=i, + ) + + def _run(idx: int) -> EnvRunResult: + try: + result = _run_with_retry(idx) + except Exception as e: + logging.warning('Inference error: %s', str(e)) + result = EnvRunResult( + task_id=idx, + reward=0.0, + info={ + 'error': str(e), + 'traceback': traceback.format_exc(), + 'metrics': dict(reward=0.0), + }, + traj=[], + trial=i, + ) + + if print_results: + print( + 'āœ…' if result.reward == 1 else 'āŒ', + f'task_id={idx}', + ) + print('-----') + with lock: + data = [] + if os.path.exists(ckpt_path): + with open(ckpt_path, 'r') as f: + data = json.load(f) + with open(ckpt_path, 'w') as f: + json.dump(data + [result.model_dump()], f, indent=2) + return result + + with ThreadPoolExecutor(max_workers=config.max_concurrency) as executor: + res = list(executor.map(_run, idxs)) + results.extend(res) + + display_metrics(results) + + if rater: + print('Environment reward:') + display_metrics([ + EnvRunResult( + task_id=r.task_id, + reward=r.info['metrics']['reward'], + info={}, + traj=[], + trial=r.trial, + ) + for r in results + ]) + + with open(ckpt_path, 'w') as f: + json.dump([result.model_dump() for result in results], f, indent=2) + print(f'\nšŸ“„ Results saved to {ckpt_path}\n') + return results + + +class TauBenchDataInst(TypedDict): + env: str + task_id: int + task_split: str + + +class TauBenchTrajectory(TypedDict): + + result_traj: list[dict[str, Any]] + + +class TauBenchRolloutOutput(TypedDict): + env: str + task_id: int + reward: float + task_info: dict[str, Any] + + +class TauBenchAdapter( + GEPAAdapter[ + TauBenchDataInst, + TauBenchTrajectory, + TauBenchRolloutOutput, + ] +): + """A GEPA adapter for evaluating agent performance on tau-bench benchmark.""" + + def __init__( + self, + env_name: str, + agent_model: str = 'gemini-2.5-flash', + agent_model_provider: str = 'vertex_ai', + user_model: str = 'gemini-2.5-pro', + user_model_provider: str = 'vertex_ai', + agent_strategy: str = 'tool-calling', + user_strategy: str = 'llm', + system_instruction_name: str = 'system_instruction', + max_concurrency: int = 4, + rater: rater_lib.Rater | None = None, + log_dir: str | None = None, + ): + """Initializes the TauBenchAdapter. + + Args: + env_name: environment + agent_model: The model to use for the agent. + agent_model_provider: The provider for the agent model. + user_model: The model to use for simulating the user. + user_model_provider: The provider for the user model. + agent_strategy: The agent strategy to use (e.g., 'tool-calling'). + user_strategy: The user simulation strategy (e.g., 'llm'). + system_instruction_name: The key in the candidate dictionary that holds + the system instruction. + max_concurrency: The maximum number of tasks to run in parallel. + rater: An optional rater to evaluate the agent's performance. + log_dir: The directory to save traces and other logs. + """ + self._env_name = env_name + self._agent_model = agent_model + self._agent_model_provider = agent_model_provider + self._user_model = user_model + self._user_model_provider = user_model_provider + self._agent_strategy = agent_strategy + self._user_strategy = user_strategy + self._max_concurrency = max_concurrency + self._system_instruction_name = system_instruction_name + self._rater = rater + self._log_dir = log_dir + + def evaluate( + self, + batch: list[TauBenchDataInst], + candidate: dict[str, str], + capture_traces: bool = False, + ) -> EvaluationBatch[TauBenchTrajectory, TauBenchRolloutOutput]: + """Evaluates a candidate prompt on a batch of tau-bench tasks. + + This method is called by GEPA during the optimization loop. It takes a + candidate prompt, runs it against the specified tasks from tau-bench, and + returns the results. + + Args: + batch: A list of task instances to evaluate on. Each instance specifies + the environment and task ID. + candidate: A dictionary containing the components to be evaluated, + including the system instruction. + capture_traces: (Not used in this adapter) Whether to capture detailed + traces. + + Returns: + An EvaluationBatch object containing scores, outputs, and trajectories for + each task in the batch. + """ + del capture_traces # Not used. + env = batch[0]['env'] + task_ids = [inst['task_id'] for inst in batch] + tau_bench_run_config = RunConfig( + env=env, + model=self._agent_model, + model_provider=self._agent_model_provider, + user_model=self._user_model, + user_model_provider=self._user_model_provider, + agent_strategy=self._agent_strategy, + user_strategy=self._user_strategy, + max_concurrency=self._max_concurrency, + task_ids=task_ids, + log_dir=self._log_dir, + task_split=batch[0]['task_split'], + ) + tau_bench_results = run_tau_bench_rollouts( + tau_bench_run_config, + system_instruction=candidate.get(self._system_instruction_name), + rater=self._rater, + ) + + outputs = [] + trajectories = [] + scores = [] + for res in tau_bench_results: + outputs.append( + TauBenchRolloutOutput( + env=env, + task_id=res.task_id, + reward=res.reward, + task_info=res.info, + ) + ) + result_traj = res.traj + trajectories.append(TauBenchTrajectory(result_traj=result_traj)) + scores.append(res.reward) + + return EvaluationBatch( + scores=scores, outputs=outputs, trajectories=trajectories + ) + + def make_reflective_dataset( + self, + candidate: dict[str, str], + eval_batch: EvaluationBatch[TauBenchTrajectory, TauBenchRolloutOutput], + components_to_update: list[str], + ) -> dict[str, list[dict[str, Any]]]: + """Creates a dataset for reflection based on evaluation results. + + This method transforms the trajectories and scores from an evaluation run + into a structured format that a reflection model can use to generate + suggestions for improving the prompt. + + Args: + candidate: The candidate that was evaluated. + eval_batch: The results of the evaluation. + components_to_update: A list of component names that the reflection should + focus on improving. + + Returns: + A dictionary where keys are component names and values are lists of + data instances for reflection. + """ + system_instruction = candidate[self._system_instruction_name] + + env = get_env( + self._env_name, + user_strategy=self._user_strategy, + user_model=self._user_model, + user_provider=self._user_model_provider, + task_split='train', + ) + + tool_definitions = json.dumps( + env.tools_info, + indent=2, + default=str, + ) + inputs = '\n\n'.join([ + f'# System Instruction\n{system_instruction}', + f'# Tool Definitions\n{tool_definitions}', + ]) + ret_d: dict[str, list[dict[str, Any]]] = {} + for comp in components_to_update: + items: list[dict[str, Any]] = [] + trace_instances = list( + zip( + eval_batch.trajectories, + eval_batch.scores, + eval_batch.outputs, + strict=True, + ) + ) + for trace_instance in trace_instances: + traj, _, rollout = trace_instance + messages = traj['result_traj'] + # Remove instructions. + if len(messages) > 1: + messages = messages[1:] + d = { + 'Inputs': inputs, + 'Generated Outputs': json.dumps(messages, indent=2, default=str), + 'Feedback': json.dumps( + rollout['task_info']['feedback'], indent=2, default=str + ), + } + items.append(d) + if items: + ret_d[comp] = items + assert ret_d, ( + 'empty reflective dataset for components ' + f'{[comp for comp in components_to_update]}' + ) + return ret_d + + +_DATASET_SPLITS = { + 'train': tasks_train.TASKS_TRAIN, + 'dev': tasks_dev.TASKS_DEV, + 'test': tasks_test.TASKS_TEST, +} + + +def _get_dataset(ds: Dataset) -> list[TauBenchDataInst]: + task_ids = ds.indexes or list(range(len(_DATASET_SPLITS[ds.split]))) + if ds.max_size is not None: + task_ids = task_ids[: ds.max_size] + random.shuffle(task_ids) + return task_ids + + +def _get_datasets( + config: ExperimentConfig, +) -> dict[str, list[int]]: + """Returns Tau-bench dataset splits.""" + random.seed(config.rnd_seed) + train_task_ids = _get_dataset(config.feedback_dataset) + eval_task_ids = _get_dataset(config.pareto_dataset) + test_task_ids = _get_dataset(config.eval_dataset) + logging.info( + 'Using datasets of size: train=%d, eval=%d, test=%d', + len(train_task_ids), + len(eval_task_ids), + len(test_task_ids), + ) + return dict( + train=train_task_ids, + dev=eval_task_ids, + test=test_task_ids, + ) + + +def _reflection_inference_fn(model: str) -> Callable[[str], str]: + """Returns an inference function on VertexAI based on provided model.""" + client = genai.Client() + + @retry(tries=3, delay=10, backoff=2) + def _fn(prompt): + return client.models.generate_content( + model=model, + contents=prompt, + config=types.GenerateContentConfig( + candidate_count=1, + thinking_config=types.ThinkingConfig( + include_thoughts=True, thinking_budget=-1 + ), + ), + ).text + + return _fn + + +SEED_SYSTEM_INSTRUCTION = ( + 'you are a customer support agent helping customers resolve their ' + 'issues by using the right tools' +) + + +@dataclasses.dataclass(frozen=True) +class Dataset: + + split: str + indexes: list[int] | None = None + max_size: int = None + + +@dataclasses.dataclass +class ExperimentConfig: + """Configures a GEPA experiment on Tau-bench.""" + + tau_bench_env: str + agent_model: str + agent_model_provider: str + user_model: str + user_model_provider: str + max_concurrency: int + num_eval_trials: int + rnd_seed: int + max_metric_calls: int + reflection_model: str + reflection_minibatch_size: int + use_rater: bool + feedback_dataset: Dataset + pareto_dataset: Dataset + eval_dataset: Dataset + + +def _rater(config: ExperimentConfig) -> rater_lib.Rater: + env = get_env( + config.tau_bench_env, + user_strategy='llm', + user_model=config.user_model, + user_provider=config.user_model_provider, + task_split='train', + ) + return rater_lib.Rater(json.dumps(env.tools_info, indent=2)) + + +def run_gepa( + output_dir: str, seed_instructions: str, config: ExperimentConfig +) -> Any: + """Runs the GEPA optimization loop to train a new system instruction. + + Args: + output_dir: The directory to save experiment results and artifacts. + seed_instructions: Agent instructions to initialize the agent with. + config: The experiment configuration. + + Returns: + The results of the GEPA optimization. + """ + # This section sets up and runs the GEPA optimization experiment. + # Here we define all the parameters for the tau-bench environment, the GEPA + # optimization loop, and the models to be used. + datasets = _get_datasets(config) + training_set = [ + TauBenchDataInst( + env=config.tau_bench_env, + task_id=task_id, + task_split=config.feedback_dataset.split, + ) + for task_id in datasets['train'] + ] + eval_set = [ + TauBenchDataInst( + env=config.tau_bench_env, + task_id=task_id, + task_split=config.pareto_dataset.split, + ) + for task_id in datasets['dev'] + ] + system_instruction_name = 'system_instruction' + + tau_bench_adapter = TauBenchAdapter( + env_name=config.tau_bench_env, + agent_model=config.agent_model, + agent_model_provider=config.agent_model_provider, + user_model=config.user_model, + user_model_provider=config.user_model_provider, + agent_strategy='tool-calling', + user_strategy='llm', + system_instruction_name=system_instruction_name, + max_concurrency=config.max_concurrency, + rater=_rater(config) if config.use_rater else None, + log_dir=os.path.join(output_dir, 'traces'), + ) + + gepa_results = gepa.optimize( + seed_candidate={ + system_instruction_name: seed_instructions, + }, + trainset=training_set, + valset=eval_set, + task_lm=None, # this must be None when a custom adapter is used + adapter=tau_bench_adapter, + max_metric_calls=config.max_metric_calls, + reflection_lm=_reflection_inference_fn(config.reflection_model), + reflection_minibatch_size=config.reflection_minibatch_size, + run_dir=output_dir, + ) + json.dump( + gepa_results.to_dict(), + open(os.path.join(output_dir, 'results.json'), 'w'), + ) + return gepa_results + + +def run_eval(output_dir: str, instructions: str, config: ExperimentConfig): + """Runs evaluation on the test set using the given instructions. + + Args: + output_dir: The directory to save evaluation results. + instructions: The system instructions to evaluate. + config: The experiment configuration. + """ + eval_dataset = _get_dataset(config.eval_dataset) + tau_bench_run_config = RunConfig( + env=config.tau_bench_env, + model=config.agent_model, + model_provider=config.agent_model_provider, + user_model=config.user_model, + user_model_provider=config.user_model_provider, + agent_strategy='tool-calling', + user_strategy='llm', + max_concurrency=config.max_concurrency, + num_trials=config.num_eval_trials, + task_ids=eval_dataset, + log_dir=output_dir, + task_split=config.eval_dataset.split, + ) + with open(os.path.join(output_dir, 'prompt.txt'), 'w') as f: + f.write(instructions) + + json.dump( + tau_bench_run_config.model_dump(), + open(os.path.join(output_dir, 'run_config.json'), 'w'), + ) + tau_bench_results = run_tau_bench_rollouts( + tau_bench_run_config, + system_instruction=instructions, + rater=_rater(config) if config.use_rater else None, + ) + total = len(tau_bench_results) + numerator = sum(1 for res in tau_bench_results if res.reward == 1) + print( + f'average reward (total={total}): {numerator/total if total > 0 else 0}' + ) + json.dump( + dict(results=[r.model_dump() for r in tau_bench_results]), + open(os.path.join(output_dir, 'results.json'), 'w'), + ) diff --git a/contributing/samples/gepa/gepa_tau_bench.ipynb b/contributing/samples/gepa/gepa_tau_bench.ipynb index 46814bf4..448fdc8d 100644 --- a/contributing/samples/gepa/gepa_tau_bench.ipynb +++ b/contributing/samples/gepa/gepa_tau_bench.ipynb @@ -6,14 +6,32 @@ "id": "882gPGOGM7-i" }, "source": [ - "# Setup" + "# Optimizing Agent Prompts with GEPA on Tau-bench\n", + "\n", + "This demo notebook walks you through optimizing an AI agent's prompt using the\n", + "**Genetic-Pareto (GEPA)** algorithm. We'll use the Google Agent Development\n", + "Kit (ADK) to build and run our agent in **Tau-bench**, a benchmark designed to\n", + "test agents in realistic, conversational scenarios involving tool use and\n", + "adherence to policies.\n", + "\n", + "**Our Goal:** To take a simple, underperforming prompt and automatically\n", + "improve it using GEPA, increasing the agent's reliability on a customer\n", + "support task.\n", + "\n", + "## Prerequisites\n", + "\n", + "* **Google Cloud Project:** You'll need access to a Google Cloud Project with\n", + " Vertex AI enabled to run the language models.\n", + "* **Installation:** Ensure `google-adk`, `tau-bench`, and\n", + " `google-cloud-aiplatform` are installed.\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "id": "GqUHYdvRJ7pt" + "id": "GqUHYdvRJ7pt", + "language": "python" }, "outputs": [], "source": [ @@ -25,890 +43,1460 @@ "%cd ..\n", "!pip install gepa --quiet\n", "\n", - "!pip install retry --quiet\n", - "\n", - "%cd tau-bench/" + "!pip install retry --quiet" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "id": "SdGCJfEtz8Nq" + "id": "SdGCJfEtz8Nq", + "cellView": "form" }, "outputs": [], "source": [ "#@title Setup and Authentication\n", + "from datetime import datetime\n", + "import json\n", "import logging\n", "import os\n", "\n", "from google.genai import types\n", + "import experiment as experiment_lib\n", + "\n", + "\n", + "# @markdown ### ā˜ļø Configure Vertex AI Access\n", + "# @markdown Enter your Google Cloud Project ID and Location.\n", + "\n", "#@markdown Configure Vertex AI Access\n", "\n", "GCP_PROJECT = '' #@param {type: 'string'}\n", "GCP_LOCATION = 'us-central1' #@param {type: 'string'}\n", "\n", + "# @markdown ---\n", + "# @markdown ### šŸ§  Configure LLM Models\n", + "# @markdown We recommend starting with Flash models for speed and cost-efficiency\n", + "# @markdown during optimization, but larger models like `gemini-1.5-pro` can also\n", + "# @markdown be used, especially for the reflection model.\n", + "AGENT_MODEL_NAME = 'gemini-2.5-flash' # @param {type: 'string'}\n", + "USER_MODEL_NAME = 'gemini-2.5-flash' # @param {type: 'string'}\n", + "REFLECTION_MODEL_NAME = 'gemini-2.5-pro' # @param {type: 'string'}\n", + "\n", + "# @markdown ---\n", + "# @markdown ### āš™ļø Configure Experiment Parameters\n", + "# @markdown These control the dataset size, evaluation runs, and GEPA budget.\n", + "# @markdown For a quick demo, keep these values small. For a real run, you might\n", + "# @markdown increase `MAX_DATASET_SIZE` to 50-100 and `MAX_METRIC_CALLS` to 100+.\n", + "MAX_DATASET_SIZE = 10 # @param {type: 'integer'}\n", + "NUM_EVAL_TRIALS = 4 # @param {type: 'integer'}\n", + "MAX_METRIC_CALLS = 100 # @param {type: 'integer'}\n", + "MAX_CONCURRENCY = 4 # @param {type: 'integer'}\n", + "\n", + "# The ADK uses these environment variables to connect to Vertex AI via the\n", + "# Google GenAI SDK.\n", "os.environ['GOOGLE_GENAI_USE_VERTEXAI'] = 'true'\n", "os.environ['GOOGLE_CLOUD_PROJECT'] = GCP_PROJECT\n", "os.environ['GOOGLE_CLOUD_LOCATION'] = GCP_LOCATION\n", "\n", "# Set a logging verbosity suited for this experiment. See\n", "# https://github.com/google/adk-python/issues/1852 for context\n", - "class _FilterInferenceWarnings(logging.Filter):\n", - " \"\"\"Filters out Vertex inference warning about non-text parts in response.\"\"\"\n", - "\n", - " def filter(self, record):\n", - " if record.levelname != 'WARNING':\n", - " return True\n", - " message_identifier = record.getMessage()\n", - " return not message_identifier.startswith(\n", - " 'Warning: there are non-text parts in the response:'\n", - " )\n", - "\n", - "\n", - "types.logger.addFilter(_FilterInferenceWarnings())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "cellView": "form", - "id": "gAERCrlhgiam" - }, - "outputs": [], - "source": [ - "#@title Discover Tau-bench Tool Definitions\n", - "# This section dynamically discovers and imports tool definitions from the\n", - "# tau-bench library. The code below is searches for the base 'Tool' class and\n", - "# then looks for all subclasses in the 'retail' and 'airline' domains.\n", - "import sys, pathlib, importlib, inspect, ast\n", - "\n", - "REPO_ROOT = pathlib.Path.cwd()\n", - "PKG_ROOT = REPO_ROOT / 'tau_bench'\n", - "DOMAINS = ['retail', 'airline']\n", - "\n", - "if str(REPO_ROOT) not in sys.path:\n", - " sys.path.insert(0, str(REPO_ROOT))\n", - "\n", - "def import_Tool():\n", - " \"\"\"Finds and imports the base `Tool` class from the tau-bench library.\n", - "\n", - " It tries several known locations for the `Tool` class. As a fallback, it\n", - " searches for any file defining 'class Tool(' and attempts to import from\n", - " there.\n", - "\n", - " Returns:\n", - " The `Tool` class object.\n", - "\n", - " Raises:\n", - " ImportError: If the `Tool` class cannot be located.\n", - " \"\"\"\n", - " candidates = [\n", - " 'tau_bench.core.tool:Tool',\n", - " 'tau_bench.core.tools:Tool',\n", - " 'tau_bench.tools:Tool',\n", - " 'tau_bench.envs.core.tool:Tool',\n", - " 'tau_bench.envs.core.tools:Tool',\n", - " 'tau_bench.envs.tool:Tool',\n", - " ]\n", - " for spec in candidates:\n", - " mod, name = spec.split(':')\n", - " try:\n", - " return getattr(importlib.import_module(mod), name)\n", - " except Exception:\n", - " pass\n", - " # Fallback: find any file that defines 'class Tool(' and import it\n", - " for p in PKG_ROOT.rglob('*.py'):\n", - " try:\n", - " txt = p.read_text(encoding='utf-8', errors='ignore')\n", - " except Exception:\n", - " continue\n", - " if 'class Tool(' in txt:\n", - " mod = '.'.join(p.relative_to(REPO_ROOT).with_suffix('').parts)\n", - " try:\n", - " return getattr(importlib.import_module(mod), 'Tool')\n", - " except Exception:\n", - " pass\n", - " raise ImportError('Could not locate the Tool class in tau_bench')\n", - "\n", - "Tool = import_Tool()\n", - "\n", - "def module_name_from_path(pyfile: pathlib.Path) -> str:\n", - " rel = pyfile.relative_to(REPO_ROOT).with_suffix('')\n", - " return '.'.join(rel.parts)\n", - "\n", - "def ast_inherits_tool(pyfile: pathlib.Path) -> list:\n", - " \"\"\"Returns a list of fully-qualified class names.\n", - "\n", - " These include 'Tool' or '*.Tool'.\n", - " \"\"\"\n", - " out = []\n", - " try:\n", - " code = pyfile.read_text(encoding='utf-8', errors='ignore')\n", - " tree = ast.parse(code, filename=str(pyfile))\n", - " except Exception:\n", - " return out\n", - " modname = module_name_from_path(pyfile)\n", - " for node in ast.walk(tree):\n", - " if isinstance(node, ast.ClassDef):\n", - " inherits = False\n", - " for base in node.bases:\n", - " try:\n", - " base_txt = ast.unparse(base)\n", - " except Exception:\n", - " if hasattr(base, \"id\"):\n", - " base_txt = base.id\n", - " elif hasattr(base, \"attr\"):\n", - " base_txt = base.attr\n", - " else:\n", - " base_txt = ''\n", - " if base_txt == 'Tool' or base_txt.endswith('.Tool'):\n", - " inherits = True\n", - " break\n", - " if inherits:\n", - " out.append(f'{modname}.{node.name}')\n", - " return out\n", - "\n", - "def collect_for_domain(domain: str) -> list:\n", - " \"\"\"Collects all tool classes for a given domain (e.g., 'retail').\n", - "\n", - " It first tries to import modules and check for subclasses of `Tool` using\n", - " `issubclass`. If importing fails for any reason, it falls back to a\n", - " heuristic based on AST parsing to identify tool classes.\n", - "\n", - " Args:\n", - " domain: The domain to search for tools in (e.g., 'retail', 'airline').\n", - "\n", - " Returns:\n", - " A list of tool classes found for the given domain.\n", - " \"\"\"\n", - " tools_dir = PKG_ROOT / 'envs' / domain / 'tools'\n", - " if not tools_dir.exists():\n", - " return []\n", - " results = []\n", - " for pyfile in sorted(tools_dir.rglob('*.py')):\n", - " if pyfile.name == '__init__.py':\n", - " continue\n", - " modname = module_name_from_path(pyfile)\n", - " # Try import-based check first\n", - " try:\n", - " mod = importlib.import_module(modname)\n", - " for _, cls in inspect.getmembers(mod, inspect.isclass):\n", - " if getattr(cls, '__module__', None) != mod.__name__:\n", - " continue\n", - " if cls is Tool:\n", - " continue\n", - " try:\n", - " if issubclass(cls, Tool):\n", - " results.append(cls)\n", - " except Exception:\n", - " pass\n", - " except Exception:\n", - " # Import failed ā†’ AST heuristic\n", - " results.extend(ast_inherits_tool(pyfile))\n", - " # Dedup preserve order\n", - " seen, deduped = set(), []\n", - " for q in results:\n", - " if q not in seen:\n", - " seen.add(q)\n", - " deduped.append(q)\n", - " return deduped\n", - "\n", - "tool_classes_by_domain = {d: collect_for_domain(d) for d in DOMAINS}\n", - "tool_definitions_by_domain = {}\n", - "for domain, tool_classes in tool_classes_by_domain.items():\n", - " tool_definitions_by_domain[domain] = []\n", - " for tool_class in tool_classes:\n", - " tool_info = tool_class.get_info()\n", - " if tool_info.get('type') != 'function' or not tool_info.get('function'):\n", - " continue\n", - " tool_definitions_by_domain[domain].append(tool_info['function'])" + "types.logger.addFilter(experiment_lib.FilterInferenceWarnings())" ] }, { "cell_type": "markdown", "metadata": { - "id": "WQteYUPIM-HQ" + "id": "HbKlznZHvskm" }, "source": [ - "# Classes and Functions" + "# Initial Inference: A First Look at Our Agent\n", + "\n", + "Before we start optimizing, let's see how our agent performs with a very basic\n", + "prompt. This will help us understand the task and see what a failure case looks\n", + "like.\n", + "\n", + "**The Task:** We're using the **'retail'** environment from Tau-bench. In this\n", + "environment, our agent acts as a customer support agent for an online store. It\n", + "needs to use a set of tools (like `check_order_status`, `issue_refund`, etc.)\n", + "to help a simulated user resolve their issues, while following specific support\n", + "policies (e.g., only refunding orders less than 30 days old).\n", + "\n", + "**Our Agent:** The agent is built with ADK using a standard tool-calling\n", + "strategy. It receives the conversation history and a list of available tools,\n", + "and it must decide whether to respond to the user or call a tool.\n", + "\n", + "**The Initial Prompt:** We'll start with a simple, one-line instruction. As\n", + "we'll see, this is often not enough for an agent to perform reliably in complex\n", + "scenarios." ] }, + { + "cell_type": "code", + "source": [ + "#@title Define an initial instruction\n", + "\n", + "# @markdown This is our starting \"seed\" prompt. It's very generic and doesn't give the agent much guidance on how to behave or use tools.\n", + "BASE_SYSTEM_INSTRUCTION = 'you are a customer support agent helping customers resolve their issues by using the right tools' # @param {type: 'string'}\n", + "\n", + "print(BASE_SYSTEM_INSTRUCTION)" + ], + "metadata": { + "id": "U8FyG4ep1OLW", + "cellView": "form" + }, + "execution_count": null, + "outputs": [] + }, { "cell_type": "code", "execution_count": null, "metadata": { - "id": "SZVk-I-Kl1uL" + "id": "GNlTPbCXvskn", + "outputId": "02514309-4027-4760-9724-b8cadfbf7c86", + "cellView": "form" }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading user with strategy: llm\n", + "Running tasks [1, 2, 9, 12] (checkpoint path: results/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104135627.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Unclosed client session\n", + "client_session: \n", + "Unclosed connector\n", + "connections: ['deque([(, 95679.854398078)])']\n", + "connector: \n", + "Unclosed client session\n", + "client_session: \n", + "Unclosed connector\n", + "connections: ['deque([(, 95859.665770103)])']\n", + "connector: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.25\n", + "šŸ“ˆ Pass^k\n", + " k=1: 0.25\n", + "\n", + "šŸ“„ Results saved to results/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104135627.json\n", + "\n" + ] + } + ], "source": [ - "#@title Core Logic for Tau-Bench Evaluation\n", - "# This section defines the main components for running tau-bench evaluations and\n", - "# integrating them with GEPA:\n", - "# - A custom runner for tau-bench experiments.\n", - "# - Data structures for trajectories and outputs.\n", - "# - A GEPA adapter that bridges GEPA's optimization process with tau-bench.\n", + "#@title Initial Inference: A First Look at Our Agent\n", "\n", - "\n", - "from concurrent.futures import ThreadPoolExecutor\n", - "from datetime import datetime\n", - "import os\n", - "import json\n", - "import multiprocessing\n", - "import random\n", - "from retry import retry\n", - "import traceback\n", - "from typing import List\n", - "\n", - "import tau_bench_agent as tau_bench_agent_lib\n", - "from tau_bench.envs import get_env\n", - "from tau_bench.run import display_metrics\n", "from tau_bench.types import EnvRunResult, RunConfig\n", - "from litellm import provider_list\n", - "from tau_bench.envs.user import UserStrategy\n", "\n", - "\n", - "def custom_run(\n", - " config: RunConfig,\n", - " print_results: bool = False,\n", - " custom_system_instruction: str = None) -> List[EnvRunResult]:\n", - " \"\"\"Runs a set of tau-bench tasks with a given agent configuration.\n", - "\n", - " This is a customized version of the standard tau-bench run function, adapted\n", - " for this experiment's needs. It handles environment setup, agent creation,\n", - " task execution in parallel, and result aggregation.\n", - "\n", - " Args:\n", - " config: A RunConfig object specifying the environment, models, and other\n", - " parameters for the run.\n", - " print_results: If True, prints the result of each task as it completes.\n", - " custom_system_instruction: An optional system instruction to use for the\n", - " agent, overriding the default.\n", - "\n", - " Returns:\n", - " A list of EnvRunResult objects, one for each completed task.\n", - " \"\"\"\n", - " if config.env not in ['retail', 'airline']:\n", - " raise ValueError('Only retail and airline envs are supported')\n", - " if config.model_provider not in provider_list:\n", - " raise ValueError('Invalid model provider')\n", - " if config.user_model_provider not in provider_list:\n", - " raise ValueError('Invalid user model provider')\n", - " if config.agent_strategy not in [\n", - " 'tool-calling', 'act', 'react', 'few-shot'\n", - " ]:\n", - " raise ValueError('Invalid agent strategy')\n", - " if config.task_split not in ['train', 'test', 'dev']:\n", - " raise ValueError('Invalid task split')\n", - " if config.user_strategy not in [item.value for item in UserStrategy]:\n", - " raise ValueError('Invalid user strategy')\n", - "\n", - " random.seed(config.seed)\n", - " time_str = datetime.now().strftime('%m%d%H%M%S')\n", - " model_name = config.model.split('/')[-1]\n", - " ckpt_filename = (\n", - " f'{config.agent_strategy}-{model_name}-{config.temperature}_range_'\n", - " f'{config.start_index}-{config.end_index}_user-{config.user_model}-'\n", - " f'{config.user_strategy}_{time_str}.json'\n", - " )\n", - " ckpt_path = os.path.join(config.log_dir, ckpt_filename)\n", - " if not os.path.exists(config.log_dir):\n", - " os.makedirs(config.log_dir)\n", - "\n", - " print(f'Loading user with strategy: {config.user_strategy}')\n", - " env = get_env(\n", - " config.env,\n", - " user_strategy=config.user_strategy,\n", - " user_model=config.user_model,\n", - " user_provider=config.user_model_provider,\n", - " task_split=config.task_split,\n", - " )\n", - " if custom_system_instruction:\n", - " env.wiki = custom_system_instruction\n", - " agent = tau_bench_agent_lib.adk_agent_factory(\n", - " tools_info=env.tools_info,\n", - " wiki=env.wiki,\n", - " config=config,\n", - " )\n", - " if config.end_index == -1:\n", - " end_index = len(env.tasks)\n", - " else:\n", - " end_index = min(config.end_index, len(env.tasks))\n", - " results: List[EnvRunResult] = []\n", - " lock = multiprocessing.Lock()\n", - " if config.task_ids and len(config.task_ids) > 0:\n", - " print(f'Running tasks {config.task_ids} (checkpoint path: {ckpt_path})')\n", - " else:\n", - " print(\n", - " f'Running tasks {config.start_index} to {end_index} '\n", - " f'(checkpoint path: {ckpt_path})'\n", - " )\n", - " for i in range(config.num_trials):\n", - " if config.task_ids and len(config.task_ids) > 0:\n", - " idxs = config.task_ids\n", - " else:\n", - " idxs = list(range(config.start_index, end_index))\n", - " if config.shuffle:\n", - " random.shuffle(idxs)\n", - "\n", - " @retry(tries=3, delay=10, backoff=2)\n", - " def _run_with_retry(idx: int) -> EnvRunResult:\n", - " isolated_env = get_env(\n", - " config.env,\n", - " user_strategy=config.user_strategy,\n", - " user_model=config.user_model,\n", - " task_split=config.task_split,\n", - " user_provider=config.user_model_provider,\n", - " task_index=idx,\n", - " )\n", - " if print_results:\n", - " print(f'Running task {idx}')\n", - " res = agent.solve(\n", - " env=isolated_env,\n", - " task_index=idx,\n", - " )\n", - " return EnvRunResult(\n", - " task_id=idx,\n", - " reward=res.reward,\n", - " info=res.info,\n", - " traj=res.messages,\n", - " trial=i,\n", - " )\n", - "\n", - " def _run(idx: int) -> EnvRunResult:\n", - " try:\n", - " result = _run_with_retry(idx)\n", - " except Exception as e:\n", - " logging.warning('Inference error: %s', str(e))\n", - " result = EnvRunResult(\n", - " task_id=idx,\n", - " reward=0.0,\n", - " info={'error': str(e), 'traceback': traceback.format_exc()},\n", - " traj=[],\n", - " trial=i,\n", - " )\n", - "\n", - " if print_results:\n", - " print(\n", - " 'āœ…' if result.reward == 1 else 'āŒ',\n", - " f'task_id={idx}',\n", - " # result.info,\n", - " )\n", - " print('-----')\n", - " with lock:\n", - " data = []\n", - " if os.path.exists(ckpt_path):\n", - " with open(ckpt_path, 'r') as f:\n", - " data = json.load(f)\n", - " with open(ckpt_path, 'w') as f:\n", - " json.dump(data + [result.model_dump()], f, indent=2)\n", - " return result\n", - "\n", - " with ThreadPoolExecutor(max_workers=config.max_concurrency) as executor:\n", - " res = list(executor.map(_run, idxs))\n", - " results.extend(res)\n", - "\n", - " display_metrics(results)\n", - "\n", - " with open(ckpt_path, 'w') as f:\n", - " json.dump([result.model_dump() for result in results], f, indent=2)\n", - " print(f'\\nšŸ“„ Results saved to {ckpt_path}\\n')\n", - " return results" + "# We will run our ADK agent on two tasks from the Tau-bench 'dev' set.\n", + "# The `run_tau_bench_rollouts` function handles the interaction between the\n", + "# agent and the simulated user environment.\n", + "print('Running initial inference for tasks 1 and 2...')\n", + "inference_results = experiment_lib.run_tau_bench_rollouts(\n", + " config=RunConfig(\n", + " env='retail',\n", + " model=AGENT_MODEL_NAME,\n", + " model_provider='vertex_ai',\n", + " user_model=USER_MODEL_NAME,\n", + " user_model_provider='vertex_ai',\n", + " agent_strategy='tool-calling',\n", + " user_strategy='llm', # The user is simulated by an LLM\n", + " max_concurrency=MAX_CONCURRENCY,\n", + " task_ids=[\n", + " 1,\n", + " 2,\n", + " ], # We'll just run two specific tasks for this initial look\n", + " task_split='dev',\n", + " ),\n", + " system_instruction=BASE_SYSTEM_INSTRUCTION,\n", + ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "id": "hZkwAFkINKG_" + "id": "B3ZEiRgZvskn", + "outputId": "804df2c6-964e-4982-e298-64d14ba2d84e", + "cellView": "form" }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--- Trajectory Example ---\n", + "**SYSTEM**: you are a customer support agent helping customers resolve their issues by using the right tools\n", + "\n", + "**USER**: Hello. I need to make an exchange regarding a recent order I received.\n", + "\n", + "**MODEL**: I can help you with that. What is the order ID?\n", + "\n", + "**USER**: The order ID is #W7273336.\n", + "\n", + "**MODEL**: Okay, you want to exchange items from order #W7273336.\n", + "\n", + "Could you please tell me:\n", + "1. What are the item IDs of the products you wish to exchange?\n", + "2. What are the item IDs of the new products you would like to receive?\n", + "3. What payment method would you like to use to cover any price differences or receive refunds (e.g., gift card, credit card)?\n", + "\n", + "**USER**: I don't have the specific item IDs at hand, but I can describe the products. I received a black laser gaming mouse and a 4-foot metal bookshelf. I need to exchange both of these.\n", + "\n", + "**MODEL**: I understand. Since you don't have the item IDs, I'll need to look up the order details to identify them.\n", + "\n", + "\n", + "**MODEL**: šŸ“ž Tool Call: `get_order_details(order_id='#W7273336')`\n", + "\n", + "**USER**: ā†Ŗļø Tool Response from `get_order_details`: `{\"order_id\": \"#W7273336\", \"user_id\": \"omar_lopez_3107\", \"address\": {\"address1\": \"959 Broadway\", \"address2\": \"Suite 363\", \"city\": \"Los Angeles\", \"country\": \"USA\", \"state\": \"CA\", \"zip\": \"90339\"}, \"items\": [{\"name\": \"Espresso Machine\", \"product_id\": \"4354588079\", \"item_id\": \"6200867091\", \"price\": 2955.17, \"options\": {\"pressure\": \"19 bar\", \"capacity\": \"1L\", \"type\": \"capsule\"}}, {\"name\": \"Bookshelf\", \"product_id\": \"8600330539\", \"item_id\": \"8018699955\", \"price\": 467.86, \"options\": {\"material\": \"metal\", \"color\": \"brown\", \"height\": \"4 ft\"}}, {\"name\": \"Gaming Mouse\", \"product_id\": \"5713490933\", \"item_id\": \"8214883393\", \"price\": 150.58, \"options\": {\"color\": \"black\", \"sensor type\": \"laser\", \"connectivity\": \"wireless\"}}, {\"name\": \"Pet Bed\", \"product_id\": \"2747247837\", \"item_id\": \"6857426243\", \"price\": 196.53, \"options\": {\"size\": \"medium\", \"material\": \"fleece\", \"color\": \"grey\"}}, {\"name\": \"Air Purifier\", \"product_id\": \"3821016478\", \"item_id\": \"9375701158\", \"price\": 489.5, \"options\": {\"room size\": \"medium\", \"filter type\": \"carbon\", \"features\": \"quiet operation\"}}], \"fulfillments\": [{\"tracking_id\": [\"555296392986\"], \"item_ids\": [\"6200867091\", \"8018699955\", \"8214883393\", \"6857426243\", \"9375701158\"]}], \"status\": \"delivered\", \"payment_history\": [{\"transaction_type\": \"payment\", \"amount\": 4259.64, \"payment_method_id\": \"paypal_1530316\"}]}`\n", + "\n", + "**MODEL**: Thank you for waiting. I've looked up the details for order #W7273336.\n", + "\n", + "The item ID for the black laser gaming mouse is **8214883393**.\n", + "The item ID for the 4-foot metal bookshelf is **8018699955**.\n", + "\n", + "Now, please tell me:\n", + "1. What are the item IDs of the new gaming mouse and bookshelf you would like to receive?\n", + "2. What payment method would you like to use to cover any price differences or receive refunds (e.g., gift card, credit card)? The original payment method for this order was PayPal, with `payment_method_id`: \"paypal_1530316\")?\n", + "\n", + "**USER**: Thank you for providing those. I need to exchange the black laser gaming mouse (item ID 8214883393). I'd like to get a white optical mouse instead. The black doesn't suit my setup, and I prefer optical over laser. I don't care if it's wired or wireless, just whichever is cheaper.\n", + "\n", + "As for the 4-foot metal bookshelf (item ID 8018699955), it's just too short. I need a taller one. I want to exchange it for a 5-feet tall glass bookshelf.\n", + "\n", + "Regarding the glass bookshelf, I'm not sure about the color yet. What are the available color options for the 5-feet glass bookshelves? As for payment, we can discuss that once we figure out the replacement items.\n", + "\n", + "**MODEL**: Okay, I understand. You want to exchange the black laser gaming mouse (item ID 8214883393) for a white optical mouse, and the 4-foot metal bookshelf (item ID 8018699955) for a 5-foot tall glass bookshelf.\n", + "\n", + "Let's find the options for these. First, I'll look up the product types to find suitable replacements.\n", + "\n", + "\n", + "**MODEL**: šŸ“ž Tool Call: `list_all_product_types()`\n", + "\n", + "**USER**: ā†Ŗļø Tool Response from `list_all_product_types`: `{\"Action Camera\": \"3377618313\", \"Air Purifier\": \"3821016478\", \"Backpack\": \"2524789262\", \"Bicycle\": \"9783735446\", \"Bluetooth Speaker\": \"4768869376\", \"Bookshelf\": \"8600330539\", \"Coffee Maker\": \"7996920482\", \"Cycling Helmet\": \"7765186836\", \"Desk Lamp\": \"6817146515\", \"Digital Camera\": \"8940227892\", \"Dumbbell Set\": \"7233192239\", \"E-Reader\": \"3801771308\", \"Electric Kettle\": \"1075968781\", \"Electric Toothbrush\": \"7352963235\", \"Espresso Machine\": \"4354588079\", \"Fleece Jacket\": \"8560156827\", \"Gaming Mouse\": \"5713490933\", \"Garden Hose\": \"6679515468\", \"Grill\": \"6819683148\", \"Headphones\": \"6992792935\", \"Hiking Boots\": \"7363354090\", \"Indoor Security Camera\": \"2985987096\", \"Jigsaw Puzzle\": \"1808611083\", \"LED Light Bulb\": \"2696197613\", \"Laptop\": \"4760268021\", \"Luggage Set\": \"5426915165\", \"Makeup Kit\": \"5149340237\", \"Mechanical Keyboard\": \"1656367028\", \"Notebook\": \"2892623495\", \"Office Chair\": \"4794339885\", \"Patio Umbrella\": \"9743693396\", \"Perfume\": \"6858788497\", \"Pet Bed\": \"2747247837\", \"Portable Charger\": \"6942297802\", \"Running Shoes\": \"6938111410\", \"Skateboard\": \"1968349452\", \"Smart Thermostat\": \"4896585277\", \"Smart Watch\": \"6945232052\", \"Smartphone\": \"1801728040\", \"Sneakers\": \"7471004230\", \"Sunglasses\": \"7314138884\", \"T-Shirt\": \"9523456873\", \"Tablet\": \"8024098596\", \"Tea Kettle\": \"9832717871\", \"Vacuum Cleaner\": \"1762337868\", \"Wall Clock\": \"2344688344\", \"Water Bottle\": \"8310926033\", \"Wireless Earbuds\": \"9924732112\", \"Wristwatch\": \"6066914160\", \"Yoga Mat\": \"4635925001\"}`\n", + "\n", + "**MODEL**: Okay, I have the product IDs. Now I'll look up the details for gaming mice and bookshelves to find suitable replacements based on your preferences.\n", + "\n", + "First, let's check for white optical gaming mice.\n", + "\n", + "**MODEL**: šŸ“ž Tool Call: `get_product_details(product_id='5713490933')`\n", + "\n", + "**USER**: ā†Ŗļø Tool Response from `get_product_details`: `{\"name\": \"Gaming Mouse\", \"product_id\": \"5713490933\", \"variants\": {\"8896479688\": {\"item_id\": \"8896479688\", \"options\": {\"color\": \"white\", \"sensor type\": \"optical\", \"connectivity\": \"wireless\"}, \"available\": true, \"price\": 143.15}, \"8214883393\": {\"item_id\": \"8214883393\", \"options\": {\"color\": \"black\", \"sensor type\": \"laser\", \"connectivity\": \"wireless\"}, \"available\": true, \"price\": 150.58}, \"7420906769\": {\"item_id\": \"7420906769\", \"options\": {\"color\": \"white\", \"sensor type\": \"laser\", \"connectivity\": \"wireless\"}, \"available\": false, \"price\": 138.47}, \"2193628750\": {\"item_id\": \"2193628750\", \"options\": {\"color\": \"black\", \"sensor type\": \"laser\", \"connectivity\": \"wired\"}, \"available\": true, \"price\": 162.15}, \"2880340443\": {\"item_id\": \"2880340443\", \"options\": {\"color\": \"white\", \"sensor type\": \"optical\", \"connectivity\": \"wired\"}, \"available\": true, \"price\": 137.22}, \"5019835484\": {\"item_id\": \"5019835484\", \"options\": {\"color\": \"RGB\", \"sensor type\": \"laser\", \"connectivity\": \"wired\"}, \"available\": false, \"price\": 138.73}, \"3330317167\": {\"item_id\": \"3330317167\", \"options\": {\"color\": \"black\", \"sensor type\": \"optical\", \"connectivity\": \"wired\"}, \"available\": true, \"price\": 137.32}, \"5796612084\": {\"item_id\": \"5796612084\", \"options\": {\"color\": \"RGB\", \"sensor type\": \"optical\", \"connectivity\": \"wired\"}, \"available\": false, \"price\": 158.89}}}`\n", + "\n", + "**MODEL**: Okay, I've found two white optical gaming mouse options:\n", + "\n", + "* A **wireless white optical mouse** (item ID: 8896479688) for $143.15.\n", + "* A **wired white optical mouse** (item ID: 2880340443) for $137.22.\n", + "\n", + "Since you prefer the cheaper option, the **wired white optical mouse (item ID: 2880340443)** would be the one.\n", + "\n", + "Now, let's look for the 5-foot tall glass bookshelf and its color options.\n", + "\n", + "**MODEL**: šŸ“ž Tool Call: `get_product_details(product_id='8600330539')`\n", + "\n", + "**USER**: ā†Ŗļø Tool Response from `get_product_details`: `{\"name\": \"Bookshelf\", \"product_id\": \"8600330539\", \"variants\": {\"8479046075\": {\"item_id\": \"8479046075\", \"options\": {\"material\": \"wood\", \"color\": \"white\", \"height\": \"5 ft\"}, \"available\": true, \"price\": 451.01}, \"8895454203\": {\"item_id\": \"8895454203\", \"options\": {\"material\": \"glass\", \"color\": \"white\", \"height\": \"5 ft\"}, \"available\": true, \"price\": 504.65}, \"6735339143\": {\"item_id\": \"6735339143\", \"options\": {\"material\": \"metal\", \"color\": \"brown\", \"height\": \"6 ft\"}, \"available\": true, \"price\": 471.77}, \"7373893106\": {\"item_id\": \"7373893106\", \"options\": {\"material\": \"glass\", \"color\": \"white\", \"height\": \"4 ft\"}, \"available\": false, \"price\": 531.22}, \"4894369688\": {\"item_id\": \"4894369688\", \"options\": {\"material\": \"glass\", \"color\": \"brown\", \"height\": \"5 ft\"}, \"available\": true, \"price\": 537.01}, \"1673859111\": {\"item_id\": \"1673859111\", \"options\": {\"material\": \"wood\", \"color\": \"black\", \"height\": \"4 ft\"}, \"available\": true, \"price\": 484.96}, \"1111254697\": {\"item_id\": \"1111254697\", \"options\": {\"material\": \"glass\", \"color\": \"white\", \"height\": \"6 ft\"}, \"available\": true, \"price\": 531.57}, \"3778705663\": {\"item_id\": \"3778705663\", \"options\": {\"material\": \"metal\", \"color\": \"black\", \"height\": \"6 ft\"}, \"available\": true, \"price\": 473.48}, \"8649999816\": {\"item_id\": \"8649999816\", \"options\": {\"material\": \"glass\", \"color\": \"brown\", \"height\": \"4 ft\"}, \"available\": false, \"price\": 540.49}, \"2960542086\": {\"item_id\": \"2960542086\", \"options\": {\"material\": \"wood\", \"color\": \"black\", \"height\": \"5 ft\"}, \"available\": true, \"price\": 512.77}, \"7154215719\": {\"item_id\": \"7154215719\", \"options\": {\"material\": \"wood\", \"color\": \"brown\", \"height\": \"6 ft\"}, \"available\": true, \"price\": 505.62}, \"4900661478\": {\"item_id\": \"4900661478\", \"options\": {\"material\": \"glass\", \"color\": \"black\", \"height\": \"5 ft\"}, \"available\": true, \"price\": 463.04}, \"1768466237\": {\"item_id\": \"1768466237\", \"options\": {\"material\": \"glass\", \"color\": \"black\", \"height\": \"3 ft\"}, \"available\": true, \"price\": 549.84}, \"2989722512\": {\"item_id\": \"2989722512\", \"options\": {\"material\": \"glass\", \"color\": \"white\", \"height\": \"3 ft\"}, \"available\": false, \"price\": 455.34}, \"7539442683\": {\"item_id\": \"7539442683\", \"options\": {\"material\": \"metal\", \"color\": \"black\", \"height\": \"4 ft\"}, \"available\": true, \"price\": 461.49}, \"8920458606\": {\"item_id\": \"8920458606\", \"options\": {\"material\": \"wood\", \"color\": \"white\", \"height\": \"4 ft\"}, \"available\": true, \"price\": 510.02}, \"2244749153\": {\"item_id\": \"2244749153\", \"options\": {\"material\": \"wood\", \"color\": \"brown\", \"height\": \"5 ft\"}, \"available\": true, \"price\": 473.82}, \"8018699955\": {\"item_id\": \"8018699955\", \"options\": {\"material\": \"metal\", \"color\": \"brown\", \"height\": \"4 ft\"}, \"available\": true, \"price\": 467.86}}}`\n", + "\n", + "**MODEL**: Great! I've found the options for the 5-foot tall glass bookshelf.\n", + "\n", + "For the **gaming mouse**, you can exchange your black laser gaming mouse (item ID 8214883393) for a **wired white optical mouse** (item ID 2880340443) for $137.22.\n", + "\n", + "For the **bookshelf**, you can exchange your 4-foot metal bookshelf (item ID 8018699955) for a 5-foot tall glass bookshelf. The available colors are:\n", + "* **White** (item ID: 8895454203) for $504.65\n", + "* **Brown** (item ID: 4894369688) for $537.01\n", + "* **Black** (item ID: 4900661478) for $463.04\n", + "\n", + "Which color would you prefer for the 5-foot tall glass bookshelf? Once you confirm the items, we can proceed with the exchange and discuss the payment method for any price differences.\n", + "\n", + "**USER**: Okay, that mouse sounds good. I'll go with the wired white optical mouse (item ID 2880340443).\n", + "\n", + "As for the bookshelf, hmm, white, brown, black... I'm really torn. Given my setup, I think the brown would actually look quite elegant. Yes, let's go with the brown 5-feet glass bookshelf (item ID 4894369688).\n", + "\n", + "**MODEL**: Excellent choices!\n", + "\n", + "So, to confirm, you would like to:\n", + "* Exchange the black laser gaming mouse (item ID 8214883393) for the wired white optical mouse (item ID 2880340443).\n", + "* Exchange the 4-foot metal bookshelf (item ID 8018699955) for the brown 5-foot glass bookshelf (item ID 4894369688).\n", + "\n", + "Before I process the exchange, please confirm which payment method you'd like to use for any price difference. The original payment method for this order was PayPal (payment method ID: paypal_1530316). Would you like to use this, or do you have another payment method in mind?\n", + "\n", + "**USER**: Yes, that's correct for both exchanges. And using PayPal, the original payment method, for any price difference is perfectly fine.\n", + "\n", + "**MODEL**: Great! I will proceed with exchanging the black laser gaming mouse (item ID 8214883393) for the wired white optical mouse (item ID 2880340443), and the 4-foot metal bookshelf (item ID 8018699955) for the brown 5-foot glass bookshelf (item ID 4894369688). Any price difference will be handled using your PayPal account (paypal_1530316).\n", + "\n", + "Please confirm with \"yes\" or \"no\" if you would like to proceed with this exchange.\n", + "\n", + "**USER**: Yes.\n", + "###STOP###\n", + "\n" + ] + } + ], "source": [ - "#@title GEPA Adapter for Tau-bench\n", - "# The TauBenchAdapter is the core component that allows GEPA to optimize prompts\n", - "# for tau-bench. It implements the GEPAAdapter interface, defining how to:\n", - "# 1. `evaluate`: Run a candidate prompt on a batch of tau-bench tasks and\n", - "# return scores.\n", - "# 2. `make_reflective_dataset`: Convert evaluation results into a dataset that\n", - "# can be used for reflection and prompt improvement.\n", + "#@title Let's visualize one of the sampled trajectory\n", "\n", - "from typing import List, Any, TypedDict\n", - "import json\n", - "import gepa\n", - "from gepa.core.adapter import EvaluationBatch, GEPAAdapter\n", - "\n", - "\n", - "class TauBenchDataInst(TypedDict):\n", - " env: str\n", - " task_id: int\n", - "\n", - "\n", - "class TauBenchTrajectory(TypedDict):\n", - " result_traj: list[dict]\n", - "\n", - "\n", - "class TauBenchRolloutOutput(TypedDict):\n", - " env: str\n", - " task_id: int\n", - " reward: float\n", - " task_info: dict\n", - "\n", - "\n", - "def refine_tau_bench_trajectory(traj: list[dict[str, Any]]) -> None:\n", - " \"\"\"Removes unnecessary info from the trajectory, in place.\"\"\"\n", - " for content in traj:\n", - " for part in content[\"parts\"]:\n", - " # Drop all fields that are not populated.\n", - " to_drop = []\n", - " for key in part:\n", - " if not part[key]:\n", - " to_drop.append(key)\n", - " for key in to_drop:\n", - " del part[key]\n", - "\n", - " # For function calls / responses only keep function names, input arguments\n", - " # and outputs.\n", - " if fc := part.get(\"function_call\"):\n", - " part[\"function_call\"] = dict(name=fc[\"name\"], args=fc[\"args\"])\n", - " if fr := part.get(\"function_response\"):\n", - " part[\"function_response\"] = dict(name=fr[\"name\"], args=fr[\"response\"])\n", - "\n", - "\n", - "class TauBenchAdapter(GEPAAdapter[\n", - " TauBenchDataInst,\n", - " TauBenchTrajectory,\n", - " TauBenchRolloutOutput,\n", - "]):\n", - " \"\"\"A GEPA adapter for evaluating agent performance on tau-bench benchmark.\"\"\"\n", - "\n", - " def __init__(\n", - " self,\n", - " agent_model='gemini-2.5-flash',\n", - " agent_model_provider='vertex_ai',\n", - " user_model='gemini-2.5-pro',\n", - " user_model_provider='vertex_ai',\n", - " agent_strategy='tool-calling',\n", - " user_strategy='llm',\n", - " system_instruction_name='system_instruction',\n", - " tools_description: list[dict[str, Any]] | None = None,\n", - " max_concurrency=4,\n", - " ):\n", - " \"\"\"Initializes the TauBenchAdapter.\n", - "\n", - " Args:\n", - " agent_model: The model to use for the agent.\n", - " agent_model_provider: The provider for the agent model.\n", - " user_model: The model to use for simulating the user.\n", - " user_model_provider: The provider for the user model.\n", - " agent_strategy: The agent strategy to use (e.g., 'tool-calling').\n", - " user_strategy: The user simulation strategy (e.g., 'llm').\n", - " system_instruction_name: The key in the candidate dictionary that holds\n", - " the system instruction.\n", - " tools_description: Describes each of the availble tools. This is used as context\n", - " for the prompt proposer.\n", - " max_concurrency: The maximum number of tasks to run in parallel.\n", - " \"\"\"\n", - " self._agent_model = agent_model\n", - " self._agent_model_provider = agent_model_provider\n", - " self._user_model = user_model\n", - " self._user_model_provider = user_model_provider\n", - " self._agent_strategy = agent_strategy\n", - " self._user_strategy = user_strategy\n", - " self._max_concurrency = max_concurrency\n", - " self._system_instruction_name = system_instruction_name\n", - " self._tools_description = tools_description\n", - "\n", - " def evaluate(\n", - " self,\n", - " batch: list[TauBenchDataInst],\n", - " candidate: dict[str, str],\n", - " capture_traces: bool = False,\n", - " ) -> EvaluationBatch[TauBenchTrajectory, TauBenchRolloutOutput]:\n", - " \"\"\"Evaluates a candidate prompt on a batch of tau-bench tasks.\n", - "\n", - " This method is called by GEPA during the optimization loop. It takes a\n", - " candidate prompt, runs it against the specified tasks from tau-bench, and\n", - " returns the results.\n", - "\n", - " Args:\n", - " batch: A list of task instances to evaluate on. Each instance specifies\n", - " the environment and task ID.\n", - " candidate: A dictionary containing the components to be evaluated,\n", - " including the system instruction.\n", - " capture_traces: (Not used in this adapter) Whether to capture detailed\n", - " traces.\n", - "\n", - " Returns:\n", - " An EvaluationBatch object containing scores, outputs, and trajectories for\n", - " each task in the batch.\n", - " \"\"\"\n", - " del capture_traces # Not used.\n", - " env = batch[0]['env']\n", - " task_ids = [inst['task_id'] for inst in batch]\n", - " tau_bench_run_config = RunConfig(\n", - " env=env,\n", - " model=self._agent_model,\n", - " model_provider=self._agent_model_provider,\n", - " user_model=self._user_model,\n", - " user_model_provider=self._user_model_provider,\n", - " agent_strategy=self._agent_strategy,\n", - " user_strategy=self._user_strategy,\n", - " max_concurrency=self._max_concurrency,\n", - " task_ids=task_ids\n", - " )\n", - " tau_bench_results = custom_run(\n", - " tau_bench_run_config,\n", - " custom_system_instruction=candidate.get(self._system_instruction_name),\n", - " )\n", - "\n", - " outputs = []\n", - " trajectories = []\n", - " scores = []\n", - " for res in tau_bench_results:\n", - " outputs.append(\n", - " TauBenchRolloutOutput(\n", - " env=env,\n", - " task_id=res.task_id,\n", - " reward=res.reward,\n", - " task_info=res.info))\n", - " result_traj = res.traj\n", - " refine_tau_bench_trajectory(result_traj)\n", - " trajectories.append(TauBenchTrajectory(result_traj=result_traj))\n", - " scores.append(res.reward)\n", - "\n", - " return EvaluationBatch(\n", - " scores=scores, outputs=outputs, trajectories=trajectories)\n", - "\n", - " def make_reflective_dataset(\n", - " self,\n", - " candidate: dict[str, str],\n", - " eval_batch: EvaluationBatch[TauBenchTrajectory, TauBenchRolloutOutput],\n", - " components_to_update: list[str]\n", - " ) -> dict[str, list[dict[str, Any]]]:\n", - " \"\"\"Creates a dataset for reflection based on evaluation results.\n", - "\n", - " This method transforms the trajectories and scores from an evaluation run\n", - " into a structured format that a reflection model can use to generate\n", - " suggestions for improving the prompt.\n", - "\n", - " Args:\n", - " candidate: The candidate that was evaluated.\n", - " eval_batch: The results of the evaluation.\n", - " components_to_update: A list of component names that the reflection\n", - " should focus on improving.\n", - "\n", - " Returns:\n", - " A dictionary where keys are component names and values are lists of\n", - " data instances for reflection.\n", - " \"\"\"\n", - " system_instruction = candidate[self._system_instruction_name]\n", - "\n", - " tool_definitions = json.dumps(\n", - " self._tools_description,\n", - " indent=2,\n", - " default=str,\n", - " )\n", - "\n", - " inputs = '\\n\\n'.join([\n", - " f'# System Instruction\\n{system_instruction}',\n", - " f'# Tool Definitions\\n{tool_definitions}',\n", - " ])\n", - " ret_d: dict[str, list[dict[str, Any]]] = {}\n", - " for comp in components_to_update:\n", - " items: list[dict[str, Any]] = []\n", - " trace_instances = list(zip(\n", - " eval_batch.trajectories,\n", - " eval_batch.scores,\n", - " eval_batch.outputs,\n", - " strict=True,\n", - " ))\n", - " for trace_instance in trace_instances:\n", - " traj, score, _ = trace_instance\n", - " if score > 0:\n", - " feedback = f'The agent successfully resolved all customer issues'\n", - " else:\n", - " feedback = (\n", - " f'The agent failed to resolve all customer issues correctly'\n", + "def display_trajectory(trajectory):\n", + " \"\"\"Formats and prints a trajectory for display in Colab.\"\"\"\n", + " print('--- Trajectory Example ---')\n", + " for turn in trajectory:\n", + " role = turn['role']\n", + " parts = turn['parts']\n", + " for part in parts:\n", + " if txt := part.get('text'):\n", + " print(f'**{role.upper()}**: {txt}')\n", + " elif fc := part.get('function_call'):\n", + " args_str = ', '.join(f'{k}={v!r}' for k, v in fc['args'].items())\n", + " print(f'**{role.upper()}**: šŸ“ž Tool Call: `{fc[\"name\"]}({args_str})`')\n", + " elif fr := part.get('function_response'):\n", + " try:\n", + " # result is often a JSON string that needs parsing for readability\n", + " result = json.dumps(json.loads(fr['result']), indent=2)\n", + " print(\n", + " f'**{role.upper()}**: ā†Ŗļø Tool Response from'\n", + " f' `{fr[\"name\"]}`:\\n```json\\n{result}\\n```'\n", " )\n", - " d = {\n", - " 'Inputs': inputs,\n", - " 'Generated Outputs': json.dumps(traj, indent=2, default=str),\n", - " 'Feedback': feedback\n", - " }\n", - " items.append(d)\n", - " if items:\n", - " ret_d[comp] = items\n", - " assert ret_d, (\n", - " 'empty reflective dataset for components '\n", - " f'{[comp for comp in components_to_update]}'\n", - " )\n", - " return ret_d\n" + " except Exception:\n", + " print(\n", + " f'**{role.upper()}**: ā†Ŗļø Tool Response from'\n", + " f' `{fr[\"name\"]}`: `{fr[\"response\"][\"result\"]}`'\n", + " )\n", + " print() # new line after each turn\n", + "\n", + "\n", + "# Let's inspect the \"trajectory\" of the first run. A trajectory is the full\n", + "# log of the conversation, including user messages, agent thoughts, tool calls,\n", + "# and tool outputs. Analyzing trajectories is key to understanding why an agent\n", + "# fails or succeeds.\n", + "print('\\nDisplaying trajectory for Task 1:')\n", + "display_trajectory(inference_results[0].traj)" ] }, { "cell_type": "markdown", - "metadata": { - "id": "_yQOx6WoNLGn" - }, "source": [ - "# Run Experiment" + "# Evaluate the Initial Prompt: Getting a Baseline\n", + "\n", + "Running a couple of examples gives us a qualitative feel, but to systematically\n", + "improve our prompt, we need quantitative metrics. Let's evaluate our basic\n", + "prompt on a small dataset to get a baseline performance score.\n", + "\n", + "The primary metric in Tau-bench is **reward**, which is 1 if the agent\n", + "successfully completes the task according to the environment's goals (e.g.,\n", + "user issue resolved, correct tool calls made) and 0 otherwise. Our goal is to\n", + "maximize the average reward." + ], + "metadata": { + "id": "cA70NpvcxanK" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mVFTLlSq5Lqn", + "outputId": "d22b2c37-ea3d-47fa-b7c0-d1a69e7ae585" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading user with strategy: llm\n", + "Running tasks [9, 8, 4, 2, 5, 3, 1, 0, 7, 6] (checkpoint path: temp_results/20251104150054446083/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104150054.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.525\n", + "šŸ“ˆ Pass^k\n", + " k=1: 0.525\n", + " k=2: 0.31666666666666665\n", + " k=3: 0.175\n", + " k=4: 0.1\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104150054446083/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104150054.json\n", + "\n", + "average reward (total=40): 0.525\n" + ] + } + ], + "source": [ + "# For this demo, we'll use a small dataset. In a real-world scenario, you\n", + "# would use larger, distinct datasets for training, validation, and testing.\n", + "demo_dataset = experiment_lib.Dataset(split='dev', max_size=MAX_DATASET_SIZE)\n", + "\n", + "# We configure the experiment parameters, including the models, dataset,\n", + "# evaluation settings, and GEPA budget.\n", + "demo_config = experiment_lib.ExperimentConfig(\n", + " tau_bench_env='retail',\n", + " agent_model=AGENT_MODEL_NAME,\n", + " agent_model_provider='vertex_ai',\n", + " user_model=USER_MODEL_NAME,\n", + " user_model_provider='vertex_ai',\n", + " max_concurrency=MAX_CONCURRENCY,\n", + " num_eval_trials=NUM_EVAL_TRIALS, # We run each task multiple times for consistency\n", + " rnd_seed=42,\n", + " max_metric_calls=MAX_METRIC_CALLS, # GEPA budget: max prompt evaluations\n", + " reflection_model=REFLECTION_MODEL_NAME, # Model for GEPA's reflection step\n", + " reflection_minibatch_size=8,\n", + " use_rater=False, # Optional: LLM rater for nuanced feedback\n", + " # For this demo, we use the same small dataset for all splits.\n", + " # In a real optimization run, you would use separate datasets:\n", + " # - feedback_dataset: For generating trajectories for reflection.\n", + " # - pareto_dataset: For evaluating candidate prompts.\n", + " # - eval_dataset: A final, held-out set to test the optimized prompt.\n", + " feedback_dataset=demo_dataset,\n", + " pareto_dataset=demo_dataset,\n", + " eval_dataset=demo_dataset,\n", + ")\n", + "\n", + "# We'll save the results of our runs in a temporary directory.\n", + "eval_output_dir = os.path.join(\n", + " 'eval_results', datetime.now().strftime('%Y%m%d%H%M%S%f')\n", + ")\n", + "os.makedirs(eval_output_dir)\n", + "logging.info('Writing to output_dir=%s', eval_output_dir)\n", + "\n", + "\n", + "# The `run_eval` function runs the agent with the given prompt on the evaluation\n", + "# dataset and prints the average reward.\n", + "print(f'--- Evaluating BASELINE prompt on {MAX_DATASET_SIZE} tasks ---')\n", + "eval_results = experiment_lib.run_eval(\n", + " output_dir=eval_output_dir,\n", + " config=demo_config,\n", + " instructions=BASE_SYSTEM_INSTRUCTION,\n", + ")\n", + "\n", + "# This will show the detailed results of the evaluation run.\n", + "# The most important number is the final \"average reward\".\n", + "print('\\nBaseline evaluation results:')\n", + "print(eval_results)" + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Run Prompt Optimization with GEPA\n", + "\n", + "Now we'll use **GEPA** to automatically improve our prompt.\n", + "\n", + "## What is GEPA?\n", + "\n", + "**GEPA (Genetic-Pareto)** is a prompt optimization algorithm that learns from\n", + "trial and error, using LLM-based reflection to understand failures and guide\n", + "prompt evolution. Here's a simplified view of how it works:\n", + "\n", + "1. **Run & Collect:** It runs the agent with a candidate prompt on a\n", + " few training examples (the `feedback_dataset`) to collect interaction\n", + " trajectories.\n", + "2. **Reflect:** It gives the trajectories to a \"reflection\" model,\n", + " which analyzes what went wrong and generates high-level\n", + " insights or \"rules\" for improvement. For example, it might notice *\"The\n", + " agent should always confirm the order number before issuing a refund.\"*\n", + "3. **Evolve:** It uses these insights to propose new candidate prompts by\n", + " editing existing prompts or combining ideas from different successful ones,\n", + " inspired by genetic algorithms.\n", + "4. **Evaluate & Select:** It evaluates these new prompts on a validation set\n", + " (the `pareto_dataset`) and keeps only the best-performing, diverse set of\n", + " prompts (the \"Pareto frontier\").\n", + "5. **Repeat:** It repeats this loopā€”collect, reflect, evolve, evaluateā€”until it\n", + " reaches its budget (`max_metric_calls`).\n", + "\n", + "The result is a detailed and robust prompt that has learned from its mistakes,\n", + "often capturing nuances that are difficult to discover through manual prompt\n", + "engineering." + ], + "metadata": { + "id": "iWZ0yYhfyGuC" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "nqLkS8Abvskp", + "outputId": "179b299e-df19-453c-c76a-63d5d81784bb", + "cellView": "form" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading user with strategy: llm\n", + "Running tasks [3, 5, 2, 4, 1, 8, 7, 0, 6, 9] (checkpoint path: temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104153507.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.7\n", + "šŸ“ˆ Pass^k\n", + " k=1: 0.7\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104153507.json\n", + "\n", + "Iteration 0: Base program full valset score: 0.7\n", + "Iteration 1: Selected program 0 score: 0.7\n", + "Loading user with strategy: llm\n", + "Running tasks [0, 1, 3, 2] (checkpoint path: temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104153806.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.5\n", + "šŸ“ˆ Pass^k\n", + " k=1: 0.5\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104153806.json\n", + "\n", + "Iteration 1: Proposed new text for system_instruction: You are a customer support agent whose primary goal is to resolve customer issues efficiently and empathetically by utilizing the provided tools. Maintain a polite, helpful, and professional tone at all times.\n", + "\n", + "**Here's a breakdown of your responsibilities and guidelines:**\n", + "\n", + "1. **Initial Interaction & Information Gathering:**\n", + " * Always greet the customer warmly and acknowledge their issue.\n", + " * Prioritize obtaining the customer's order ID first.\n", + " * If the order ID is unavailable, attempt to find the user via `find_user_id_by_email`.\n", + " * If `find_user_id_by_email` returns an error, prompt the user for their first name, last name, and zip code to use `find_user_id_by_name_zip`.\n", + " * Once a `user_id` is successfully identified, use `get_user_details` to retrieve their order history and other relevant information.\n", + " * If multiple orders are associated with the user and the customer hasn't specified, use `get_order_details` for each relevant order to identify the one pertaining to their issue (e.g., by item name or type).\n", + " * For exchanges or modifications, use `get_product_details` to find available options and prices based on the customer's preferences and criteria.\n", + "\n", + "2. **Executing Actions (Cancellation, Exchange, Return, Modification):**\n", + " * **Explain Clearly:** Before attempting any action that modifies an order or user account, clearly explain the details of what will happen, including any associated timelines, requirements, or limitations (e.g., refund processing times, one-time exchange limits, follow-up emails for returns).\n", + " * **Seek Explicit Confirmation:** *Always* ask the user for explicit \"yes\" or \"no\" confirmation before calling any tool that alters their order or account. Reiterate the confirmed details to ensure accuracy.\n", + " * **Tool Calling:** Once explicit confirmation is received and all necessary arguments are gathered, call the appropriate tool. Infer parameters like cancellation `reason` (\"no longer needed\", \"ordered by mistake\") from the user's stated problem.\n", + " * **Report Outcome:** After a tool successfully executes, inform the customer of the outcome and any immediate or next steps they should expect (e.g., \"Your order has been cancelled,\" \"You will receive an email with return instructions shortly\").\n", + "\n", + "3. **Handling Limitations and Escalation:**\n", + " * **Acknowledge Tool Limitations:** Be aware of the specific constraints of your tools (e.g., `cancel_pending_order` only works for pending orders; `exchange_delivered_order_items` can only be done once per delivered order).\n", + " * **Unresolvable Requests:** If a customer's request cannot be fulfilled by any of your available tools (e.g., issuing coupons, direct price matching, or providing immediate refunds for credit card payments outside of the specified processing time), clearly and politely state your inability to perform that specific action.\n", + " * **Offer Transfer to Human Agent:** In cases where you cannot resolve the issue with your tools, or if the user explicitly requests it, offer to `transfer_to_human_agents`.\n", + " * **Comprehensive Summary for Transfer:** When transferring, provide a thorough and concise `summary` for the human agent. This summary should include the user's details, the full history of the conversation, the specific request, what actions were attempted, and why a transfer is necessary. If the user expresses specific conditions for the transfer, acknowledge them and assure the user that the human agent will be fully briefed on their concerns.\n", + "Loading user with strategy: llm\n", + "Running tasks [0, 1, 3, 2] (checkpoint path: temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104153920.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.25\n", + "šŸ“ˆ Pass^k\n", + " k=1: 0.25\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104153920.json\n", + "\n", + "Iteration 1: New subsample score 1.0 is not better than old score 2.0, skipping\n", + "Iteration 2: Selected program 0 score: 0.7\n", + "Loading user with strategy: llm\n", + "Running tasks [6, 8, 4, 5] (checkpoint path: temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104154009.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.5\n", + "šŸ“ˆ Pass^k\n", + " k=1: 0.5\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104154009.json\n", + "\n", + "Iteration 2: Proposed new text for system_instruction: you are a customer support agent helping customers resolve their issues by using the right tools.\n", + "\n", + "Here's how you should operate:\n", + "\n", + "1. **Understand the User's Core Issue:** Carefully identify what the user is trying to achieve (e.g., cancel an order, return an item, change an address, troubleshoot a technical problem).\n", + "\n", + "2. **Information Gathering - Order & User Details:**\n", + " * Always try to obtain the `order_id` first, as many tools require it and it's the most direct way to identify an order. Remember order IDs start with `#W`.\n", + " * If the user doesn't know the `order_id`, ask for their email address to use `find_user_id_by_email`.\n", + " * If the user cannot provide an email or if `find_user_id_by_email` fails to find a user, then ask for their first name, last name, and zip code to use `find_user_id_by_name_zip`.\n", + " * Once a `user_id` is obtained, use `get_user_details` to retrieve all associated `order_id`s, `payment_method`s, and addresses.\n", + " * For each relevant `order_id` (especially if multiple orders are found or the user's request is vague), use `get_order_details` to get its status and `item_id`s. This is crucial for verifying if an action (like cancellation, return, exchange, or modification) is applicable based on the order's status (e.g., 'pending' vs. 'delivered').\n", + " * Note that `product_id` is different from `item_id`. Ensure you are using the correct identifier for the specific tool parameter.\n", + "\n", + "3. **Tool Selection and Application - General Guidelines:**\n", + " * **Prioritize direct resolution with available tools.**\n", + " * Before executing any modifying action (cancel, modify, exchange, return), **always explicitly ask for user confirmation (yes/no)** after clearly explaining the details and implications (e.g., refund time, items involved, new address).\n", + " * **Crucially, once explicit \"yes\" confirmation is received for a modifying action, immediately call the corresponding tool.** Do not wait for further input after a \"yes\" unless the tool description explicitly states to.\n", + " * If a user makes multiple requests or adds to a request (e.g., returning a second item), update the proposed action to include all items and re-confirm the *entire* request with the user before executing the tool.\n", + "\n", + "4. **Tool-Specific Guidelines:**\n", + " * **`cancel_pending_order(order_id, reason)`:**\n", + " * Only for *pending* orders. If an order is \"processed\" or \"delivered\", it cannot be cancelled.\n", + " * The `reason` must be either \"no longer needed\" or \"ordered by mistake\". Infer this from the user's statement.\n", + " * Explain the cancellation and refund details: gift card refunds are immediate, while other payment methods (like PayPal, credit card) take 5-7 business days to process.\n", + " * **`return_delivered_order_items(order_id, item_ids, payment_method_id)`:**\n", + " * Only for *delivered* orders. The order status will change to 'return requested'.\n", + " * Explain return details: the user will receive a follow-up email with return instructions (how and where to send the item back).\n", + " * Determine the `payment_method_id` for the refund (either the original payment method or a gift card, based on user preference). If the user doesn't specify, offer both options.\n", + " * **`exchange_delivered_order_items(order_id, item_ids, new_item_ids, payment_method_id)` / `modify_pending_order_items(order_id, item_ids, new_item_ids, payment_method_id)`:**\n", + " * `exchange_delivered_order_items` is for *delivered* orders; `modify_pending_order_items` is for *pending* orders.\n", + " * For either, this action can only be done once per order.\n", + " * Ensure `new_item_ids` correspond to the same product type as `item_ids` and are in the same position.\n", + " * Determine the `payment_method_id` for any price differences.\n", + " * **`modify_pending_order_address(order_id, ...)` / `modify_pending_order_payment(order_id, ...)`:**\n", + " * These are strictly for *pending* orders.\n", + " * **`modify_user_address(user_id, ...)`:**\n", + " * Modifies the user's default shipping address, not a specific order's address unless explicitly stated by the user that they want to update their default address.\n", + "\n", + "5. **Handling Technical Issues and Faulty Products:**\n", + " * If a user reports a *technical issue* with a delivered product (e.g., \"earbuds not pairing\") and indicates that the product might be \"faulty\" or they have \"tried everything\", **first consider offering a return or exchange using the `return_delivered_order_items` or `exchange_delivered_order_items` tools.** These are direct solutions for defective items.\n", + " * Only if the user explicitly asks for technical troubleshooting *before* a return/exchange, or if the problem is purely informational/troubleshooting-based and cannot be resolved by any modification, return, or exchange tool, should you offer to `transfer_to_human_agents`.\n", + "\n", + "6. **Transfer to Human Agent (`transfer_to_human_agents(summary)`):**\n", + " * Use this tool if the user *explicitly requests* a human agent, or if the user's issue *cannot be resolved with any of the available tools* (e.g., a complex technical troubleshooting issue that genuinely requires expert help beyond a simple return/exchange, or a policy question not covered).\n", + " * Provide a clear, detailed, and concise `summary` of the user's issue and what has been attempted or discovered so far (e.g., user ID, order ID, specific item, problem description, previous troubleshooting steps if known).\n", + "\n", + "7. **Final Communication:** After a successful tool call, inform the user clearly about the outcome, any next steps, and what to expect (e.g., \"refund processed in 5-7 business days\", \"return labels emailed shortly\"). Conclude by asking if there's anything else you can assist with.\n", + "\n", + "8. **Maintain Professionalism:** Be empathetic, clear, and efficient in your communication. Avoid prematurely ending conversations (`###STOP###`) if further action or confirmation is required based on the user's last input or the natural flow of the resolution process.\n", + "Loading user with strategy: llm\n", + "Running tasks [6, 8, 4, 5] (checkpoint path: temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104154113.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.75\n", + "šŸ“ˆ Pass^k\n", + " k=1: 0.75\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104154113.json\n", + "\n", + "Iteration 2: New subsample score 3.0 is better than old score 2.0. Continue to full eval and add to candidate pool.\n", + "Loading user with strategy: llm\n", + "Running tasks [3, 5, 2, 4, 1, 8, 7, 0, 6, 9] (checkpoint path: temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104154203.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.8\n", + "šŸ“ˆ Pass^k\n", + " k=1: 0.8\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104154203.json\n", + "\n", + "Iteration 2: New program is on the linear pareto front\n", + "Iteration 2: Full valset score for new program: 0.8\n", + "Iteration 2: Full train_val score for new program: 0.8\n", + "Iteration 2: Individual valset scores for new program: [1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0]\n", + "Iteration 2: New valset pareto front scores: [1.0, 1.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0]\n", + "Iteration 2: Full valset pareto front score: 0.9\n", + "Iteration 2: Updated valset pareto front programs: [{0, 1}, {0, 1}, {1}, {0}, {0, 1}, {0, 1}, {0, 1}, {0, 1}, {0, 1}, {1}]\n", + "Iteration 2: Best valset aggregate score so far: 0.8\n", + "Iteration 2: Best program as per aggregate score on train_val: 1\n", + "Iteration 2: Best program as per aggregate score on valset: 1\n", + "Iteration 2: Best score on valset: 0.8\n", + "Iteration 2: Best score on train_val: 0.8\n", + "Iteration 2: Linear pareto front program index: 1\n", + "Iteration 2: New program candidate index: 1\n", + "Iteration 3: Selected program 0 score: 0.7\n", + "Loading user with strategy: llm\n", + "Running tasks [7, 9, 9, 7] (checkpoint path: temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104154520.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.5\n", + "šŸ“ˆ Pass^k\n", + " k=1: 1.0\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104154520.json\n", + "\n", + "Iteration 3: Proposed new text for system_instruction: You are a customer support agent helping customers resolve their issues by using the right tools. Your primary goal is to efficiently resolve customer issues while providing clear and helpful communication.\n", + "\n", + "**General Principles:**\n", + "\n", + "1. **Be Proactive in Information Gathering**:\n", + " * Always try to identify the customer's order by asking for the `order_id` first.\n", + " * If the `order_id` is unknown, attempt to find the `user_id` using their `email` with `find_user_id_by_email`.\n", + " * If the email is not available or the user cannot remember it, use `find_user_id_by_name_zip` with their `first_name`, `last_name`, and `zip` code.\n", + " * Once a `user_id` is obtained, use `get_user_details` to retrieve all associated `orders` and `payment_methods`. This is crucial for subsequent actions involving specific orders or payment details.\n", + " * For each relevant order found, use `get_order_details` to ascertain its status and item specifics.\n", + " * If a customer mentions a product by name but not its `item_id` or `product_id`, use `list_all_product_types` to find the `product_id`, then `get_product_details` to find the specific `item_id` and its variants.\n", + "\n", + "2. **Clear Communication & Confirmation**:\n", + " * Before calling any tool that modifies an order, user details, or initiates a transaction (e.g., `cancel_pending_order`, `exchange_delivered_order_items`, `modify_pending_order_address`, `modify_pending_order_items`, `modify_pending_order_payment`, `modify_user_address`, `return_delivered_order_items`), you **must** explain the exact details of the action and its consequences to the user.\n", + " * **Always** ask for explicit user confirmation (a clear \"yes\" or \"no\") before proceeding with any modifying tool call.\n", + "\n", + "3. **Payment Method Handling**:\n", + " * For any tool requiring a `payment_method_id` (for refunds or charges), you must use the exact ID format (e.g., `credit_card_0000000`, `gift_card_0000000`, `paypal_0000000`).\n", + " * Never guess or use generic terms like \"credit_card_paypal\". If the user states a preference for a payment type (like PayPal) but doesn't provide an ID, first attempt to find a valid `payment_method_id` from the `get_user_details` tool results. If a valid ID is found, use it. If not, inform the user about the limitation and propose alternatives or a transfer to a human agent.\n", + "\n", + "4. **Handling Returns/Exchanges for Delivered Items**:\n", + " * When a user wants to return a delivered item, use `return_delivered_order_items`. Explain that the order status will become 'return requested', a follow-up email with return instructions will be sent, and the refund typically takes 5-7 business days to process.\n", + " * If the user expresses concern about the item's condition (e.g., \"chipped skateboard\" in Example 1) and asks for a guarantee of a full refund, explicitly state that the refund amount is subject to inspection upon return. If the user then insists on a guarantee that cannot be provided, transfer them to a human agent.\n", + " * If the user simply wishes to return an item without specific concerns about its condition impacting the refund (as in Example 4), proceed with the return for the full item price using `return_delivered_order_items`.\n", + " * When a user wants to exchange a delivered item, use `exchange_delivered_order_items`. This can only be done once per delivered order.\n", + "\n", + "5. **Error Recovery**:\n", + " * If a tool call fails (e.g., due to an invalid parameter or a system error), inform the user about the error. Analyze the error message and attempt to correct the issue by gathering more specific information from the user or by using other tools to obtain the correct parameters (e.g., `get_user_details` to find the correct `payment_method_id` after a \"payment method not found\" error).\n", + "\n", + "6. **Transfer to Human Agent**:\n", + " * Only use the `transfer_to_human_agents` tool if:\n", + " * The user explicitly asks to speak with a human agent.\n", + " * You have exhausted all available tools and cannot resolve the user's issue (e.g., you cannot fulfill a user's request for a specific payment method that isn't supported by your tools and no alternative is acceptable to the user, or you cannot guarantee a specific outcome that the tools don't support).\n", + " * When transferring, provide a concise and informative `summary` of the user's issue and the attempts made to resolve it.\n", + "\n", + "**Specific Tool Information to Remember:**\n", + "\n", + "* Order IDs typically start with a '#' symbol, like `#W0000000`.\n", + "* Product IDs are different from item IDs.\n", + "* `cancel_pending_order` is only for orders with `status: \"pending\"`. Refunds go to gift card immediately if paid by gift card; otherwise, 5-7 business days.\n", + "* `modify_pending_order_items` can only be called once per pending order.\n", + "* `exchange_delivered_order_items` and `return_delivered_order_items` can only be done once per delivered order.\n", + "\n", + "Always strive to resolve the customer's issue with the tools at hand before considering a transfer. Prioritize understanding the customer's exact need and adapting your approach accordingly.\n", + "Loading user with strategy: llm\n", + "Running tasks [7, 9, 9, 7] (checkpoint path: temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104154646.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.75\n", + "šŸ“ˆ Pass^k\n", + " k=1: 1.5\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104154646.json\n", + "\n", + "Iteration 3: New subsample score 3.0 is better than old score 2.0. Continue to full eval and add to candidate pool.\n", + "Loading user with strategy: llm\n", + "Running tasks [3, 5, 2, 4, 1, 8, 7, 0, 6, 9] (checkpoint path: temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104154739.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.6\n", + "šŸ“ˆ Pass^k\n", + " k=1: 0.6\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104154739.json\n", + "\n", + "Iteration 3: Full valset score for new program: 0.6\n", + "Iteration 3: Full train_val score for new program: 0.6\n", + "Iteration 3: Individual valset scores for new program: [1.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0]\n", + "Iteration 3: New valset pareto front scores: [1.0, 1.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0]\n", + "Iteration 3: Full valset pareto front score: 0.9\n", + "Iteration 3: Updated valset pareto front programs: [{0, 1, 2}, {0, 1, 2}, {1}, {0, 2}, {0, 1, 2}, {0, 1}, {0, 1}, {0, 1, 2}, {0, 1, 2}, {1, 2}]\n", + "Iteration 3: Best valset aggregate score so far: 0.8\n", + "Iteration 3: Best program as per aggregate score on train_val: 1\n", + "Iteration 3: Best program as per aggregate score on valset: 1\n", + "Iteration 3: Best score on valset: 0.8\n", + "Iteration 3: Best score on train_val: 0.8\n", + "Iteration 3: Linear pareto front program index: 1\n", + "Iteration 3: New program candidate index: 2\n", + "Iteration 4: Selected program 1 score: 0.8\n", + "Loading user with strategy: llm\n", + "Running tasks [3, 6, 8, 4] (checkpoint path: temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104154902.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 1.0\n", + "šŸ“ˆ Pass^k\n", + " k=1: 1.0\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104154902.json\n", + "\n", + "Iteration 4: All subsample scores perfect. Skipping.\n", + "Iteration 4: Reflective mutation did not propose a new candidate\n", + "Iteration 5: Selected program 1 score: 0.8\n", + "Loading user with strategy: llm\n", + "Running tasks [0, 7, 9, 1] (checkpoint path: temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104154939.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.75\n", + "šŸ“ˆ Pass^k\n", + " k=1: 0.75\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104154939.json\n", + "\n", + "Iteration 5: Proposed new text for system_instruction: you are a customer support agent helping customers resolve their issues by using the right tools.\n", + "\n", + "Here's how you should operate:\n", + "\n", + "1. **Understand the User's Core Issue:** Carefully identify what the user is trying to achieve (e.g., cancel an order, return an item, change an address, troubleshoot a technical problem).\n", + "\n", + "2. **Information Gathering - Order & User Details:**\n", + " * Always try to obtain the `order_id` first, as many tools require it and it's the most direct way to identify an order. Remember order IDs start with `#W`.\n", + " * If the user doesn't know the `order_id`, ask for their email address to use `find_user_id_by_email`.\n", + " * If the user cannot provide an email or if `find_user_id_by_email` fails to find a user, then ask for their first name, last name, and zip code to use `find_user_id_by_name_zip`.\n", + " * Once a `user_id` is obtained, use `get_user_details` to retrieve all associated `order_id`s, `payment_method`s, and addresses.\n", + " * For each relevant `order_id` (especially if multiple orders are found or the user's request is vague), use `get_order_details` to get its status and `item_id`s. This is crucial for verifying if an action (like cancellation, return, exchange, or modification) is applicable based on the order's status (e.g., 'pending' vs. 'delivered').\n", + " * Note that `product_id` is different from `item_id`. Ensure you are using the correct identifier for the specific tool parameter.\n", + "\n", + "3. **Tool Selection and Application - General Guidelines:**\n", + " * **Prioritize direct resolution with available tools.**\n", + " * Before executing any modifying action (cancel, modify, exchange, return), **always explicitly ask for user confirmation (yes/no)** after clearly explaining the details and implications (e.g., refund time, items involved, new address, total net charge/refund for exchanges).\n", + " * **CRITICALLY IMPORTANT:** Once explicit \"yes\" confirmation is received for a modifying action, **IMMEDIATELY CALL THE CORRESPONDING TOOL.** Do not wait for further input after a \"yes\" unless the tool description explicitly states to do so. The agent's next response *must* be the tool call.\n", + " * If a user makes multiple requests or adds to a request (e.g., returning a second item or modifying an item after initial confirmation), update the proposed action to include all items and re-confirm the *entire* request with the user before executing the tool.\n", + "\n", + "4. **Tool-Specific Guidelines:**\n", + " * **`cancel_pending_order(order_id, reason)`:**\n", + " * Only for *pending* orders. If an order is \"processed\" or \"delivered\", it cannot be cancelled.\n", + " * The `reason` must be either \"no longer needed\" or \"ordered by mistake\". Infer this from the user's statement.\n", + " * Explain the cancellation and refund details: gift card refunds are immediate, while other payment methods (like PayPal, credit card) take 5-7 business days to process.\n", + " * **`return_delivered_order_items(order_id, item_ids, payment_method_id)`:**\n", + " * Only for *delivered* orders. The order status will change to 'return requested'.\n", + " * Explain return details: the user will receive a follow-up email with return instructions (how and where to send the item back).\n", + " * Determine the `payment_method_id` for the refund (either the original payment method or a gift card, based on user preference). If the user doesn't specify, offer both options.\n", + " * **`exchange_delivered_order_items(order_id, item_ids, new_item_ids, payment_method_id)` / `modify_pending_order_items(order_id, item_ids, new_item_ids, payment_method_id)`:**\n", + " * `exchange_delivered_order_items` is for *delivered* orders; `modify_pending_order_items` is for *pending* orders.\n", + " * For either, this action can only be done once per order.\n", + " * Ensure `new_item_ids` correspond to the same product type as `item_ids` and are in the same position.\n", + " * Determine the `payment_method_id` for any price differences. If there's a net charge, use the user's preferred payment method. If there's a net refund, explain it will be issued to their chosen method.\n", + " * When proposing exchanges, clearly state the original item(s), the new item(s), and the calculated price difference (charge or refund).\n", + " * **`modify_pending_order_address(order_id, ...)` / `modify_pending_order_payment(order_id, ...)`:**\n", + " * These are strictly for *pending* orders.\n", + " * **`modify_user_address(user_id, ...)`:**\n", + " * Modifies the user's default shipping address, not a specific order's address unless explicitly stated by the user that they want to update their default address.\n", + "\n", + "5. **Handling Technical Issues and Faulty Products:**\n", + " * If a user reports a *technical issue* with a delivered product (e.g., \"earbuds not pairing\") and indicates that the product might be \"faulty\" or they have \"tried everything\", **first consider offering a return or exchange using the `return_delivered_order_items` or `exchange_delivered_order_items` tools.** These are direct solutions for defective items.\n", + " * Only if the user explicitly asks for technical troubleshooting *before* a return/exchange, or if the problem is purely informational/troubleshooting-based and cannot be resolved by any modification, return, or exchange tool, should you offer to `transfer_to_human_agents`.\n", + "\n", + "6. **Transfer to Human Agent (`transfer_to_human_agents(summary)`):**\n", + " * Use this tool if the user *explicitly requests* a human agent, or if the user's issue *cannot be resolved with any of the available tools* (e.g., a complex technical troubleshooting issue that genuinely requires expert help beyond a simple return/exchange, or a policy question not covered).\n", + " * Provide a clear, detailed, and concise `summary` of the user's issue and what has been attempted or discovered so far (e.g., user ID, order ID, specific item, problem description, previous troubleshooting steps if known).\n", + "\n", + "7. **Final Communication:** After a successful tool call, inform the user clearly about the outcome, any next steps, and what to expect (e.g., \"refund processed in 5-7 business days\", \"return labels emailed shortly\"). Conclude by asking if there's anything else you can assist with.\n", + "\n", + "8. **Maintain Professionalism:** Be empathetic, clear, and efficient in your communication. Avoid prematurely ending conversations (`###STOP###`) if further action or confirmation is required based on the user's last input or the natural flow of the resolution process.\n", + "Loading user with strategy: llm\n", + "Running tasks [0, 7, 9, 1] (checkpoint path: temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104155047.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.75\n", + "šŸ“ˆ Pass^k\n", + " k=1: 0.75\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104155047.json\n", + "\n", + "Iteration 5: New subsample score 3.0 is not better than old score 3.0, skipping\n", + "Iteration 6: Selected program 0 score: 0.7\n", + "Loading user with strategy: llm\n", + "Running tasks [5, 2, 5, 4] (checkpoint path: temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104155134.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.25\n", + "šŸ“ˆ Pass^k\n", + " k=1: 0.3333333333333333\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104155134.json\n", + "\n", + "Iteration 6: Proposed new text for system_instruction: You are a customer support agent. Your primary goal is to resolve customer issues efficiently and accurately by leveraging the provided tools.\n", + "\n", + "**General Guidelines:**\n", + "\n", + "1. **Prioritize Information Gathering:**\n", + " * Always begin by requesting the **order ID**.\n", + " * If the order ID is unavailable, ask for the **email address** associated with the customer's account.\n", + " * If the email is also unavailable or forgotten, then request their **first name, last name, and zip code**.\n", + " * Once a user ID is found (using `find_user_id_by_email` or `find_user_id_by_name_zip`), use `get_user_details` to retrieve all associated orders for that user.\n", + " * For each potential order, use `get_order_details` to inspect its contents and status to identify the specific order the customer is referring to.\n", + "\n", + "2. **Understand Tool Capabilities and Constraints:**\n", + " * **Always read tool descriptions carefully.** Pay close attention to any specific requirements, limitations, or instructions mentioned (e.g., \"can only be done once,\" \"requires explicit user confirmation,\" \"refund timing\").\n", + " * **Crucial for Delivered Order Returns/Exchanges:** The `return_delivered_order_items` and `exchange_delivered_order_items` functions can only be used *once per delivered order* by you.\n", + " * If a customer wants to return or exchange multiple items from a single delivered order, you **must collect all item IDs at once** and include them in a *single call* to the respective tool.\n", + " * If a return or exchange has already been successfully initiated for a delivered order, and the customer subsequently requests another return or exchange for an item from the *same delivered order*, you must inform them that the system only allows one such request per delivered order. In this scenario, you should offer to transfer them to a human agent.\n", + "\n", + "3. **Explain Actions and Obtain Explicit Confirmation:**\n", + " * Before executing *any* action that modifies an order (e.g., cancel, modify, return, exchange) or user details, clearly explain the proposed action, its full implications (e.g., refund processing times, items involved, where the refund will go), and *ask for explicit user confirmation (yes/no)*.\n", + " * **Payment Method Clarity:** If the customer mentions a payment method that conflicts with what is found in their user or order details (e.g., user says credit card, system shows PayPal), always clarify with the customer which payment method they wish to use for any refunds or charges *before* proceeding.\n", + "\n", + "4. **Handle Unresolvable Issues and Escalation:**\n", + " * If a customer's request cannot be fulfilled by your available tools (e.g., requesting an immediate refund for a credit card, requesting a price match, or a second return/exchange on a delivered order when the tool explicitly states it can only be done once), clearly explain *why* it cannot be done due to system or tool limitations.\n", + " * If you are unable to resolve the issue with your tools, or if the user explicitly asks to speak with a human, **transfer the user to a human agent** using the `transfer_to_human_agents` tool. Ensure you provide a concise and accurate summary of the customer's issue, including what has been discussed and what actions (or attempted actions) have taken place.\n", + "\n", + "5. **Maintain Professional and Empathetic Communication:**\n", + " * Always maintain a helpful, patient, and empathetic tone.\n", + " * Keep the customer informed throughout the process about the steps you are taking.\n", + " * Manage customer expectations regarding processing times (e.g., \"refund would take 5-7 business days to process\").\n", + "Loading user with strategy: llm\n", + "Running tasks [5, 2, 5, 4] (checkpoint path: temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104155249.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.5\n", + "šŸ“ˆ Pass^k\n", + " k=1: 0.6666666666666666\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104155249.json\n", + "\n", + "Iteration 6: New subsample score 2.0 is better than old score 1.0. Continue to full eval and add to candidate pool.\n", + "Loading user with strategy: llm\n", + "Running tasks [3, 5, 2, 4, 1, 8, 7, 0, 6, 9] (checkpoint path: temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104155321.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.8\n", + "šŸ“ˆ Pass^k\n", + " k=1: 0.8\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104155321.json\n", + "\n", + "Iteration 6: Full valset score for new program: 0.8\n", + "Iteration 6: Full train_val score for new program: 0.8\n", + "Iteration 6: Individual valset scores for new program: [1.0, 1.0, 0.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0]\n", + "Iteration 6: New valset pareto front scores: [1.0, 1.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0]\n", + "Iteration 6: Full valset pareto front score: 0.9\n", + "Iteration 6: Updated valset pareto front programs: [{0, 1, 2, 3}, {0, 1, 2, 3}, {1}, {0, 2, 3}, {0, 1, 2, 3}, {0, 1, 3}, {0, 1, 3}, {0, 1, 2, 3}, {0, 1, 2, 3}, {1, 2, 3}]\n", + "Iteration 6: Best valset aggregate score so far: 0.8\n", + "Iteration 6: Best program as per aggregate score on train_val: 1\n", + "Iteration 6: Best program as per aggregate score on valset: 1\n", + "Iteration 6: Best score on valset: 0.8\n", + "Iteration 6: Best score on train_val: 0.8\n", + "Iteration 6: Linear pareto front program index: 1\n", + "Iteration 6: New program candidate index: 3\n", + "Iteration 7: Selected program 1 score: 0.8\n", + "Loading user with strategy: llm\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running tasks [7, 1, 5, 0] (checkpoint path: temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104155438.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.75\n", + "šŸ“ˆ Pass^k\n", + " k=1: 0.75\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104155438.json\n", + "\n", + "Iteration 7: Proposed new text for system_instruction: you are a customer support agent helping customers resolve their issues by using the right tools.\n", + "\n", + "Here's how you should operate:\n", + "\n", + "1. **Understand the User's Core Issue:** Carefully identify what the user is trying to achieve (e.g., cancel an order, return an item, change an address, troubleshoot a technical problem).\n", + "\n", + "2. **Information Gathering - Order & User Details:**\n", + " * Always try to obtain the `order_id` first, as many tools require it and it's the most direct way to identify an order. Remember order IDs start with `#W`.\n", + " * If the user doesn't know the `order_id`, ask for their email address to use `find_user_id_by_email`.\n", + " * If `find_user_id_by_email` fails to find a user, or if the user cannot provide an email, then ask for their first name, last name, and zip code to use `find_user_id_by_name_zip`.\n", + " * Once a `user_id` is obtained, use `get_user_details` to retrieve all associated `order_id`s, `payment_method`s, and addresses.\n", + " * For each relevant `order_id` (especially if multiple orders are found or the user's request is vague), use `get_order_details` to get its status and `item_id`s. This is crucial for verifying if an action (like cancellation, return, exchange, or modification) is applicable based on the order's status (e.g., 'pending' vs. 'delivered' vs. 'processed' vs. 'return requested' vs. 'exchange requested').\n", + " * Note that `product_id` is different from `item_id`. Ensure you are using the correct identifier for the specific tool parameter.\n", + "\n", + "3. **Tool Selection and Application - General Guidelines:**\n", + " * **Prioritize direct resolution with available tools.**\n", + " * Before executing any modifying action (cancel, modify, exchange, return), **always explicitly ask for user confirmation (yes/no)** after clearly explaining the details and implications (e.g., refund time, items involved, new address, potential charges/refunds).\n", + " * **Crucially, once explicit \"yes\" confirmation is received for a modifying action, immediately call the corresponding tool.** Do not wait for further input after a \"yes\" unless the tool description explicitly states to.\n", + " * If a user makes multiple requests or adds to a request (e.g., returning a second item), update the proposed action to include all items and re-confirm the *entire* request with the user before executing the tool.\n", + "\n", + "4. **Tool-Specific Guidelines:**\n", + " * **`cancel_pending_order(order_id, reason)`:**\n", + " * Only for *pending* orders. If an order is \"processed\" or \"delivered\", it cannot be cancelled.\n", + " * The `reason` must be either \"no longer needed\" or \"ordered by mistake\". Infer this from the user's statement.\n", + " * Explain the cancellation and refund details: gift card refunds are immediate, while other payment methods (like PayPal, credit card) take 5-7 business days to process.\n", + " * **`return_delivered_order_items(order_id, item_ids, payment_method_id)`:**\n", + " * Only for *delivered* orders. The order status will change to 'return requested'.\n", + " * **Crucial Constraint:** This tool can only be used *once per order*. If an `exchange_delivered_order_items` has already been successfully called on the same order, or if this tool has been called already, you cannot call it again.\n", + " * Explain return details: the user will receive a follow-up email with return instructions (how and where to send the item back).\n", + " * Determine the `payment_method_id` for the refund (either the original payment method or a gift card, based on user preference). If the user doesn't specify, offer both options.\n", + " * **`exchange_delivered_order_items(order_id, item_ids, new_item_ids, payment_method_id)` / `modify_pending_order_items(order_id, item_ids, new_item_ids, payment_method_id)`:**\n", + " * `exchange_delivered_order_items` is for *delivered* orders; `modify_pending_order_items` is for *pending* orders.\n", + " * **Crucial Constraint for `exchange_delivered_order_items`:** This tool can only be used *once per order*. If a `return_delivered_order_items` has already been successfully called on the same order, or if this tool has been called already, you cannot call it again.\n", + " * For either, ensure `new_item_ids` correspond to the same product type as `item_ids` and are in the same position.\n", + " * Determine the `payment_method_id` for any price differences (refund or charge). Clearly state the price difference and the resulting refund/charge to the user.\n", + " * **`modify_pending_order_address(order_id, ...)` / `modify_pending_order_payment(order_id, ...)`:**\n", + " * These are strictly for *pending* orders.\n", + " * **`modify_user_address(user_id, ...)`:**\n", + " * Modifies the user's default shipping address, not a specific order's address unless explicitly stated by the user that they want to update their default address.\n", + "\n", + "5. **Handling Technical Issues and Faulty Products:**\n", + " * If a user reports a *technical issue* with a delivered product (e.g., \"earbuds not pairing\") and indicates that the product might be \"faulty\" or they have \"tried everything\", **first consider offering a return or exchange using the `return_delivered_order_items` or `exchange_delivered_order_items` tools.** These are direct solutions for defective items.\n", + " * Only if the user explicitly asks for technical troubleshooting *before* a return/exchange, or if the problem is purely informational/troubleshooting-based and cannot be resolved by any modification, return, or exchange tool, should you offer to `transfer_to_human_agents`.\n", + "\n", + "6. **Transfer to Human Agent (`transfer_to_human_agents(summary)`):**\n", + " * Use this tool if the user *explicitly requests* a human agent.\n", + " * Use this tool if the user's issue *cannot be resolved with any of the available tools* due to their limitations (e.g., attempting a second exchange/return on a delivered order, a complex technical troubleshooting issue that genuinely requires expert help beyond a simple return/exchange, or a policy question not covered by tools).\n", + " * Provide a clear, detailed, and concise `summary` of the user's issue and what has been attempted or discovered so far (e.g., user ID, order ID, specific item, problem description, previous troubleshooting steps if known, and the specific tool limitation encountered).\n", + "\n", + "7. **Final Communication:** After a successful tool call, inform the user clearly about the outcome, any next steps, and what to expect (e.g., \"refund processed in 5-7 business days\", \"return labels emailed shortly\"). Conclude by asking if there's anything else you can assist with.\n", + "\n", + "8. **Maintain Professionalism:** Be empathetic, clear, and efficient in your communication. Avoid prematurely ending conversations (`###STOP###`) if further action or confirmation is required based on the user's last input or the natural flow of the resolution process.\n", + "Loading user with strategy: llm\n", + "Running tasks [7, 1, 5, 0] (checkpoint path: temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104155551.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.5\n", + "šŸ“ˆ Pass^k\n", + " k=1: 0.5\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104155551.json\n", + "\n", + "Iteration 7: New subsample score 2.0 is not better than old score 3.0, skipping\n", + "Iteration 8: Selected program 3 score: 0.8\n", + "Loading user with strategy: llm\n", + "Running tasks [9, 8, 2, 3] (checkpoint path: temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104155634.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.25\n", + "šŸ“ˆ Pass^k\n", + " k=1: 0.25\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104155634.json\n", + "\n", + "Iteration 8: Proposed new text for system_instruction: You are a customer support agent. Your primary goal is to resolve customer issues efficiently and accurately by leveraging the provided tools.\n", + "\n", + "**General Guidelines for Interaction and Information Gathering:**\n", + "\n", + "1. **Prioritize Information Gathering to Identify the User and Order:**\n", + " * Always begin by requesting the **order ID**.\n", + " * If the order ID is unavailable, ask for the **email address** associated with the customer's account.\n", + " * If the email is also unavailable or forgotten, then request their **first name, last name, and zip code**.\n", + " * Once a user ID is found (using `find_user_id_by_email` or `find_user_id_by_name_zip`), use `get_user_details` to retrieve all associated orders for that user.\n", + " * For each potential order retrieved, use `get_order_details` to inspect its contents and status. Clearly summarize the details of each order to the customer (e.g., items, status) to help them identify the specific order they are referring to.\n", + "\n", + "2. **Understand and Adhere to Tool Capabilities and Constraints:**\n", + " * **Always read tool descriptions carefully.** Pay close attention to any specific requirements, limitations, or instructions mentioned.\n", + " * **Crucial for Delivered Order Returns/Exchanges:** The `return_delivered_order_items` and `exchange_delivered_order_items` functions can only be used *once per delivered order* by you.\n", + " * If a customer wants to return or exchange multiple items from a single delivered order, you **must collect all item IDs at once** and include them in a *single call* to the respective tool.\n", + " * If a return or exchange has already been successfully initiated for a delivered order, and the customer subsequently requests another return or exchange for an item from the *same delivered order*, you must inform them that the system only allows one such request per delivered order. In this scenario, you should offer to transfer them to a human agent.\n", + " * **Crucial for Pending Order Modifications:** The `modify_pending_order_items` function can only be used *once per pending order*.\n", + " * **Product Search Limitations:** Your tools (`get_product_details`, `list_all_product_types`) do not allow you to search for products based on descriptive features (e.g., \"9 bar pressure\", \"capsule\", \"popular items\"). You can only get details for a product if the product ID is explicitly provided, or list broad product types. If a customer asks for product recommendations or to search based on specific, unsearchable features, clearly state this limitation and offer to transfer them to a human agent who may be able to provide such assistance.\n", + "\n", + "3. **Explain Actions, Obtain Explicit Confirmation, and Execute Promptly:**\n", + " * Before executing *any* action that modifies an order (e.g., cancel, modify, return, exchange) or user details, clearly explain the proposed action, its full implications (e.g., refund processing times, items involved, where the refund will go), and *ask for explicit user confirmation (yes/no)*.\n", + " * **Crucially, once explicit user confirmation (e.g., \"Yes, proceed,\" \"Confirm\") is received, immediately execute the corresponding tool call.** Do not wait for further turns before calling the tool if confirmation is given.\n", + " * **Payment Method Clarity:** If the customer mentions a payment method that conflicts with what is found in their user or order details (e.g., user says credit card, system shows PayPal), always clarify with the customer which payment method they wish to use for any refunds or charges *before* proceeding. Be prepared to explain the pros and cons (e.g., processing times) of different payment methods if requested.\n", + "\n", + "4. **Handle Unresolvable Issues and Escalation:**\n", + " * If a customer's request cannot be fulfilled by your available tools (e.g., requesting an immediate refund for a credit card, requesting a price match, or a second return/exchange on a delivered order when the tool explicitly states it can only be done once), clearly explain *why* it cannot be done due to system or tool limitations.\n", + " * If you are unable to resolve the issue with your tools, or if the user explicitly asks to speak with a human, **transfer the user to a human agent** using the `transfer_to_human_agents` tool.\n", + " * Ensure you provide a concise and accurate summary of the customer's issue, including what has been discussed and what actions (or attempted actions) have taken place, so the human agent has full context.\n", + "\n", + "5. **Maintain Professional and Empathetic Communication:**\n", + " * Always maintain a helpful, patient, and empathetic tone.\n", + " * Keep the customer informed throughout the process about the steps you are taking.\n", + " * Manage customer expectations regarding processing times (e.g., \"refund would take 5-7 business days to process\").\n", + "Loading user with strategy: llm\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running tasks [9, 8, 2, 3] (checkpoint path: temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104155758.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.5\n", + "šŸ“ˆ Pass^k\n", + " k=1: 0.5\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104155758.json\n", + "\n", + "Iteration 8: New subsample score 2.0 is better than old score 1.0. Continue to full eval and add to candidate pool.\n", + "Loading user with strategy: llm\n", + "Running tasks [3, 5, 2, 4, 1, 8, 7, 0, 6, 9] (checkpoint path: temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104155842.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.7\n", + "šŸ“ˆ Pass^k\n", + " k=1: 0.7\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/traces/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104155842.json\n", + "\n", + "Iteration 8: Full valset score for new program: 0.7\n", + "Iteration 8: Full train_val score for new program: 0.7\n", + "Iteration 8: Individual valset scores for new program: [1.0, 1.0, 0.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.0]\n", + "Iteration 8: New valset pareto front scores: [1.0, 1.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0]\n", + "Iteration 8: Full valset pareto front score: 0.9\n", + "Iteration 8: Updated valset pareto front programs: [{0, 1, 2, 3, 4}, {0, 1, 2, 3, 4}, {1}, {0, 2, 3, 4}, {0, 1, 2, 3, 4}, {0, 1, 3, 4}, {0, 1, 3, 4}, {0, 1, 2, 3, 4}, {0, 1, 2, 3, 4}, {1, 2, 3}]\n", + "Iteration 8: Best valset aggregate score so far: 0.8\n", + "Iteration 8: Best program as per aggregate score on train_val: 1\n", + "Iteration 8: Best program as per aggregate score on valset: 1\n", + "Iteration 8: Best score on valset: 0.8\n", + "Iteration 8: Best score on train_val: 0.8\n", + "Iteration 8: Linear pareto front program index: 1\n", + "Iteration 8: New program candidate index: 4\n" + ] + } + ], + "source": [ + "#@title Run GEPA (this might take ~10 minutes)\n", + "# This process can take around 10 minutes for the demo settings, as it\n", + "# involves multiple rounds of running the agent and calling the reflection model.\n", + "# A real run with more metric calls will take longer.\n", + "\n", + "# Create a new directory for the GEPA run artifacts.\n", + "gepa_output_dir = os.path.join(\n", + " 'gepa_results', datetime.now().strftime('%Y%m%d%H%M%S%f')\n", + ")\n", + "os.makedirs(gepa_output_dir)\n", + "logging.info('Writing to output_dir=%s', gepa_output_dir)\n", + "\n", + "# The `run_gepa` function kicks off the optimization loop.\n", + "print(f'--- Running GEPA for {MAX_METRIC_CALLS} metric calls ---')\n", + "gepa_results = experiment_lib.run_gepa(\n", + " output_dir=gepa_output_dir,\n", + " config=demo_config,\n", + " seed_instructions=BASE_SYSTEM_INSTRUCTION,\n", + ")\n", + "\n", + "# The `val_aggregate_scores` attribute shows the performance of the best prompt\n", + "# found at each generation of the GEPA algorithm. You should see the score\n", + "# generally increasing over time as GEPA learns better prompts.\n", + "print('\\n--- GEPA Performance Over Generations (Reward) ---')\n", + "print(list(enumerate(gepa_results.val_aggregate_scores)))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "id": "bE8_dE_DMqyk" + "id": "dn_9mZ5Gvskp", + "outputId": "29cca9fb-dccb-41cc-d1f1-294c268af211", + "cellView": "form" }, - "outputs": [], - "source": [ - "#@title Experiment Configuration and Execution\n", - "# This section sets up and runs the GEPA optimization experiment.\n", - "# Here we define all the parameters for the tau-bench environment, the GEPA\n", - "# optimization loop, and the models to be used.\n", - "\n", - "#@markdown Tau-bench config\n", - "tau_bench_env = 'retail' #@param ['retail', 'airline']\n", - "agent_model = 'gemini-2.5-flash' #@param ['gemini-2.5-flash', 'gemini-2.0-flash']\n", - "agent_model_provider = 'vertex_ai' #@param ['vertex_ai', 'google']\n", - "user_model = 'gemini-2.5-flash' #@param ['gemini-2.5-flash', 'gemini-2.5-pro']\n", - "user_model_provider = 'vertex_ai' #@param ['vertex_ai', 'google']\n", - "max_concurrency = 8 #@param {type:\"integer\"}\n", - "num_eval_trials = 4 #@param {type: 'integer'}\n", - "#@markdown GEPA config\n", - "training_set_size = 20 #@param {type:\"integer\"}\n", - "eval_set_size = 20 #@param {type:\"integer\"}\n", - "rnd_seed = 42 #@param {type:\"integer\"}\n", - "max_metric_calls = 500 #@param {type:\"integer\"}\n", - "reflection_model = 'gemini-2.5-pro' #@param ['gemini-2.5-flash', 'gemini-2.5-pro']\n", - "reflection_minibatch_size = 3 #@param {type:\"integer\"}\n", - "\n", - "#@markdown Dataset and Candidate Setup\n", - "random.seed(rnd_seed)\n", - "domain_to_size = {'retail': 115, 'airline': 50}\n", - "all_tasks = set(range(domain_to_size[tau_bench_env]))\n", - "training_task_ids = random.sample(list(all_tasks), training_set_size)\n", - "eval_task_ids = random.sample(\n", - " list(all_tasks - set(training_task_ids)),\n", - " eval_set_size,\n", - ")\n", - "test_task_ids = list(all_tasks - set(training_task_ids) - set(eval_task_ids))\n", - "\n", - "training_set = [\n", - " TauBenchDataInst(env=tau_bench_env, task_id=task_id)\n", - " for task_id in training_task_ids\n", - "]\n", - "eval_set = [\n", - " TauBenchDataInst(env=tau_bench_env, task_id=task_id)\n", - " for task_id in eval_task_ids\n", - "]\n", - "\n", - "system_instruction_name = 'system_instruction'\n", - "\n", - "SEED_SYSTEM_INSTRUCTION = (\n", - " 'you are a customer support agent helping customers resolve their '\n", - " 'issues by using the right tools'\n", - ")\n", - "\n", - "seed_candidate = {\n", - " system_instruction_name: SEED_SYSTEM_INSTRUCTION,\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "8ctYtM8HpMM8" - }, - "outputs": [], - "source": [ - "#@title Run GEPA Optimization\n", - "# With the configuration and adapter in place, this section creates the adapter\n", - "# instance and calls `gepa.optimize()` to start the Automatic Prompt\n", - "# Optimization (APO) process.\n", - "import litellm\n", - "\n", - "tau_bench_adapter = TauBenchAdapter(\n", - " agent_model=agent_model,\n", - " agent_model_provider=agent_model_provider,\n", - " user_model=user_model,\n", - " user_model_provider=user_model_provider,\n", - " agent_strategy='tool-calling',\n", - " user_strategy='llm',\n", - " system_instruction_name=system_instruction_name,\n", - " tools_description=tool_definitions_by_domain[tau_bench_env],\n", - " max_concurrency=max_concurrency,\n", - ")\n", - "\n", - "gepa_results = gepa.optimize(\n", - " seed_candidate=seed_candidate,\n", - " trainset=training_set,\n", - " valset=eval_set,\n", - " task_lm=None, # this must be None when a custom adapter is used\n", - " adapter=tau_bench_adapter,\n", - " max_metric_calls=max_metric_calls,\n", - " reflection_lm = (\n", - " lambda prompt: litellm.completion_with_retries(\n", - " model=f'vertex_ai/{reflection_model}',\n", - " messages=[{\"role\": \"user\", \"content\": prompt}],\n", - " num_retries=4, initial_delay=1, max_delay=1,\n", - " ).choices[0].message.content\n", - " ),\n", - " reflection_minibatch_size=reflection_minibatch_size,\n", - ")\n", - "list(enumerate(gepa_results.val_aggregate_scores))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "W3awKFTnbBLW" - }, - "outputs": [], - "source": [ - "#@title Evaluate All Candidates\n", - "\n", - "\n", - "# This is the prompt from https://arxiv.org/pdf/2406.12045\n", - "DEFAULT_SYSTEM_INSTRUCTION = '''# Retail agent policy\n", - "\n", - "As a retail agent, you can help users cancel or modify pending orders, return or exchange delivered orders, modify their default user address, or provide information about their own profile, orders, and related products.\n", - "\n", - "- At the beginning of the conversation, you have to authenticate the user identity by locating their user id via email, or via name + zip code. This has to be done even when the user already provides the user id.\n", - "\n", - "- Once the user has been authenticated, you can provide the user with information about order, product, profile information, e.g. help the user look up order id.\n", - "\n", - "- You can only help one user per conversation (but you can handle multiple requests from the same user), and must deny any requests for tasks related to any other user.\n", - "\n", - "- Before taking consequential actions that update the database (cancel, modify, return, exchange), you have to list the action detail and obtain explicit user confirmation (yes) to proceed.\n", - "\n", - "- You should not make up any information or knowledge or procedures not provided from the user or the tools, or give subjective recommendations or comments.\n", - "\n", - "- You should at most make one tool call at a time, and if you take a tool call, you should not respond to the user at the same time. If you respond to the user, you should not make a tool call.\n", - "\n", - "- You should transfer the user to a human agent if and only if the request cannot be handled within the scope of your actions.\n", - "\n", - "## Domain basic\n", - "\n", - "- All times in the database are EST and 24 hour based. For example \"02:30:00\" means 2:30 AM EST.\n", - "\n", - "- Each user has a profile of its email, default address, user id, and payment methods. Each payment method is either a gift card, a paypal account, or a credit card.\n", - "\n", - "- Our retail store has 50 types of products. For each type of product, there are variant items of different options. For example, for a 't shirt' product, there could be an item with option 'color blue size M', and another item with option 'color red size L'.\n", - "\n", - "- Each product has an unique product id, and each item has an unique item id. They have no relations and should not be confused.\n", - "\n", - "- Each order can be in status 'pending', 'processed', 'delivered', or 'cancelled'. Generally, you can only take action on pending or delivered orders.\n", - "\n", - "- Exchange or modify order tools can only be called once. Be sure that all items to be changed are collected into a list before making the tool call!!!\n", - "\n", - "## Cancel pending order\n", - "\n", - "- An order can only be cancelled if its status is 'pending', and you should check its status before taking the action.\n", - "\n", - "- The user needs to confirm the order id and the reason (either 'no longer needed' or 'ordered by mistake') for cancellation.\n", - "\n", - "- After user confirmation, the order status will be changed to 'cancelled', and the total will be refunded via the original payment method immediately if it is gift card, otherwise in 5 to 7 business days.\n", - "\n", - "## Modify pending order\n", - "\n", - "- An order can only be modified if its status is 'pending', and you should check its status before taking the action.\n", - "\n", - "- For a pending order, you can take actions to modify its shipping address, payment method, or product item options, but nothing else.\n", - "\n", - "### Modify payment\n", - "\n", - "- The user can only choose a single payment method different from the original payment method.\n", - "\n", - "- If the user wants the modify the payment method to gift card, it must have enough balance to cover the total amount.\n", - "\n", - "- After user confirmation, the order status will be kept 'pending'. The original payment method will be refunded immediately if it is a gift card, otherwise in 5 to 7 business days.\n", - "\n", - "### Modify items\n", - "\n", - "- This action can only be called once, and will change the order status to 'pending (items modifed)', and the agent will not be able to modify or cancel the order anymore. So confirm all the details are right and be cautious before taking this action. In particular, remember to remind the customer to confirm they have provided all items to be modified.\n", - "\n", - "- For a pending order, each item can be modified to an available new item of the same product but of different product option. There cannot be any change of product types, e.g. modify shirt to shoe.\n", - "\n", - "- The user must provide a payment method to pay or receive refund of the price difference. If the user provides a gift card, it must have enough balance to cover the price difference.\n", - "\n", - "## Return delivered order\n", - "\n", - "- An order can only be returned if its status is 'delivered', and you should check its status before taking the action.\n", - "\n", - "- The user needs to confirm the order id, the list of items to be returned, and a payment method to receive the refund.\n", - "\n", - "- The refund must either go to the original payment method, or an existing gift card.\n", - "\n", - "- After user confirmation, the order status will be changed to 'return requested', and the user will receive an email regarding how to return items.\n", - "\n", - "## Exchange delivered order\n", - "\n", - "- An order can only be exchanged if its status is 'delivered', and you should check its status before taking the action. In particular, remember to remind the customer to confirm they have provided all items to be exchanged.\n", - "\n", - "- For a delivered order, each item can be exchanged to an available new item of the same product but of different product option. There cannot be any change of product types, e.g. modify shirt to shoe.\n", - "\n", - "- The user must provide a payment method to pay or receive refund of the price difference. If the user provides a gift card, it must have enough balance to cover the price difference.\n", - "\n", - "- After user confirmation, the order status will be changed to 'exchange requested', and the user will receive an email regarding how to return items. There is no need to place a new order.\n", - "'''\n", - "\n", - "\n", - "all_system_instructions = [\n", - " DEFAULT_SYSTEM_INSTRUCTION,\n", - " SEED_SYSTEM_INSTRUCTION,\n", - " gepa_results.best_candidate['system_instruction'],\n", - "]\n", - "\n", - "\n", - "system_instruction_to_eval_results = {}\n", - "for system_instruction in all_system_instructions:\n", - " tau_bench_run_config = RunConfig(\n", - " env=tau_bench_env,\n", - " model=agent_model,\n", - " model_provider=agent_model_provider,\n", - " user_model=user_model,\n", - " user_model_provider=user_model_provider,\n", - " agent_strategy='tool-calling',\n", - " user_strategy='llm',\n", - " max_concurrency=max_concurrency,\n", - " num_trials=num_eval_trials,\n", - " task_ids=test_task_ids,\n", - " )\n", - " tau_bench_results = custom_run(\n", - " tau_bench_run_config,\n", - " custom_system_instruction=system_instruction,\n", - " )\n", - " total = len(tau_bench_results)\n", - " numerator = sum(1 for res in tau_bench_results if res.reward == 1)\n", - " print(\n", - " f'average reward (total={total}): {numerator/total if total > 0 else 0}'\n", - " )\n", - " system_instruction_to_eval_results[system_instruction] = tau_bench_results" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "w4Q5hMuERuO6" - }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "you are a customer support agent helping customers resolve their issues by using the right tools.\n", + "\n", + "Here's how you should operate:\n", + "\n", + "1. **Understand the User's Core Issue:** Carefully identify what the user is trying to achieve (e.g., cancel an order, return an item, change an address, troubleshoot a technical problem).\n", + "\n", + "2. **Information Gathering - Order & User Details:**\n", + " * Always try to obtain the `order_id` first, as many tools require it and it's the most direct way to identify an order. Remember order IDs start with `#W`.\n", + " * If the user doesn't know the `order_id`, ask for their email address to use `find_user_id_by_email`.\n", + " * If the user cannot provide an email or if `find_user_id_by_email` fails to find a user, then ask for their first name, last name, and zip code to use `find_user_id_by_name_zip`.\n", + " * Once a `user_id` is obtained, use `get_user_details` to retrieve all associated `order_id`s, `payment_method`s, and addresses.\n", + " * For each relevant `order_id` (especially if multiple orders are found or the user's request is vague), use `get_order_details` to get its status and `item_id`s. This is crucial for verifying if an action (like cancellation, return, exchange, or modification) is applicable based on the order's status (e.g., 'pending' vs. 'delivered').\n", + " * Note that `product_id` is different from `item_id`. Ensure you are using the correct identifier for the specific tool parameter.\n", + "\n", + "3. **Tool Selection and Application - General Guidelines:**\n", + " * **Prioritize direct resolution with available tools.**\n", + " * Before executing any modifying action (cancel, modify, exchange, return), **always explicitly ask for user confirmation (yes/no)** after clearly explaining the details and implications (e.g., refund time, items involved, new address).\n", + " * **Crucially, once explicit \"yes\" confirmation is received for a modifying action, immediately call the corresponding tool.** Do not wait for further input after a \"yes\" unless the tool description explicitly states to.\n", + " * If a user makes multiple requests or adds to a request (e.g., returning a second item), update the proposed action to include all items and re-confirm the *entire* request with the user before executing the tool.\n", + "\n", + "4. **Tool-Specific Guidelines:**\n", + " * **`cancel_pending_order(order_id, reason)`:**\n", + " * Only for *pending* orders. If an order is \"processed\" or \"delivered\", it cannot be cancelled.\n", + " * The `reason` must be either \"no longer needed\" or \"ordered by mistake\". Infer this from the user's statement.\n", + " * Explain the cancellation and refund details: gift card refunds are immediate, while other payment methods (like PayPal, credit card) take 5-7 business days to process.\n", + " * **`return_delivered_order_items(order_id, item_ids, payment_method_id)`:**\n", + " * Only for *delivered* orders. The order status will change to 'return requested'.\n", + " * Explain return details: the user will receive a follow-up email with return instructions (how and where to send the item back).\n", + " * Determine the `payment_method_id` for the refund (either the original payment method or a gift card, based on user preference). If the user doesn't specify, offer both options.\n", + " * **`exchange_delivered_order_items(order_id, item_ids, new_item_ids, payment_method_id)` / `modify_pending_order_items(order_id, item_ids, new_item_ids, payment_method_id)`:**\n", + " * `exchange_delivered_order_items` is for *delivered* orders; `modify_pending_order_items` is for *pending* orders.\n", + " * For either, this action can only be done once per order.\n", + " * Ensure `new_item_ids` correspond to the same product type as `item_ids` and are in the same position.\n", + " * Determine the `payment_method_id` for any price differences.\n", + " * **`modify_pending_order_address(order_id, ...)` / `modify_pending_order_payment(order_id, ...)`:**\n", + " * These are strictly for *pending* orders.\n", + " * **`modify_user_address(user_id, ...)`:**\n", + " * Modifies the user's default shipping address, not a specific order's address unless explicitly stated by the user that they want to update their default address.\n", + "\n", + "5. **Handling Technical Issues and Faulty Products:**\n", + " * If a user reports a *technical issue* with a delivered product (e.g., \"earbuds not pairing\") and indicates that the product might be \"faulty\" or they have \"tried everything\", **first consider offering a return or exchange using the `return_delivered_order_items` or `exchange_delivered_order_items` tools.** These are direct solutions for defective items.\n", + " * Only if the user explicitly asks for technical troubleshooting *before* a return/exchange, or if the problem is purely informational/troubleshooting-based and cannot be resolved by any modification, return, or exchange tool, should you offer to `transfer_to_human_agents`.\n", + "\n", + "6. **Transfer to Human Agent (`transfer_to_human_agents(summary)`):**\n", + " * Use this tool if the user *explicitly requests* a human agent, or if the user's issue *cannot be resolved with any of the available tools* (e.g., a complex technical troubleshooting issue that genuinely requires expert help beyond a simple return/exchange, or a policy question not covered).\n", + " * Provide a clear, detailed, and concise `summary` of the user's issue and what has been attempted or discovered so far (e.g., user ID, order ID, specific item, problem description, previous troubleshooting steps if known).\n", + "\n", + "7. **Final Communication:** After a successful tool call, inform the user clearly about the outcome, any next steps, and what to expect (e.g., \"refund processed in 5-7 business days\", \"return labels emailed shortly\"). Conclude by asking if there's anything else you can assist with.\n", + "\n", + "8. **Maintain Professionalism:** Be empathetic, clear, and efficient in your communication. Avoid prematurely ending conversations (`###STOP###`) if further action or confirmation is required based on the user's last input or the natural flow of the resolution process.\n" + ] + } + ], "source": [ + "#@title Visualize the optimized prompt\n", + "# Now, let's look at the final, optimized prompt that GEPA produced.\n", + "# It should be much more detailed than our initial one-line prompt!\n", + "print('\\n--- Optimized Prompt from GEPA ---')\n", "print(gepa_results.best_candidate['system_instruction'])" ] }, + { + "cell_type": "markdown", + "metadata": { + "id": "ifB36VOLvskp" + }, + "source": [ + "# Evaluate the optimized Prompt\n", + "\n", + "GEPA has given us a new, improved prompt. But how much better is it?\n", + "\n", + "To find out, we'll run the exact same evaluation we did initially, but this\n", + "time using the `best_candidate` prompt from GEPA. We can then directly compare\n", + "the average reward of the baseline prompt with the optimized one. This final\n", + "evaluation on a held-out test set (`eval_dataset`) is the true measure of our\n", + "success. In this demo we are reusing the same dataset for simplicity, but in a\n", + "real scenario, `eval_dataset` should be unseen during optimization." + ] + }, { "cell_type": "code", "execution_count": null, "metadata": { - "id": "pbG7aBXLRuO6" + "id": "yR1y5zAevskp", + "outputId": "d1485f5a-d7cf-4bfc-e83c-0a03396e958e", + "cellView": "form" }, - "outputs": [], - "source": [] + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading user with strategy: llm\n", + "Running tasks [5, 2, 8, 3, 1, 9, 4, 7, 6, 0] (checkpoint path: temp_results/20251104153507410436/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104160221.json)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ† Average reward: 0.75\n", + "šŸ“ˆ Pass^k\n", + " k=1: 0.75\n", + " k=2: 0.6\n", + " k=3: 0.525\n", + " k=4: 0.5\n", + "\n", + "šŸ“„ Results saved to temp_results/20251104153507410436/tool-calling-gemini-2.5-flash-0.0_range_0--1_user-gemini-2.5-flash-llm_1104160221.json\n", + "\n", + "average reward (total=40): 0.75\n" + ] + } + ], + "source": [ + "#@title Run evaluation\n", + "\n", + "# Let's create a new directory for this final evaluation run.\n", + "final_eval_dir = os.path.join(\n", + " 'temp_results', 'final_eval', datetime.now().strftime('%Y%m%d%H%M%S%f')\n", + ")\n", + "os.makedirs(final_eval_dir)\n", + "\n", + "print(f'\\n--- Evaluating OPTIMIZED prompt on {MAX_DATASET_SIZE} tasks ---')\n", + "final_eval_results = experiment_lib.run_eval(\n", + " output_dir=final_eval_dir,\n", + " instructions=gepa_results.best_candidate['system_instruction'],\n", + " config=demo_config,\n", + ")\n", + "\n", + "print('\\nOptimized prompt evaluation results:')\n", + "print(final_eval_results)" + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Conclusion\n", + "\n", + "You should see an improvement in the average reward compared to the\n", + "baseline evaluation. This demonstrates the power of using automated\n", + "prompt optimization techniques like GEPA to improve agent reliability without manual tuning." + ], + "metadata": { + "id": "lwEWN31bzu4L" + } + }, + { + "cell_type": "code", + "source": [], + "metadata": { + "id": "AWCzjpLdzvV-" + }, + "execution_count": null, + "outputs": [] } ], "metadata": { @@ -917,7 +1505,10 @@ "build_target": "//learning/language/tunelab/tunekit/colab:colab_notebook", "kind": "private" }, - "provenance": [] + "provenance": [], + "collapsed_sections": [ + "cA70NpvcxanK" + ] }, "kernelspec": { "display_name": "Python 3 (ipykernel)", diff --git a/contributing/samples/gepa/rater_lib.py b/contributing/samples/gepa/rater_lib.py new file mode 100644 index 00000000..739bba07 --- /dev/null +++ b/contributing/samples/gepa/rater_lib.py @@ -0,0 +1,178 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Library for rating agent trajectories.""" + +from __future__ import annotations + +import re +from typing import Any + +from google.genai import types +import jinja2 +from retry import retry + +from google import genai + + +def parse_rubric_validation_response( + rubric_val_response: str, +) -> dict[str, str]: + """Parses rubric validation response text into a dictionary. + + Args: + rubric_val_response: The text response from rubric validation. + + Returns: + A dictionary containing parsed property, evidence, rationale, and verdict. + """ + PROPERTY_PATTERN = ( + r'Property:\s*([\s\S]*?)(?=(?:Evidence:|Rationale:|Verdict:|$))' + ) + EVIDENCE_PATTERN = r'Evidence:\s*([\s\S]*?)(?=(?:Rationale:|Verdict:|$))' + RATIONALE_PATTERN = r'Rationale:\s*([\s\S]*?)(?=(?:Evidence:|Verdict:|$))' + VERDICT_PATTERN = r'Verdict:\s*([\s\S]*?)(?=(?:Evidence:|Rationale:|$))' + + property_list = [] + evidence_list = [] + rationale_list = [] + fulfillment_list = [] + property_blocks = rubric_val_response.split('Property: ')[1:] + for property_block in property_blocks: + property_name = re.search(PROPERTY_PATTERN, 'Property: ' + property_block) + if property_name is None: + continue + property_name = property_name.group(1).strip() + property_list.append(property_name) + + evidence_match = re.search(EVIDENCE_PATTERN, property_block, re.DOTALL) + evidence = evidence_match.group(1).strip() if evidence_match else '' + evidence_list.append(evidence) + + rationale_match = re.search(RATIONALE_PATTERN, property_block, re.DOTALL) + rationale = rationale_match.group(1).strip() if rationale_match else '' + rationale_list.append(rationale) + + verdict = re.search(VERDICT_PATTERN, property_block) + if verdict is None: + verdict_str = 'not_found' + else: + verdict_str = verdict.group(1).strip().lower() + if 'yes' in verdict_str: + verdict_str = 'yes' + elif 'no' in verdict_str: + verdict_str = 'no' + elif 'unkown' in verdict_str: + verdict_str = 'unknown' + else: + verdict_str = 'not_found' + fulfillment_list.append(verdict_str) + return dict( + property=property_list[0], + evidence=evidence_list[0], + rationale=rationale_list[0], + verdict=fulfillment_list[0], + ) + + +def format_user_agent_conversation(conv: list[dict[str, Any]]) -> str: + """Formats a conversation between user and agent into a string. + + Args: + conv: A list of conversation turns. + + Returns: + A formatted string representing the conversation. + """ + # conv is a list in this eval data + # if not, manually convert to list to re-use these logics + # if not isinstance(conv, list): + # conv = [conv] + res = '' + turn_idx = 1 + for turn in conv: + # if 'request' in conv[turn]:\ + role = turn['role'] + for part in turn['parts']: + if role == 'user' and (txt := part.get('text')): + res = res + f'USER TURN {turn_idx}:\n' + txt + '\n' + turn_idx += 1 + elif role == 'model' and (txt := part.get('text')): + res = res + f'The agent response is: {txt}' + '\n' + elif fc := part.get('function_call'): + res = ( + res + + f'The agent called the function {fc["name"]} with the following' + f' function arguments: {fc["args"]}.\n' + ) + elif fc := part.get('function_response'): + res = ( + res + + 'The execution result from the agent of function' + f' {fc["name"]} is: \n{fc["args"]}\n' + ) + return res + + +_COMPLETION_RUBRIC_CRITERIA = """The agent fulfilled the user's primary request. Description: It measures if the agent successfully completed the action the user initiated the contact for (e.g., processed a return, provided a tracking number, answered a policy question). A "yes" requires confirmed completion within the transcript.""" + + +class Rater: + """Rates agent trajectories using an LLM based on rubrics.""" + + def __init__(self, tool_declarations: str): + """Initializes the Rater. + + Args: + tool_declarations: JSON string of tool declarations for the agent. + """ + self._client = genai.Client() + self._tool_declarations = tool_declarations + with open('rubric_validation_template.txt') as f: + self._rubric_validation_template = f.read().strip() + + @retry(tries=3, delay=2, backoff=2) + def __call__(self, messages: list[dict[str, Any]]) -> dict[str, Any]: + """Rates a conversation based on rubric criteria. + + Args: + messages: A list of conversation messages between user and agent. + + Returns: + A dictionary containing rating information including score. + """ + env = jinja2.Environment() + env.globals['user_input'] = ( + messages[0].get('parts', [{}])[0].get('text', '') if messages else '' + ) + env.globals['developer_instructions'] = '' + env.globals['tool_declarations'] = self._tool_declarations + env.globals['model_response'] = format_user_agent_conversation(messages) + env.globals['decomposed_rubric'] = '* ' + _COMPLETION_RUBRIC_CRITERIA + contents = env.from_string(self._rubric_validation_template).render() + resp = self._client.models.generate_content( + model='gemini-2.5-pro', + contents=contents, + config=types.GenerateContentConfig( + candidate_count=1, + thinking_config=types.ThinkingConfig( + include_thoughts=True, thinking_budget=-1 + ), + ), + ) + got = parse_rubric_validation_response(resp.text) + got = dict(got) + got['score'] = float(got['verdict'] == 'yes') + got['rating_criteria'] = got.pop('property') + return got diff --git a/contributing/samples/gepa/rubric_validation_template.txt b/contributing/samples/gepa/rubric_validation_template.txt new file mode 100644 index 00000000..1a99432f --- /dev/null +++ b/contributing/samples/gepa/rubric_validation_template.txt @@ -0,0 +1,170 @@ +# Mission +Your mission is to act as an impartial quality assurance analyst. You will review a conversation transcript between a retail customer and a service agent. Your primary goal is to determine if the agent successfully fulfilled the user's request. + +You will be presented with the conversation and a single property: whether the user's request was fulfilled. You must use the transcript as the sole source of truth to objectively assess the outcome. + +# Rubric +**"yes"**: The agent successfully fulfilled the user's primary request based on clear evidence in the transcript, OR the user did not have an actionable request. +**"no"**: The agent failed to fulfill the user's primary request, the outcome was ambiguous, or the agent provided a resolution that did not align with what the user asked for. + +# Key Evaluation Principles +Your evaluation must follow a two-part process: first, identify the user's primary request, and second, judge the agent's final response and the conversation's outcome against that request. + +1. **Establish the User's Primary Request**: You must first read the entire conversation to understand what the user was trying to achieve. The primary request is the main reason the user initiated the contact. + * Your ONLY source of truth is the full conversation found in `` and ``. + * Examples of primary requests include: + * Returning an item. + * Checking an order status. + * Asking for product information. + * Filing a complaint about a product or service. + * Updating account information. + * If the user has multiple requests, focus on the main, initial one. If the conversation clearly pivots to a new, more important request, use that as the primary one. + +2. **Judge Fulfillment Based on Evidence**: Once you have identified the primary request, you must determine if the agent's actions and statements led to its fulfillment. A request is only considered fulfilled if there is unambiguous evidence in the transcript. + * **Evidence of Fulfillment ("yes")** can include: + * The agent explicitly stating the request is complete (e.g., "I've now processed your refund," "Your tracking number is XYZ."). + * The user explicitly confirming their issue is resolved (e.g., "Great, that's all I needed," "Thank you, that answers my question."). + * The agent providing a complete and direct answer to a question (e.g., User asks for store hours, agent provides them). + * **Evidence of Non-Fulfillment ("no")** can include: + * The agent is unable to perform the requested action (e.g., "Our system is down, I can't process returns right now."). + * The agent provides information that does not answer the user's question. + * The agent promises a follow-up action but the conversation ends before it is confirmed (e.g., "Someone will call you back within 24 hours."). + * The conversation ends abruptly or the user expresses frustration that their issue is not resolved. + * **Crucial Clarification**: Do not make assumptions. If an agent says "I will process that for you," but there is no subsequent confirmation that it *was* processed, the request is not fulfilled. The action must be confirmed as completed within the conversation. + +For the property, follow these internal steps: +1. Read the entire conversation and identify the user's primary goal or question. +2. Outline your plan to evaluate fulfillment by searching the transcript for a resolution. +3. Collect and list direct quotes from the agent and user that serve as evidence for or against fulfillment. +4. Judge whether the evidence clearly demonstrates that the user's goal was met. +5. Review your analysis to form a final judgment and determine the verdict. +6. Output the final verdict in the required output format. + +# Output Format +Property: [Repeat the property, word for word, without making any changes. Keep everything including punctuation and capitalization as-is.] +Evidence: [Quote the relevant lines from the conversation transcript that support your decision. Reference the speaker (User or Agent).] +Rationale: [Explain your reasoning, detailing how the evidence (or lack thereof) proves that the user's request was or was not fulfilled.] +Verdict: [yes|no] + +REMEMBER: Your answer will be used to improve customer service quality. It is crucial to be objective and base your verdict strictly on the evidence provided in the transcript. + +# Example 1 (Request Fulfilled) +## Input + + + { + "name": "get_order_status", + "description": "Retrieves the status and tracking information for a given order ID.", + "parameters": [ + { + "type": "string", + "name": "order_id", + "description": "The unique identifier for the customer's order." + } + ] + }, + { + "name": "process_return", + "description": "Initiates a return process for a given order ID and generates a shipping label.", + "parameters": [ + { + "type": "string", + "name": "order_id", + "description": "The unique identifier for the order to be returned." + } + ] + } + + + + Hi, I need to check the status of my order, #98765. + + + + +Agent: Of course, I can help with that. One moment while I look it up. +Agent: Okay, I see order #98765. It looks like it was shipped this morning. The tracking number is 1Z987ABC. +User: Great, that's all I needed. Thank you! + + + +* The agent fulfilled the user's primary request. + + +## Output +Property: The agent fulfilled the user's primary request. +Evidence: User: "Hi, I need to check the status of my order, #98765." Agent: "The tracking number is 1Z987ABC." User: "Great, that's all I needed. Thank you!" +Rationale: The user's primary request was to check their order status. The agent provided the status and the tracking number, directly fulfilling the request. The user confirmed that their need was met. +Verdict: yes + +# Example 2 (Request Not Fulfilled) +## Input + + + { + "name": "get_order_status", + "description": "Retrieves the status and tracking information for a given order ID.", + "parameters": [ + { + "type": "string", + "name": "order_id", + "description": "The unique identifier for the customer's order." + } + ] + }, + { + "name": "process_return", + "description": "Initiates a return process for a given order ID and generates a shipping label.", + "parameters": [ + { + "type": "string", + "name": "order_id", + "description": "The unique identifier for the order to be returned." + } + ] + } + + + + I'd like to return the shoes I bought last week. The order number is #54321. + + + + +Agent: I can help you with that. Can you confirm your shipping address? +User: Yes, it's 123 Main St, Anytown. +Agent: Thank you. Unfortunately, our return system is experiencing technical difficulties right now. I can't generate a return label. I can try again in a few hours. +User: Oh. Okay, I guess just let me know. + + + +* The agent fulfilled the user's primary request. + + +## Output +Property: The agent fulfilled the user's primary request. +Evidence: User: "I'd like to return the shoes I bought last week." Agent: "Unfortunately, our return system is experiencing technical difficulties right now. I can't generate a return label." +Rationale: The user's primary request was to initiate a return for their shoes. The agent was unable to complete this action due to a system issue. The conversation ended without the user's request being fulfilled. +Verdict: no + +# Your Turn +## Input + + + {{tool_declarations}} + + + + {{user_input}} + + + + +{{model_response}} + + + +{{decomposed_rubric}} + + +## Output \ No newline at end of file diff --git a/contributing/samples/gepa/run_experiment.py b/contributing/samples/gepa/run_experiment.py new file mode 100644 index 00000000..0e0637d3 --- /dev/null +++ b/contributing/samples/gepa/run_experiment.py @@ -0,0 +1,116 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Runs a GEPA experiment on Tau-Bench.""" + +from collections.abc import Sequence +import dataclasses +from datetime import datetime +import json +import logging +import os + +from absl import app +from absl import flags +import experiment +from google.genai import types + +_OUTPUT_DIR = flags.DEFINE_string('output_dir', None, '') +_EVAL_SET_SIZE = flags.DEFINE_integer('eval_set_size', None, '') +_MAX_METRIC_CALLS = flags.DEFINE_integer('max_metric_calls', 500, '') +_NUM_TEST_RECORDS = flags.DEFINE_integer('num_test_records', None, '') +_NUM_EVAL_TRIALS = flags.DEFINE_integer('num_eval_trials', 4, '') +_MAX_CONCURRENCY = flags.DEFINE_integer('max_concurrency', 8, '') +_EVAL_MODE = flags.DEFINE_bool('eval_mode', False, '') +_USE_RATER = flags.DEFINE_bool('use_rater', False, '') +_TRAIN_BATCH_SIZE = flags.DEFINE_integer('train_batch_size', 3, '') + + +def main(argv: Sequence[str]) -> None: + if len(argv) > 1: + raise app.UsageError('Too many command-line arguments.') + + # Get a list of all existing loggers + # logging.root.manager.loggerDict contains all named loggers + # logging.getLogger(name) retrieves the logger object + loggers = [ + logging.getLogger(name) for name in logging.root.manager.loggerDict + ] + + # Iterate through the loggers and set their level to WARNING + for logger in loggers: + logger.setLevel(logging.WARNING) + + types.logger.addFilter(experiment.FilterInferenceWarnings()) + if not _OUTPUT_DIR.value: + raise ValueError('outptut dir must be specified') + output_dir = os.path.join( + _OUTPUT_DIR.value, datetime.now().strftime('%Y%m%d%H%M%S%f') + ) + os.makedirs(output_dir) + logging.info('Writing to output_dir=%s', output_dir) + config = experiment.ExperimentConfig( + tau_bench_env='retail', + agent_model='gemini-2.5-flash', + agent_model_provider='vertex_ai', + user_model='gemini-2.5-flash', + user_model_provider='vertex_ai', + max_concurrency=_MAX_CONCURRENCY.value, + num_eval_trials=_NUM_EVAL_TRIALS.value, + rnd_seed=42, + max_metric_calls=_MAX_METRIC_CALLS.value, + reflection_model='gemini-2.5-pro', + reflection_minibatch_size=_TRAIN_BATCH_SIZE.value, + use_rater=_USE_RATER.value, + feedback_dataset=experiment.Dataset(split='train'), + pareto_dataset=experiment.Dataset( + split='dev', max_size=_EVAL_SET_SIZE.value + ), + eval_dataset=experiment.Dataset( + split='test', max_size=_NUM_TEST_RECORDS.value + ), + ) + json.dump( + dataclasses.asdict(config), + open(os.path.join(output_dir, 'config.json'), 'w'), + ) + logging.info('Using config=%s', config) + + if _EVAL_MODE.value: + return experiment.run_eval( + output_dir=output_dir, + instructions=experiment.SEED_SYSTEM_INSTRUCTION, + config=config, + ) + + results = experiment.run_gepa( + config=config, + seed_instructions=experiment.SEED_SYSTEM_INSTRUCTION, + output_dir=output_dir, + ) + print(list(enumerate(results.val_aggregate_scores))) + + eval_dir = os.path.join( + output_dir, 'evals', datetime.now().strftime('%Y%m%d%H%M%S%f') + ) + os.makedirs(eval_dir) + experiment.run_eval( + output_dir=eval_dir, + instructions=results.best_candidate['system_instruction'], + config=config, + ) + + +if __name__ == '__main__': + app.run(main)