mirror of
https://github.com/encounter/adk-python.git
synced 2026-07-09 18:19:28 -07:00
feat: Bigquery ADK support for search catalog tool
Merge https://github.com/google/adk-python/pull/4171 **Problem:** The BigQuery ADK tools currently lack the ability to search for and discover BigQuery assets using the Dataplex Catalog. Users cannot leverage Dataplex's search capabilities within the ADK to find relevant data assets before querying them. **Solution:** This PR integrates a new search_catalog_tool into the BigQuery ADK. This tool utilizes the dataplex catalog client library to interact with the Dataplex API, allowing users to search the catalog. **Unit Tests:** - [x] I have added or updated unit tests for my change. - [x] All unit tests pass locally. Added the screenshots of the manual adk web UI tests - https://docs.google.com/document/d/1c_lMW7NYGKuLAvPFmSkLehbqySeNyXQIhzQlvo3ixmQ/edit?usp=sharing ### Checklist - [x] I have read the [CONTRIBUTING.md](https://github.com/google/adk-python/blob/main/CONTRIBUTING.md) document. - [x] I have performed a self-review of my own code. - [x] I have commented my code, particularly in hard-to-understand areas. - [x] I have added tests that prove my fix is effective or that my feature works. - [x] New and existing unit tests pass locally with my changes. - [x] I have manually tested my changes end-to-end. - [x] Any dependent changes have been merged and published in downstream modules. COPYBARA_INTEGRATE_REVIEW=https://github.com/google/adk-python/pull/4171 from sahaajaaa:sahaajaaa-bq-adk 3dbbaa4f909cb25259e8e7d73a00a58fbe9c2f09 PiperOrigin-RevId: 872951141
This commit is contained in:
committed by
Copybara-Service
parent
a39ca946d6
commit
bef3f117b4
@@ -19,6 +19,10 @@ from ...features import FeatureName
|
||||
from .._google_credentials import BaseGoogleCredentialsConfig
|
||||
|
||||
BIGQUERY_TOKEN_CACHE_KEY = "bigquery_token_cache"
|
||||
BIGQUERY_SCOPES = [
|
||||
"https://www.googleapis.com/auth/bigquery",
|
||||
"https://www.googleapis.com/auth/dataplex",
|
||||
]
|
||||
BIGQUERY_DEFAULT_SCOPE = ["https://www.googleapis.com/auth/bigquery"]
|
||||
|
||||
|
||||
@@ -34,8 +38,8 @@ class BigQueryCredentialsConfig(BaseGoogleCredentialsConfig):
|
||||
super().__post_init__()
|
||||
|
||||
if not self.scopes:
|
||||
self.scopes = BIGQUERY_DEFAULT_SCOPE
|
||||
|
||||
self.scopes = BIGQUERY_SCOPES
|
||||
# Set the token cache key
|
||||
self._token_cache_key = BIGQUERY_TOKEN_CACHE_KEY
|
||||
|
||||
return self
|
||||
|
||||
@@ -24,6 +24,7 @@ from typing_extensions import override
|
||||
from . import data_insights_tool
|
||||
from . import metadata_tool
|
||||
from . import query_tool
|
||||
from . import search_tool
|
||||
from ...features import experimental
|
||||
from ...features import FeatureName
|
||||
from ...tools.base_tool import BaseTool
|
||||
@@ -87,6 +88,7 @@ class BigQueryToolset(BaseToolset):
|
||||
query_tool.analyze_contribution,
|
||||
query_tool.detect_anomalies,
|
||||
data_insights_tool.ask_data_insights,
|
||||
search_tool.search_catalog,
|
||||
]
|
||||
]
|
||||
|
||||
|
||||
@@ -14,19 +14,22 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Union
|
||||
|
||||
import google.api_core.client_info
|
||||
from google.api_core.gapic_v1 import client_info as gapic_client_info
|
||||
from google.auth.credentials import Credentials
|
||||
from google.cloud import bigquery
|
||||
from google.cloud import dataplex_v1
|
||||
|
||||
from ... import version
|
||||
|
||||
USER_AGENT = f"adk-bigquery-tool google-adk/{version.__version__}"
|
||||
|
||||
|
||||
from typing import List
|
||||
from typing import Union
|
||||
USER_AGENT_BASE = f"google-adk/{version.__version__}"
|
||||
BQ_USER_AGENT = f"adk-bigquery-tool {USER_AGENT_BASE}"
|
||||
DP_USER_AGENT = f"adk-dataplex-tool {USER_AGENT_BASE}"
|
||||
USER_AGENT = BQ_USER_AGENT
|
||||
|
||||
|
||||
def get_bigquery_client(
|
||||
@@ -48,7 +51,7 @@ def get_bigquery_client(
|
||||
A BigQuery client.
|
||||
"""
|
||||
|
||||
user_agents = [USER_AGENT]
|
||||
user_agents = [BQ_USER_AGENT]
|
||||
if user_agent:
|
||||
if isinstance(user_agent, str):
|
||||
user_agents.append(user_agent)
|
||||
@@ -67,3 +70,33 @@ def get_bigquery_client(
|
||||
)
|
||||
|
||||
return bigquery_client
|
||||
|
||||
|
||||
def get_dataplex_catalog_client(
|
||||
*,
|
||||
credentials: Credentials,
|
||||
user_agent: Optional[Union[str, List[str]]] = None,
|
||||
) -> dataplex_v1.CatalogServiceClient:
|
||||
"""Get a Dataplex CatalogServiceClient with minimal necessary arguments.
|
||||
|
||||
Args:
|
||||
credentials: The credentials to use for the request.
|
||||
user_agent: Additional user agent string(s) to append.
|
||||
|
||||
Returns:
|
||||
A Dataplex Client.
|
||||
"""
|
||||
|
||||
user_agents = [DP_USER_AGENT]
|
||||
if user_agent:
|
||||
if isinstance(user_agent, str):
|
||||
user_agents.append(user_agent)
|
||||
else:
|
||||
user_agents.extend([ua for ua in user_agent if ua])
|
||||
|
||||
client_info = gapic_client_info.ClientInfo(user_agent=" ".join(user_agents))
|
||||
|
||||
return dataplex_v1.CatalogServiceClient(
|
||||
credentials=credentials,
|
||||
client_info=client_info,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from google.api_core import exceptions as api_exceptions
|
||||
from google.auth.credentials import Credentials
|
||||
from google.cloud import dataplex_v1
|
||||
|
||||
from . import client
|
||||
from .config import BigQueryToolConfig
|
||||
|
||||
|
||||
def _construct_search_query_helper(
|
||||
predicate: str, operator: str, items: list[str]
|
||||
) -> str:
|
||||
"""Constructs a search query part for a specific predicate and items."""
|
||||
if not items:
|
||||
return ""
|
||||
|
||||
clauses = [f'{predicate}{operator}"{item}"' for item in items]
|
||||
return "(" + " OR ".join(clauses) + ")" if len(items) > 1 else clauses[0]
|
||||
|
||||
|
||||
def search_catalog(
|
||||
prompt: str,
|
||||
project_id: str,
|
||||
*,
|
||||
credentials: Credentials,
|
||||
settings: BigQueryToolConfig,
|
||||
location: str | None = None,
|
||||
page_size: int = 10,
|
||||
project_ids_filter: list[str] | None = None,
|
||||
dataset_ids_filter: list[str] | None = None,
|
||||
types_filter: list[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Searches for BigQuery assets within Dataplex.
|
||||
|
||||
Args:
|
||||
prompt: The base search query (natural language or keywords).
|
||||
project_id: The Google Cloud project ID to scope the search.
|
||||
credentials: Credentials for the request.
|
||||
settings: BigQuery tool settings.
|
||||
location: The Dataplex location to use.
|
||||
page_size: Maximum number of results.
|
||||
project_ids_filter: Specific project IDs to include in the search results.
|
||||
If None, defaults to the scoping project_id.
|
||||
dataset_ids_filter: BigQuery dataset IDs to filter by.
|
||||
types_filter: Entry types to filter by (e.g., BigQueryEntryType.TABLE,
|
||||
BigQueryEntryType.DATASET).
|
||||
|
||||
Returns:
|
||||
Search results or error. The "results" list contains items with:
|
||||
- name: The Dataplex Entry name (e.g.,
|
||||
"projects/p/locations/l/entryGroups/g/entries/e").
|
||||
- linked_resource: The underlying BigQuery resource name (e.g.,
|
||||
"//bigquery.googleapis.com/projects/p/datasets/d/tables/t").
|
||||
- display_name, entry_type, description, location, update_time.
|
||||
|
||||
Examples:
|
||||
Search for tables related to customer data:
|
||||
|
||||
>>> search_catalog(
|
||||
... prompt="Search for tables related to customer data",
|
||||
... project_id="my-project",
|
||||
... credentials=creds,
|
||||
... settings=settings
|
||||
... )
|
||||
{
|
||||
"status": "SUCCESS",
|
||||
"results": [
|
||||
{
|
||||
"name":
|
||||
"projects/my-project/locations/us/entryGroups/@bigquery/entries/entry-id",
|
||||
"display_name": "customer_table",
|
||||
"entry_type":
|
||||
"projects/p/locations/l/entryTypes/bigquery-table",
|
||||
"linked_resource":
|
||||
"//bigquery.googleapis.com/projects/my-project/datasets/d/tables/customer_table",
|
||||
"description": "Table containing customer details.",
|
||||
"location": "us",
|
||||
"update_time": "2024-01-01 12:00:00+00:00"
|
||||
}
|
||||
]
|
||||
}
|
||||
"""
|
||||
|
||||
try:
|
||||
if not project_id:
|
||||
return {
|
||||
"status": "ERROR",
|
||||
"error_details": "project_id must be provided.",
|
||||
}
|
||||
|
||||
with client.get_dataplex_catalog_client(
|
||||
credentials=credentials,
|
||||
user_agent=[settings.application_name, "search_catalog"],
|
||||
) as dataplex_client:
|
||||
query_parts = []
|
||||
if prompt:
|
||||
query_parts.append(f"({prompt})")
|
||||
|
||||
# Filter by project IDs
|
||||
projects_to_filter = (
|
||||
project_ids_filter if project_ids_filter else [project_id]
|
||||
)
|
||||
if projects_to_filter:
|
||||
query_parts.append(
|
||||
_construct_search_query_helper("projectid", "=", projects_to_filter)
|
||||
)
|
||||
|
||||
# Filter by dataset IDs
|
||||
if dataset_ids_filter:
|
||||
dataset_resource_filters = []
|
||||
for pid in projects_to_filter:
|
||||
for did in dataset_ids_filter:
|
||||
dataset_resource_filters.append(
|
||||
f'linked_resource:"//bigquery.googleapis.com/projects/{pid}/datasets/{did}/*"'
|
||||
)
|
||||
if dataset_resource_filters:
|
||||
query_parts.append(f"({' OR '.join(dataset_resource_filters)})")
|
||||
# Filter by entry types
|
||||
if types_filter:
|
||||
query_parts.append(
|
||||
_construct_search_query_helper("type", "=", types_filter)
|
||||
)
|
||||
|
||||
# Always scope to BigQuery system
|
||||
query_parts.append("system=BIGQUERY")
|
||||
|
||||
full_query = " AND ".join(filter(None, query_parts))
|
||||
|
||||
search_location = location or settings.location or "global"
|
||||
search_scope = f"projects/{project_id}/locations/{search_location}"
|
||||
|
||||
request = dataplex_v1.SearchEntriesRequest(
|
||||
name=search_scope,
|
||||
query=full_query,
|
||||
page_size=page_size,
|
||||
semantic_search=True,
|
||||
)
|
||||
|
||||
response = dataplex_client.search_entries(request=request)
|
||||
|
||||
results = []
|
||||
for result in response.results:
|
||||
entry = result.dataplex_entry
|
||||
source = entry.entry_source
|
||||
results.append({
|
||||
"name": entry.name,
|
||||
"display_name": source.display_name or "",
|
||||
"entry_type": entry.entry_type,
|
||||
"update_time": str(entry.update_time),
|
||||
"linked_resource": source.resource or "",
|
||||
"description": source.description or "",
|
||||
"location": source.location or "",
|
||||
})
|
||||
return {"status": "SUCCESS", "results": results}
|
||||
|
||||
except api_exceptions.GoogleAPICallError as e:
|
||||
logging.exception("search_catalog tool: API call failed")
|
||||
return {"status": "ERROR", "error_details": f"Dataplex API Error: {e}"}
|
||||
except Exception as e:
|
||||
logging.exception("search_catalog tool: Unexpected error")
|
||||
return {"status": "ERROR", "error_details": repr(e)}
|
||||
Reference in New Issue
Block a user