fix: Prevent retry_on_errors from retrying asyncio.CancelledError

The retry_on_errors decorator in mcp_session_manager.py now catches asyncio.CancelledError and re-raises it immediately, ensuring that cancellation requests are not suppressed or retried

Close #4009

Co-authored-by: George Weale <gweale@google.com>
PiperOrigin-RevId: 852468382
This commit is contained in:
George Weale
2026-01-05 14:58:26 -08:00
committed by Copybara-Service
parent 6f2c70fc33
commit 30d3411d60
2 changed files with 105 additions and 15 deletions
@@ -15,6 +15,7 @@
from __future__ import annotations from __future__ import annotations
import asyncio import asyncio
from collections import deque
from contextlib import AsyncExitStack from contextlib import AsyncExitStack
from datetime import timedelta from datetime import timedelta
import functools import functools
@@ -30,7 +31,6 @@ from typing import runtime_checkable
from typing import TextIO from typing import TextIO
from typing import Union from typing import Union
import anyio
from mcp import ClientSession from mcp import ClientSession
from mcp import StdioServerParameters from mcp import StdioServerParameters
from mcp.client.sse import sse_client from mcp.client.sse import sse_client
@@ -44,6 +44,30 @@ from pydantic import ConfigDict
logger = logging.getLogger('google_adk.' + __name__) logger = logging.getLogger('google_adk.' + __name__)
def _has_cancelled_error_context(exc: BaseException) -> bool:
"""Returns True if `exc` is/was caused by `asyncio.CancelledError`.
Cancellation can be translated into other exceptions during teardown (e.g.
connection errors) while still retaining the original cancellation in an
exception's context chain.
"""
seen: set[int] = set()
queue = deque([exc])
while queue:
current = queue.popleft()
if id(current) in seen:
continue
seen.add(id(current))
if isinstance(current, asyncio.CancelledError):
return True
if current.__cause__ is not None:
queue.append(current.__cause__)
if current.__context__ is not None:
queue.append(current.__context__)
return False
class StdioConnectionParams(BaseModel): class StdioConnectionParams(BaseModel):
"""Parameters for the MCP Stdio connection. """Parameters for the MCP Stdio connection.
@@ -119,6 +143,10 @@ def retry_on_errors(func):
action once. The create_session method will handle creating a new session action once. The create_session method will handle creating a new session
if the old one was disconnected. if the old one was disconnected.
Cancellation is not retried and must be allowed to propagate. In async
runtimes, cancellation may surface as `asyncio.CancelledError` or as another
exception while the task is cancelling.
Args: Args:
func: The function to decorate. func: The function to decorate.
@@ -131,6 +159,13 @@ def retry_on_errors(func):
try: try:
return await func(self, *args, **kwargs) return await func(self, *args, **kwargs)
except Exception as e: except Exception as e:
task = asyncio.current_task()
if task is not None:
cancelling = getattr(task, 'cancelling', None)
if cancelling is not None and cancelling() > 0:
raise
if _has_cancelled_error_context(e):
raise
# If an error is thrown, we will retry the function to reconnect to the # If an error is thrown, we will retry the function to reconnect to the
# server. create_session will handle detecting and replacing disconnected # server. create_session will handle detecting and replacing disconnected
# sessions. # sessions.
@@ -391,7 +391,8 @@ class TestMCPSessionManager:
assert "Close error 1" in error_output assert "Close error 1" in error_output
def test_retry_on_errors_decorator(): @pytest.mark.asyncio
async def test_retry_on_errors_decorator():
"""Test the retry_on_errors decorator.""" """Test the retry_on_errors decorator."""
call_count = 0 call_count = 0
@@ -401,23 +402,77 @@ def test_retry_on_errors_decorator():
nonlocal call_count nonlocal call_count
call_count += 1 call_count += 1
if call_count == 1: if call_count == 1:
import anyio raise ConnectionError("Resource closed")
raise anyio.ClosedResourceError("Resource closed")
return "success" return "success"
@pytest.mark.asyncio mock_self = Mock()
async def test_retry(): result = await mock_function(mock_self)
assert result == "success"
assert call_count == 2 # First call fails, second succeeds
@pytest.mark.asyncio
async def test_retry_on_errors_decorator_does_not_retry_cancelled_error():
"""Test the retry_on_errors decorator does not retry cancellation."""
call_count = 0
@retry_on_errors
async def mock_function(self):
nonlocal call_count nonlocal call_count
call_count = 0 call_count += 1
raise asyncio.CancelledError()
mock_self = Mock() mock_self = Mock()
result = await mock_function(mock_self) with pytest.raises(asyncio.CancelledError):
await mock_function(mock_self)
assert result == "success" assert call_count == 1
assert call_count == 2 # First call fails, second succeeds
# Run the test
import asyncio
asyncio.run(test_retry()) @pytest.mark.asyncio
async def test_retry_on_errors_decorator_does_not_retry_when_task_is_cancelling():
"""Test the retry_on_errors decorator does not retry when cancelling."""
call_count = 0
@retry_on_errors
async def mock_function(self):
nonlocal call_count
call_count += 1
raise ConnectionError("Resource closed")
class _MockTask:
def cancelling(self):
return 1
mock_self = Mock()
with patch.object(asyncio, "current_task", return_value=_MockTask()):
with pytest.raises(ConnectionError):
await mock_function(mock_self)
assert call_count == 1
@pytest.mark.asyncio
async def test_retry_on_errors_decorator_does_not_retry_exception_from_cancel():
"""Test the retry_on_errors decorator does not retry exceptions on cancel."""
call_count = 0
@retry_on_errors
async def mock_function(self):
nonlocal call_count
call_count += 1
try:
raise asyncio.CancelledError()
except asyncio.CancelledError:
raise ConnectionError("Resource closed")
mock_self = Mock()
with pytest.raises(ConnectionError):
await mock_function(mock_self)
assert call_count == 1