From 154b03275eeda44d6d2aaf1861d2fe9f2d7b7dd5 Mon Sep 17 00:00:00 2001 From: Abhishek Acharya Date: Tue, 20 Oct 2026 03:37:10 +1100 Subject: [PATCH] fix(anthropic): keep the client default when no batch timeout is set The batch entry points fall back to 600s when the caller sets no timeout. Forwarding that replaced the HTTP client's own default, which has a 5s connect timeout and honors an explicitly configured litellm.request_timeout. Only forward a timeout the caller asked for (timeout, or the legacy request_timeout) and pass None otherwise, so the client default applies as before. --- litellm/batches/main.py | 22 +++++++++- litellm/llms/anthropic/batches/handler.py | 4 +- .../llms/anthropic/batches/test_handler.py | 43 ++++++++++++++++--- 3 files changed, 59 insertions(+), 10 deletions(-) diff --git a/litellm/batches/main.py b/litellm/batches/main.py index 9b5630d81ad..b65e2c498a2 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -13,7 +13,7 @@ https://platform.openai.com/docs/api-reference/batch import asyncio import contextvars import os -from collections.abc import Coroutine +from collections.abc import Coroutine, Mapping from functools import partial from typing import Any, Final, Literal, cast @@ -104,6 +104,24 @@ def _resolve_timeout( return float(timeout) +def _caller_timeout( + optional_params: GenericLiteLLMParams, + resolved_timeout: float | httpx.Timeout, +) -> float | httpx.Timeout | None: + """ + ``resolved_timeout`` when the caller asked for a timeout, else None. + + The resolution above falls back to 600s when nothing was set. Sending that on every + request would replace the HTTP client's own default (a 5s connect timeout, and an + explicitly configured ``litellm.request_timeout``), so None keeps that default. + """ + # ``request_timeout`` is the legacy spelling; GenericLiteLLMParams keeps it as an extra field + extra_params: Final[Mapping[str, object]] = optional_params.model_extra or {} + if optional_params.timeout or extra_params.get("request_timeout"): + return resolved_timeout + return None + + @client async def acreate_batch( completion_window: Literal["24h"], @@ -538,7 +556,7 @@ def _handle_retrieve_batch_providers_without_provider_config( batch_id=batch_id, api_base=api_base, api_key=api_key, - timeout=timeout, + timeout=_caller_timeout(optional_params, timeout), max_retries=optional_params.max_retries, litellm_params=batch_params, ) diff --git a/litellm/llms/anthropic/batches/handler.py b/litellm/llms/anthropic/batches/handler.py index e93bd14ecef..794f8123860 100644 --- a/litellm/llms/anthropic/batches/handler.py +++ b/litellm/llms/anthropic/batches/handler.py @@ -39,7 +39,7 @@ class AnthropicBatchesHandler: batch_id: str, api_base: str | None, api_key: str | None, - timeout: float | httpx.Timeout, + timeout: float | httpx.Timeout | None, max_retries: int | None, logging_obj: LiteLLMLoggingObj | None = None, litellm_params: dict | None = None, # mutable-ok: handed straight to validate_environment @@ -128,7 +128,7 @@ class AnthropicBatchesHandler: batch_id: str, api_base: str | None, api_key: str | None, - timeout: float | httpx.Timeout, + timeout: float | httpx.Timeout | None, max_retries: int | None, logging_obj: LiteLLMLoggingObj | None = None, litellm_params: dict | None = None, # mutable-ok: handed straight to validate_environment diff --git a/tests/unit/llms/anthropic/batches/test_handler.py b/tests/unit/llms/anthropic/batches/test_handler.py index 76c663ba11f..688b10e0993 100644 --- a/tests/unit/llms/anthropic/batches/test_handler.py +++ b/tests/unit/llms/anthropic/batches/test_handler.py @@ -22,6 +22,7 @@ import httpx import pytest import litellm +from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS from litellm.llms.anthropic.batches.handler import AnthropicBatchesHandler from litellm.types.utils import LiteLLMBatch @@ -359,6 +360,17 @@ async def test_aretrieve_batch_wif_exchange_does_not_block_event_loop(handler, p ANTHROPIC_BATCH_URL = "https://api.anthropic.com/v1/messages/batches/msgbatch_abc" SEVEN_SECONDS = {"connect": 7.0, "read": 7.0, "write": 7.0, "pool": 7.0} +# The cached client's own default: 600s, with a 5s connect timeout +CLIENT_DEFAULT = {"connect": 5.0, "read": 600.0, "write": 600.0, "pool": 600.0} + +TIMEOUT_CASES = [ + pytest.param({"timeout": 7.0}, None, SEVEN_SECONDS, id="callers-timeout"), + pytest.param({"request_timeout": 7.0}, None, SEVEN_SECONDS, id="legacy-request-timeout"), + pytest.param({}, None, CLIENT_DEFAULT, id="no-timeout-keeps-client-default"), + pytest.param( + {}, 30.0, {"connect": 5.0, "read": 30.0, "write": 30.0, "pool": 30.0}, id="no-timeout-uses-configured-timeout" + ), +] @pytest.fixture @@ -374,22 +386,41 @@ def anthropic_batch_route(respx_mock, monkeypatch): return respx_mock.get(ANTHROPIC_BATCH_URL).mock(return_value=httpx.Response(200, json=in_progress)) +def _configure_request_timeout(monkeypatch, configured_request_timeout): + if configured_request_timeout is None: + # No global timeout configured, so a REQUEST_TIMEOUT in the environment cannot move the default + monkeypatch.setattr(litellm, "request_timeout", DEFAULT_REQUEST_TIMEOUT_SECONDS) + monkeypatch.setattr(litellm, "request_timeout_explicitly_set", False) + else: + monkeypatch.setattr(litellm, "request_timeout", configured_request_timeout) + + +@pytest.mark.parametrize("kwargs, configured_request_timeout, expected_timeout", TIMEOUT_CASES) @pytest.mark.respx(assert_all_called=True) -def test_retrieve_batch_sends_the_callers_timeout(anthropic_batch_route): +def test_retrieve_batch_sends_the_callers_timeout_or_keeps_the_client_default( + anthropic_batch_route, monkeypatch, kwargs, configured_request_timeout, expected_timeout +): + _configure_request_timeout(monkeypatch, configured_request_timeout) + batch = litellm.retrieve_batch( - batch_id="msgbatch_abc", custom_llm_provider="anthropic", api_key="sk-ant-test", timeout=7.0 + batch_id="msgbatch_abc", custom_llm_provider="anthropic", api_key="sk-ant-test", **kwargs ) assert (batch.id, batch.status) == ("msgbatch_abc", "in_progress") - assert anthropic_batch_route.calls[0].request.extensions["timeout"] == SEVEN_SECONDS + assert anthropic_batch_route.calls[0].request.extensions["timeout"] == expected_timeout +@pytest.mark.parametrize("kwargs, configured_request_timeout, expected_timeout", TIMEOUT_CASES) @pytest.mark.asyncio @pytest.mark.respx(assert_all_called=True) -async def test_aretrieve_batch_sends_the_callers_timeout(anthropic_batch_route): +async def test_aretrieve_batch_sends_the_callers_timeout_or_keeps_the_client_default( + anthropic_batch_route, monkeypatch, kwargs, configured_request_timeout, expected_timeout +): + _configure_request_timeout(monkeypatch, configured_request_timeout) + batch = await litellm.aretrieve_batch( - batch_id="msgbatch_abc", custom_llm_provider="anthropic", api_key="sk-ant-test", timeout=7.0 + batch_id="msgbatch_abc", custom_llm_provider="anthropic", api_key="sk-ant-test", **kwargs ) assert (batch.id, batch.status) == ("msgbatch_abc", "in_progress") - assert anthropic_batch_route.calls[0].request.extensions["timeout"] == SEVEN_SECONDS + assert anthropic_batch_route.calls[0].request.extensions["timeout"] == expected_timeout