From cdd9b6147f3814b040c2c063f3af504fc0c9ddfa Mon Sep 17 00:00:00 2001 From: Mubashir Osmani Date: Tue, 22 Sep 2026 21:36:57 +0000 Subject: [PATCH] refactor(batches): route anthropic list and cancel through the provider-config http handler Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/batches/main.py | 40 ++- litellm/llms/anthropic/batches/handler.py | 219 --------------- .../llms/anthropic/batches/transformation.py | 35 ++- .../llms/base_llm/batches/transformation.py | 94 +++++++ litellm/llms/custom_httpx/llm_http_handler.py | 255 ++++++++++++++++++ .../llms/anthropic/batches/test_handler.py | 184 ------------- .../anthropic/batches/test_transformation.py | 12 +- .../custom_httpx/test_llm_http_handler.py | 159 +++++++++++ tests/unit/batches/test_main.py | 12 +- 9 files changed, 579 insertions(+), 431 deletions(-) delete mode 100644 tests/test_litellm/llms/anthropic/batches/test_handler.py diff --git a/litellm/batches/main.py b/litellm/batches/main.py index 44729fa2c2e..7c2f54ba257 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -826,14 +826,26 @@ def list_batches( optional_params.api_key or litellm.api_key or litellm.azure_key or get_secret_str("ANTHROPIC_API_KEY") ) - response = anthropic_batches_instance.list_batches( - _is_async=_is_async, - after=after, - limit=limit, + provider_config = ProviderConfigManager.get_provider_batches_config( + model="", + provider=LlmProviders.ANTHROPIC, + ) + litellm_logging_obj: Final = cast( # cast-ok: the client decorator injects litellm_logging_obj into kwargs + LiteLLMLoggingObj, kwargs.get("litellm_logging_obj", None) + ) + response = base_llm_http_handler.list_batches( + litellm_params=litellm_params, + provider_config=provider_config, + headers=extra_headers or {}, # mutable-ok: validate_environment fills provider headers in place api_base=api_base, api_key=api_key, + logging_obj=litellm_logging_obj, + after=after, + limit=limit, + _is_async=_is_async, + client=kwargs.get("client"), timeout=timeout, - max_retries=optional_params.max_retries, + model="", ) else: raise litellm.exceptions.BadRequestError( @@ -1036,13 +1048,25 @@ def cancel_batch( optional_params.api_key or litellm.api_key or litellm.azure_key or get_secret_str("ANTHROPIC_API_KEY") ) - response = anthropic_batches_instance.cancel_batch( - _is_async=_is_async, + provider_config = ProviderConfigManager.get_provider_batches_config( + model=model or "", + provider=LlmProviders.ANTHROPIC, + ) + litellm_logging_obj: Final = cast( # cast-ok: callers pass litellm_logging_obj through kwargs + LiteLLMLoggingObj, kwargs.get("litellm_logging_obj", None) + ) + response = base_llm_http_handler.cancel_batch( batch_id=batch_id, + litellm_params=litellm_params, + provider_config=provider_config, + headers=extra_headers or {}, # mutable-ok: validate_environment fills provider headers in place api_base=api_base, api_key=api_key, + logging_obj=litellm_logging_obj, + _is_async=_is_async, + client=kwargs.get("client"), timeout=timeout, - max_retries=optional_params.max_retries, + model=model or "", ) elif custom_llm_provider == "bedrock": response = BedrockBatchesHandler.cancel_batch( diff --git a/litellm/llms/anthropic/batches/handler.py b/litellm/llms/anthropic/batches/handler.py index fe9afa6ce46..f418e7e08be 100644 --- a/litellm/llms/anthropic/batches/handler.py +++ b/litellm/llms/anthropic/batches/handler.py @@ -18,11 +18,6 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any -_EMPTY_PARAMS: Final[ - dict -] = {} # mutable-ok: shared empty mapping for signatures that declare dict; every callee here only reads it -_EMPTY_MESSAGES: Final[list] = [] # mutable-ok: shared empty list for the messages signature; callees only read it - from ..common_utils import AnthropicModelInfo from .transformation import AnthropicBatchesConfig @@ -33,30 +28,12 @@ class AnthropicBatchesHandler: Supports: - retrieve_batch() - Retrieve batch status and information - - list_batches() - List batches - - cancel_batch() - Cancel a batch """ def __init__(self): self.anthropic_model_info = AnthropicModelInfo() self.provider_config = AnthropicBatchesConfig() - @staticmethod - def _default_logging_obj(call_type: str, call_id: str) -> LiteLLMLoggingObj: - from litellm.litellm_core_utils.litellm_logging import ( - Logging as LiteLLMLoggingObjClass, - ) - - return LiteLLMLoggingObjClass( - model="anthropic/unknown", - messages=_EMPTY_MESSAGES, - stream=False, - call_type=call_type, - start_time=None, - litellm_call_id=call_id, - function_id=call_type, - ) - async def aretrieve_batch( self, batch_id: str, @@ -189,199 +166,3 @@ class AnthropicBatchesHandler: logging_obj=logging_obj, ) ) - - async def alist_batches( - self, - after: str | None, - limit: int | None, - api_base: str | None, - api_key: str | None, - timeout: float | httpx.Timeout, - max_retries: int | None, - logging_obj: LiteLLMLoggingObj | None = None, - ) -> dict: - """ - Async: List batches from Anthropic. - """ - resolved_api_base: Final = api_base or self.anthropic_model_info.get_api_base(api_base) - resolved_api_key: Final = api_key or self.anthropic_model_info.get_api_key() - - if not resolved_api_key: - raise ValueError("Missing Anthropic API Key") - - resolved_logging_obj: Final = logging_obj or self._default_logging_obj( - call_type="batch_list", call_id="batch_list" - ) - - list_url: Final = self.provider_config.get_list_batches_url( - api_base=resolved_api_base, - after=after, - limit=limit, - optional_params=_EMPTY_PARAMS, - litellm_params=_EMPTY_PARAMS, - ) - - headers: Final = self.provider_config.validate_environment( - headers={}, # mutable-ok: validate_environment fills this mapping in place - model="", - messages=_EMPTY_MESSAGES, - optional_params=_EMPTY_PARAMS, - litellm_params=_EMPTY_PARAMS, - api_key=resolved_api_key, - api_base=resolved_api_base, - ) - - resolved_logging_obj.pre_call( - input="", - api_key=resolved_api_key, - additional_args={ # mutable-ok: pre_call logging contract takes a plain dict - "api_base": list_url, - "headers": headers, - "complete_input_dict": _EMPTY_PARAMS, - }, - ) - async_client: Final = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC) - response: Final = await async_client.get(url=list_url, headers=headers, timeout=timeout) - if response.status_code >= 400: - raise self.provider_config.get_error_class( - error_message=response.text, - status_code=response.status_code, - headers=response.headers, - ) - - return self.provider_config.transform_list_batches_response(raw_response=response) - - def list_batches( - self, - _is_async: bool, - after: str | None, - limit: int | None, - api_base: str | None, - api_key: str | None, - timeout: float | httpx.Timeout, - max_retries: int | None, - logging_obj: LiteLLMLoggingObj | None = None, - ) -> dict | Coroutine[Any, Any, dict]: - """ - List batches from Anthropic. - """ - if _is_async: - return self.alist_batches( - after=after, - limit=limit, - api_base=api_base, - api_key=api_key, - timeout=timeout, - max_retries=max_retries, - logging_obj=logging_obj, - ) - else: - return asyncio.run( - self.alist_batches( - after=after, - limit=limit, - api_base=api_base, - api_key=api_key, - timeout=timeout, - max_retries=max_retries, - logging_obj=logging_obj, - ) - ) - - async def acancel_batch( - self, - batch_id: str, - api_base: str | None, - api_key: str | None, - timeout: float | httpx.Timeout, - max_retries: int | None, - logging_obj: LiteLLMLoggingObj | None = None, - ) -> LiteLLMBatch: - """ - Async: Cancel a batch on Anthropic. - """ - resolved_api_base: Final = api_base or self.anthropic_model_info.get_api_base(api_base) - resolved_api_key: Final = api_key or self.anthropic_model_info.get_api_key() - - if not resolved_api_key: - raise ValueError("Missing Anthropic API Key") - - resolved_logging_obj: Final = logging_obj or self._default_logging_obj( - call_type="batch_cancel", call_id=f"batch_cancel_{batch_id}" - ) - - cancel_url: Final = self.provider_config.get_cancel_batch_url( - api_base=resolved_api_base, - batch_id=batch_id, - optional_params=_EMPTY_PARAMS, - litellm_params=_EMPTY_PARAMS, - ) - - headers: Final = self.provider_config.validate_environment( - headers={}, # mutable-ok: validate_environment fills this mapping in place - model="", - messages=_EMPTY_MESSAGES, - optional_params=_EMPTY_PARAMS, - litellm_params=_EMPTY_PARAMS, - api_key=resolved_api_key, - api_base=resolved_api_base, - ) - - resolved_logging_obj.pre_call( - input=batch_id, - api_key=resolved_api_key, - additional_args={ # mutable-ok: pre_call logging contract takes a plain dict - "api_base": cancel_url, - "headers": headers, - "complete_input_dict": _EMPTY_PARAMS, - }, - ) - async_client: Final = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC) - response: Final = await async_client.post(url=cancel_url, headers=headers, timeout=timeout) - if response.status_code >= 400: - raise self.provider_config.get_error_class( - error_message=response.text, - status_code=response.status_code, - headers=response.headers, - ) - - return self.provider_config.transform_cancel_batch_response( - model=None, - raw_response=response, - logging_obj=resolved_logging_obj, - litellm_params=_EMPTY_PARAMS, - ) - - def cancel_batch( - self, - _is_async: bool, - batch_id: str, - api_base: str | None, - api_key: str | None, - timeout: float | httpx.Timeout, - max_retries: int | None, - logging_obj: LiteLLMLoggingObj | None = None, - ) -> LiteLLMBatch | Coroutine[Any, Any, LiteLLMBatch]: - """ - Cancel a batch on Anthropic. - """ - if _is_async: - return self.acancel_batch( - batch_id=batch_id, - api_base=api_base, - api_key=api_key, - timeout=timeout, - max_retries=max_retries, - logging_obj=logging_obj, - ) - else: - return asyncio.run( - self.acancel_batch( - batch_id=batch_id, - api_base=api_base, - api_key=api_key, - timeout=timeout, - max_retries=max_retries, - logging_obj=logging_obj, - ) - ) diff --git a/litellm/llms/anthropic/batches/transformation.py b/litellm/llms/anthropic/batches/transformation.py index bdf96795857..409f4552b28 100644 --- a/litellm/llms/anthropic/batches/transformation.py +++ b/litellm/llms/anthropic/batches/transformation.py @@ -138,7 +138,7 @@ def _parse_anthropic_timestamp(ts_str: str | None) -> int | None: dt: Final = datetime.fromisoformat(ts_str.replace("Z", "+00:00")) return int(dt.timestamp()) - except Exception: + except (ValueError, OverflowError, OSError): return None @@ -413,17 +413,19 @@ class AnthropicBatchesConfig(BaseBatchesConfig): response_data: Final[AnthropicMessageBatch] = _ANTHROPIC_MESSAGE_BATCH_ADAPTER.validate_python( raw_response.json() ) - except Exception as e: + except ValueError as e: raise ValueError(f"Failed to parse Anthropic batch response: {e}") return transform_anthropic_message_batch(response_data) def get_list_batches_url( self, api_base: str | None, + api_key: str | None, + model: str, + optional_params: dict, # mutable-ok: batch-config signature contract is a plain dict + litellm_params: dict, # mutable-ok: batch-config signature contract is a plain dict after: str | None, limit: int | None, - optional_params: dict, - litellm_params: dict, ) -> str: """Get the URL for listing Anthropic message batches, with pagination params.""" resolved_api_base: Final = self.anthropic_model_info.get_api_base(api_base) or "https://api.anthropic.com" @@ -439,12 +441,21 @@ class AnthropicBatchesConfig(BaseBatchesConfig): def transform_list_batches_response( self, + model: str | None, raw_response: httpx.Response, - ) -> dict[str, object]: + logging_obj: LoggingClass, + litellm_params: dict, # mutable-ok: batch-config signature contract is a plain dict + ) -> dict[str, object]: # mutable-ok: list responses mirror the dict payload the proxy serializes """Transform the Anthropic batch list response into the OpenAI list shape.""" + if raw_response.status_code >= 400: + raise self.get_error_class( + error_message=raw_response.text, + status_code=raw_response.status_code, + headers=raw_response.headers, + ) response_json: Final = _ANTHROPIC_MESSAGE_BATCH_LIST_ADAPTER.validate_python(raw_response.json()) data: Final = tuple(transform_anthropic_message_batch(b) for b in response_json.get("data", ())) - return { + return { # mutable-ok: list responses mirror the dict payload the proxy serializes "object": "list", "data": data, "first_id": response_json.get("first_id") or (data[0].id if data else None), @@ -455,9 +466,11 @@ class AnthropicBatchesConfig(BaseBatchesConfig): def get_cancel_batch_url( self, api_base: str | None, + api_key: str | None, + model: str, batch_id: str, - optional_params: dict, - litellm_params: dict, + optional_params: dict, # mutable-ok: batch-config signature contract is a plain dict + litellm_params: dict, # mutable-ok: batch-config signature contract is a plain dict ) -> str: """Get the URL for cancelling an Anthropic message batch.""" resolved_api_base: Final = self.anthropic_model_info.get_api_base(api_base) or "https://api.anthropic.com" @@ -469,7 +482,7 @@ class AnthropicBatchesConfig(BaseBatchesConfig): model: str | None, raw_response: httpx.Response, logging_obj: LoggingClass, - litellm_params: dict, + litellm_params: dict, # mutable-ok: batch-config signature contract is a plain dict ) -> LiteLLMBatch: """Transform Anthropic MessageBatch cancel response to LiteLLM format.""" if raw_response.status_code >= 400: @@ -482,7 +495,7 @@ class AnthropicBatchesConfig(BaseBatchesConfig): response_data: Final[AnthropicMessageBatch] = _ANTHROPIC_MESSAGE_BATCH_ADAPTER.validate_python( raw_response.json() ) - except Exception as e: + except ValueError as e: raise ValueError(f"Failed to parse Anthropic batch response: {e}") return transform_anthropic_message_batch(response_data) @@ -534,7 +547,7 @@ class AnthropicBatchesConfig(BaseBatchesConfig): """Transform Anthropic MessageBatch retrieval response to LiteLLM format.""" try: response_data: Final[AnthropicMessageBatch] = raw_response.json() - except Exception as e: + except ValueError as e: raise ValueError(f"Failed to parse Anthropic batch response: {e}") return transform_anthropic_message_batch(response_data) diff --git a/litellm/llms/base_llm/batches/transformation.py b/litellm/llms/base_llm/batches/transformation.py index 34c622d4cf6..b496480cca2 100644 --- a/litellm/llms/base_llm/batches/transformation.py +++ b/litellm/llms/base_llm/batches/transformation.py @@ -193,6 +193,100 @@ class BaseBatchesConfig(ABC): LiteLLM batch object """ + def get_list_batches_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: dict, # mutable-ok: batch-config signature contract is a plain dict + litellm_params: dict, # mutable-ok: batch-config signature contract is a plain dict + after: str | None, + limit: int | None, + ) -> str: + """ + Get the complete URL for listing batches. + + Args: + api_base: Base API URL + api_key: API key + model: Model name + optional_params: Optional parameters + litellm_params: LiteLLM parameters + after: Pagination cursor + limit: Maximum number of batches to return + + Returns: + Complete URL for the list batches request + """ + raise NotImplementedError("Provider does not support listing batches") + + def transform_list_batches_response( + self, + model: str | None, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + litellm_params: dict, # mutable-ok: batch-config signature contract is a plain dict + ) -> dict: # mutable-ok: list responses mirror the dict payload the proxy serializes + """ + Transform provider-specific batch list response to LiteLLM format. + + Args: + model: Model name + raw_response: Raw HTTP response + logging_obj: Logging object + litellm_params: LiteLLM parameters + + Returns: + Batch list in LiteLLM format + """ + raise NotImplementedError("Provider does not support listing batches") + + def get_cancel_batch_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + batch_id: str, + optional_params: dict, # mutable-ok: batch-config signature contract is a plain dict + litellm_params: dict, # mutable-ok: batch-config signature contract is a plain dict + ) -> str: + """ + Get the complete URL for cancelling a batch. + + Args: + api_base: Base API URL + api_key: API key + model: Model name + batch_id: Batch ID to cancel + optional_params: Optional parameters + litellm_params: LiteLLM parameters + + Returns: + Complete URL for the batch cancel request + """ + raise NotImplementedError("Provider does not support cancelling batches") + + def transform_cancel_batch_response( + self, + model: str | None, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + litellm_params: dict, # mutable-ok: batch-config signature contract is a plain dict + ) -> LiteLLMBatch: + """ + Transform provider-specific batch cancel response to LiteLLM format. + + Args: + model: Model name + raw_response: Raw HTTP response + logging_obj: Logging object + litellm_params: LiteLLM parameters + + Returns: + LiteLLM batch object + """ + raise NotImplementedError("Provider does not support cancelling batches") + @abstractmethod def get_error_class(self, error_message: str, status_code: int, headers: dict | Headers) -> "BaseLLMException": """ diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index c9e9906e931..d7a9fde47a6 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -4313,6 +4313,261 @@ class BaseLLMHTTPHandler: litellm_params=litellm_params, ) + def list_batches( + self, + litellm_params: dict, # mutable-ok: litellm_params contract across batch handlers is a plain dict + provider_config: "BaseBatchesConfig", + headers: dict, # mutable-ok: validate_environment fills provider headers in place + api_base: str | None, + api_key: str | None, + logging_obj: "LiteLLMLoggingObj", + after: str | None = None, + limit: int | None = None, + _is_async: bool = False, + client: Union["HTTPHandler", "AsyncHTTPHandler"] | None = None, + timeout: float | httpx.Timeout | None = None, + model: str | None = None, + ) -> ( + dict | Coroutine[object, object, dict] + ): # mutable-ok: list_batches returns the provider's dict payload like create_batch returns dict request shapes + """ + List batches using provider-specific configuration. + """ + if model is None: + raise ValueError("model is required for list_batches") + + headers = provider_config.validate_environment( + api_key=api_key, + headers=headers, + model=model, + messages=[], # mutable-ok: validate_environment signature declares list[AllMessageValues] + optional_params=litellm_params, + litellm_params=litellm_params, + ) + + list_url: Final = provider_config.get_list_batches_url( + api_base=api_base, + api_key=api_key, + model=model, + optional_params=litellm_params, + litellm_params=litellm_params, + after=after, + limit=limit, + ) + + if _is_async: + return self.async_list_batches( + list_url=list_url, + litellm_params=litellm_params, + provider_config=provider_config, + headers=headers, + logging_obj=logging_obj, + client=client, + timeout=timeout, + model=model, + ) + + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client() + else: + sync_httpx_client = client + + try: + batch_response = sync_httpx_client.get( + url=list_url, + headers=headers, + timeout=timeout, + ) + except httpx.HTTPError as e: + verbose_logger.exception("Error listing batches: %s", e) + raise self._handle_error( + e=e, + provider_config=provider_config, + ) + + return provider_config.transform_list_batches_response( + model=model, + raw_response=batch_response, + logging_obj=logging_obj, + litellm_params=litellm_params, + ) + + async def async_list_batches( + self, + list_url: str, + litellm_params: dict, # mutable-ok: litellm_params contract across batch handlers is a plain dict + provider_config: "BaseBatchesConfig", + headers: dict, # mutable-ok: validate_environment fills provider headers in place + logging_obj: "LiteLLMLoggingObj", + client: Union["HTTPHandler", "AsyncHTTPHandler"] | None = None, + timeout: float | httpx.Timeout | None = None, + model: str | None = None, + ) -> dict: # mutable-ok: returns the provider list payload mirroring create_batch's dict contract + """ + Async version of list_batches + """ + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client(llm_provider=provider_config.custom_llm_provider) + else: + async_httpx_client = client + + if logging_obj is not None: + logging_obj.pre_call( + input="", + api_key="", + additional_args={ # mutable-ok: pre_call logging contract takes a plain dict + "api_base": list_url, + "headers": headers, + }, + ) + + try: + batch_response = await async_httpx_client.get( + url=list_url, + headers=headers, + timeout=timeout, + ) + except httpx.HTTPError as e: + verbose_logger.exception("Error listing batches: %s", e) + raise self._handle_error( + e=e, + provider_config=provider_config, + ) + + return provider_config.transform_list_batches_response( + model=model, + raw_response=batch_response, + logging_obj=logging_obj, + litellm_params=litellm_params, + ) + + def cancel_batch( + self, + batch_id: str, + litellm_params: dict, # mutable-ok: litellm_params contract across batch handlers is a plain dict + provider_config: "BaseBatchesConfig", + headers: dict, # mutable-ok: validate_environment fills provider headers in place + api_base: str | None, + api_key: str | None, + logging_obj: "LiteLLMLoggingObj", + _is_async: bool = False, + client: Union["HTTPHandler", "AsyncHTTPHandler"] | None = None, + timeout: float | httpx.Timeout | None = None, + model: str | None = None, + ) -> Union["LiteLLMBatch", Coroutine[object, object, "LiteLLMBatch"]]: + """ + Cancel a batch using provider-specific configuration. + """ + if model is None: + raise ValueError("model is required for cancel_batch") + + headers = provider_config.validate_environment( + api_key=api_key, + headers=headers, + model=model, + messages=[], # mutable-ok: validate_environment signature declares list[AllMessageValues] + optional_params=litellm_params, + litellm_params=litellm_params, + ) + + cancel_url: Final = provider_config.get_cancel_batch_url( + api_base=api_base, + api_key=api_key, + model=model, + batch_id=batch_id, + optional_params=litellm_params, + litellm_params=litellm_params, + ) + + if _is_async: + return self.async_cancel_batch( + cancel_url=cancel_url, + litellm_params=litellm_params, + provider_config=provider_config, + headers=headers, + logging_obj=logging_obj, + client=client, + timeout=timeout, + batch_id=batch_id, + model=model, + ) + + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client() + else: + sync_httpx_client = client + + try: + batch_response = sync_httpx_client.post( + url=cancel_url, + headers=headers, + timeout=timeout, + ) + except httpx.HTTPError as e: + verbose_logger.exception("Error cancelling batch: %s", e) + raise self._handle_error( + e=e, + provider_config=provider_config, + ) + + return provider_config.transform_cancel_batch_response( + model=model, + raw_response=batch_response, + logging_obj=logging_obj, + litellm_params=litellm_params, + ) + + async def async_cancel_batch( + self, + cancel_url: str, + litellm_params: dict, # mutable-ok: litellm_params contract across batch handlers is a plain dict + provider_config: "BaseBatchesConfig", + headers: dict, # mutable-ok: validate_environment fills provider headers in place + logging_obj: "LiteLLMLoggingObj", + client: Union["HTTPHandler", "AsyncHTTPHandler"] | None = None, + timeout: float | httpx.Timeout | None = None, + batch_id: str | None = None, + model: str | None = None, + ) -> "LiteLLMBatch": + """ + Async version of cancel_batch + """ + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client(llm_provider=provider_config.custom_llm_provider) + else: + async_httpx_client = client + + if logging_obj is not None: + logging_obj.pre_call( + input=batch_id or "", + api_key="", + additional_args={ # mutable-ok: pre_call logging contract takes a plain dict + "api_base": cancel_url, + "headers": headers, + "batch_id": batch_id, + }, + ) + + try: + batch_response = await async_httpx_client.post( + url=cancel_url, + headers=headers, + timeout=timeout, + ) + except httpx.HTTPError as e: + verbose_logger.exception("Error cancelling batch: %s", e) + raise self._handle_error( + e=e, + provider_config=provider_config, + ) + + return provider_config.transform_cancel_batch_response( + model=model, + raw_response=batch_response, + logging_obj=logging_obj, + litellm_params=litellm_params, + ) + def cancel_response_api_handler( self, response_id: str, diff --git a/tests/test_litellm/llms/anthropic/batches/test_handler.py b/tests/test_litellm/llms/anthropic/batches/test_handler.py deleted file mode 100644 index 13c33772496..00000000000 --- a/tests/test_litellm/llms/anthropic/batches/test_handler.py +++ /dev/null @@ -1,184 +0,0 @@ -""" -Tests for AnthropicBatchesHandler list_batches and cancel_batch - -Same seam discipline as tests/unit/llms/anthropic/batches/test_handler.py: -only the async httpx client and credential resolvers are mocked. -""" - -from unittest.mock import AsyncMock, MagicMock, patch - -import httpx -import pytest - -from litellm.llms.anthropic.batches.handler import AnthropicBatchesHandler -from litellm.llms.anthropic.common_utils import AnthropicError -from litellm.types.utils import LiteLLMBatch - - -@pytest.fixture -def handler(): - return AnthropicBatchesHandler() - - -def _list_response(): - return httpx.Response( - status_code=200, - json={ - "data": [{"id": "msgbatch_1", "processing_status": "ended", "request_counts": {"succeeded": 1}}], - "has_more": False, - "first_id": "msgbatch_1", - "last_id": "msgbatch_1", - }, - request=httpx.Request("GET", "https://api.anthropic.com/v1/messages/batches"), - ) - - -def _cancel_response(): - return httpx.Response( - status_code=200, - json={ - "id": "msgbatch_abc", - "processing_status": "canceling", - "created_at": "2024-09-24T10:00:00Z", - "cancel_initiated_at": "2024-09-24T10:30:00Z", - "request_counts": {}, - }, - request=httpx.Request("POST", "https://api.anthropic.com/v1/messages/batches/msgbatch_abc/cancel"), - ) - - -@pytest.fixture -def patched_client(): - fake_client = MagicMock() - fake_client.get = AsyncMock(return_value=_list_response()) - fake_client.post = AsyncMock(return_value=_cancel_response()) - with patch( - "litellm.llms.anthropic.batches.handler.get_async_httpx_client", - return_value=fake_client, - ): - yield fake_client - - -@pytest.mark.asyncio -async def test_alist_batches_gets_list_url_with_pagination_and_headers(handler, patched_client): - result = await handler.alist_batches( - after="msgbatch_cursor", - limit=10, - api_base="https://api.anthropic.com", - api_key="sk-ant-test", - timeout=60.0, - max_retries=0, - ) - - fake_client = patched_client - fake_client.get.assert_awaited_once() - _, call_kwargs = fake_client.get.call_args - url = httpx.URL(call_kwargs["url"]) - assert url.path == "/v1/messages/batches" - assert url.params["limit"] == "10" - assert url.params["after_id"] == "msgbatch_cursor" - headers = call_kwargs["headers"] - assert headers["x-api-key"] == "sk-ant-test" - assert headers["anthropic-version"] == "2023-06-01" - assert headers["anthropic-beta"] == "message-batches-2024-09-24" - assert call_kwargs["timeout"] == 60.0 - - assert result["object"] == "list" - assert result["has_more"] is False - assert [b.id for b in result["data"]] == ["msgbatch_1"] - - -@pytest.mark.asyncio -async def test_acancel_batch_posts_to_cancel_url_and_maps_response(handler, patched_client): - batch = await handler.acancel_batch( - batch_id="msgbatch_abc", - api_base="https://api.anthropic.com", - api_key="sk-ant-test", - timeout=60.0, - max_retries=0, - ) - - fake_client = patched_client - fake_client.post.assert_awaited_once() - _, call_kwargs = fake_client.post.call_args - assert call_kwargs["url"] == "https://api.anthropic.com/v1/messages/batches/msgbatch_abc/cancel" - assert call_kwargs["headers"]["x-api-key"] == "sk-ant-test" - assert call_kwargs["timeout"] == 60.0 - - assert isinstance(batch, LiteLLMBatch) - assert batch.id == "msgbatch_abc" - assert batch.status == "cancelling" - - -@pytest.mark.asyncio -async def test_alist_batches_4xx_raises_anthropic_error(handler): - error_response = httpx.Response( - status_code=401, - json={"error": {"type": "authentication_error", "message": "bad key"}}, - request=httpx.Request("GET", "https://api.anthropic.com/v1/messages/batches"), - ) - fake_client = MagicMock() - fake_client.get = AsyncMock(return_value=error_response) - with patch( - "litellm.llms.anthropic.batches.handler.get_async_httpx_client", - return_value=fake_client, - ): - with pytest.raises(AnthropicError) as exc_info: - await handler.alist_batches( - after=None, - limit=None, - api_base="https://api.anthropic.com", - api_key="sk-ant-test", - timeout=60.0, - max_retries=0, - ) - assert exc_info.value.status_code == 401 - - -@pytest.mark.asyncio -async def test_acancel_batch_4xx_raises_anthropic_error(handler): - error_response = httpx.Response( - status_code=404, - json={"error": {"type": "not_found_error", "message": "no batch"}}, - request=httpx.Request("POST", "https://api.anthropic.com/v1/messages/batches/missing/cancel"), - ) - fake_client = MagicMock() - fake_client.post = AsyncMock(return_value=error_response) - with patch( - "litellm.llms.anthropic.batches.handler.get_async_httpx_client", - return_value=fake_client, - ): - with pytest.raises(AnthropicError) as exc_info: - await handler.acancel_batch( - batch_id="missing", - api_base="https://api.anthropic.com", - api_key="sk-ant-test", - timeout=60.0, - max_retries=0, - ) - assert exc_info.value.status_code == 404 - - -def test_list_batches_sync_dispatch(handler, patched_client): - result = handler.list_batches( - _is_async=False, - after=None, - limit=5, - api_base="https://api.anthropic.com", - api_key="sk-ant-test", - timeout=60.0, - max_retries=0, - ) - assert result["object"] == "list" - - -def test_cancel_batch_sync_dispatch(handler, patched_client): - batch = handler.cancel_batch( - _is_async=False, - batch_id="msgbatch_abc", - api_base="https://api.anthropic.com", - api_key="sk-ant-test", - timeout=60.0, - max_retries=0, - ) - assert batch.id == "msgbatch_abc" diff --git a/tests/test_litellm/llms/anthropic/batches/test_transformation.py b/tests/test_litellm/llms/anthropic/batches/test_transformation.py index 8cea774a303..9ae4131bc0a 100644 --- a/tests/test_litellm/llms/anthropic/batches/test_transformation.py +++ b/tests/test_litellm/llms/anthropic/batches/test_transformation.py @@ -203,10 +203,12 @@ def test_create_batch_response_error_raises_anthropic_error(config): def test_get_list_batches_url_encodes_limit_and_after_id(config): url = config.get_list_batches_url( api_base="https://api.anthropic.com", - after="msgbatch_cursor", - limit=10, + api_key="sk-ant-test", + model="", optional_params={}, litellm_params={}, + after="msgbatch_cursor", + limit=10, ) parsed = httpx.URL(url) assert parsed.path == "/v1/messages/batches" @@ -227,7 +229,9 @@ def test_transform_list_batches_response_maps_batches_and_has_more(config): }, method="GET", ) - result = config.transform_list_batches_response(raw_response=raw) + result = config.transform_list_batches_response( + model=None, raw_response=raw, logging_obj=MagicMock(), litellm_params={} + ) assert result["object"] == "list" assert result["has_more"] is True assert result["first_id"] == "msgbatch_1" @@ -244,6 +248,8 @@ def test_transform_list_batches_response_maps_batches_and_has_more(config): def test_get_cancel_batch_url_ends_with_cancel(config): url = config.get_cancel_batch_url( api_base="https://api.anthropic.com", + api_key="sk-ant-test", + model="", batch_id="msgbatch_abc", optional_params={}, litellm_params={}, diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index 67a8d045036..53c555b7fd7 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -4131,3 +4131,162 @@ async def test_chat_completion_agentic_followup_does_not_repeat_request_params_f assert followup_calls[0]["temperature"] == 0.2 assert followup_calls[0]["api_base"] == "https://a" assert followup_calls[0]["model"] == "openai/gpt-5" +# =========================================================================== # +# batches: list / cancel through BaseBatchesConfig (Anthropic) +# =========================================================================== # + + +from litellm.llms.anthropic.batches.transformation import AnthropicBatchesConfig +from litellm.types.utils import LiteLLMBatch + + +def _async_anthropic_client(response: httpx.Response) -> Mock: + client = Mock(spec=AsyncHTTPHandler) + client.get = AsyncMock(return_value=response) + client.post = AsyncMock(return_value=response) + return client + + +def _anthropic_list_response() -> httpx.Response: + return httpx.Response( + status_code=200, + json={ + "data": [{"id": "msgbatch_1", "processing_status": "ended", "request_counts": {"succeeded": 1}}], + "has_more": False, + "first_id": "msgbatch_1", + "last_id": "msgbatch_1", + }, + request=httpx.Request("GET", "https://api.anthropic.com/v1/messages/batches"), + ) + + +def _anthropic_cancel_response() -> httpx.Response: + return httpx.Response( + status_code=200, + json={ + "id": "msgbatch_abc", + "processing_status": "canceling", + "created_at": "2024-09-24T10:00:00Z", + "cancel_initiated_at": "2024-09-24T10:30:00Z", + "request_counts": {}, + }, + request=httpx.Request("POST", "https://api.anthropic.com/v1/messages/batches/msgbatch_abc/cancel"), + ) + + +@pytest.mark.asyncio +async def test_list_batches_anthropic_gets_paginated_url_and_headers(): + client = _async_anthropic_client(_anthropic_list_response()) + result = await BaseLLMHTTPHandler().list_batches( + litellm_params={}, + provider_config=AnthropicBatchesConfig(), + headers={}, + api_base="https://api.anthropic.com", + api_key="sk-ant-test", + logging_obj=None, + after="msgbatch_cursor", + limit=10, + _is_async=True, + client=client, + timeout=60.0, + model="", + ) + + client.get.assert_awaited_once() + _, call_kwargs = client.get.call_args + url = httpx.URL(call_kwargs["url"]) + assert url.path == "/v1/messages/batches" + assert url.params["limit"] == "10" + assert url.params["after_id"] == "msgbatch_cursor" + headers = call_kwargs["headers"] + assert headers["x-api-key"] == "sk-ant-test" + assert headers["anthropic-beta"] == "message-batches-2024-09-24" + assert call_kwargs["timeout"] == 60.0 + + assert result["object"] == "list" + assert result["has_more"] is False + assert [b.id for b in result["data"]] == ["msgbatch_1"] + + +def test_list_batches_anthropic_sync_gets_url_and_headers(): + client = Mock(spec=HTTPHandler) + client.get = Mock(return_value=_anthropic_list_response()) + result = BaseLLMHTTPHandler().list_batches( + litellm_params={}, + provider_config=AnthropicBatchesConfig(), + headers={}, + api_base="https://api.anthropic.com", + api_key="sk-ant-test", + logging_obj=None, + after=None, + limit=5, + _is_async=False, + client=client, + timeout=60.0, + model="", + ) + + client.get.assert_called_once() + _, call_kwargs = client.get.call_args + url = httpx.URL(call_kwargs["url"]) + assert url.path == "/v1/messages/batches" + assert url.params["limit"] == "5" + assert call_kwargs["headers"]["x-api-key"] == "sk-ant-test" + assert call_kwargs["timeout"] == 60.0 + assert result["object"] == "list" + + +@pytest.mark.asyncio +async def test_cancel_batch_anthropic_posts_to_cancel_url_and_maps_response(): + client = _async_anthropic_client(_anthropic_cancel_response()) + batch = await BaseLLMHTTPHandler().cancel_batch( + batch_id="msgbatch_abc", + litellm_params={}, + provider_config=AnthropicBatchesConfig(), + headers={}, + api_base="https://api.anthropic.com", + api_key="sk-ant-test", + logging_obj=None, + _is_async=True, + client=client, + timeout=60.0, + model="", + ) + + client.post.assert_awaited_once() + _, call_kwargs = client.post.call_args + assert call_kwargs["url"] == "https://api.anthropic.com/v1/messages/batches/msgbatch_abc/cancel" + assert call_kwargs["headers"]["x-api-key"] == "sk-ant-test" + assert call_kwargs["headers"]["anthropic-beta"] == "message-batches-2024-09-24" + assert call_kwargs["timeout"] == 60.0 + + assert isinstance(batch, LiteLLMBatch) + assert batch.id == "msgbatch_abc" + assert batch.status == "cancelling" + + +@pytest.mark.asyncio +async def test_list_batches_anthropic_4xx_raises_anthropic_error(): + from litellm.llms.anthropic.common_utils import AnthropicError + + client = _async_anthropic_client( + httpx.Response( + status_code=401, + json={"error": {"type": "authentication_error", "message": "bad key"}}, + request=httpx.Request("GET", "https://api.anthropic.com/v1/messages/batches"), + ) + ) + with pytest.raises(AnthropicError) as exc_info: + await BaseLLMHTTPHandler().list_batches( + litellm_params={}, + provider_config=AnthropicBatchesConfig(), + headers={}, + api_base="https://api.anthropic.com", + api_key="sk-ant-test", + logging_obj=None, + _is_async=True, + client=client, + timeout=60.0, + model="", + ) + assert exc_info.value.status_code == 401 diff --git a/tests/unit/batches/test_main.py b/tests/unit/batches/test_main.py index 271cf4ab0ee..42e02f7f1ea 100644 --- a/tests/unit/batches/test_main.py +++ b/tests/unit/batches/test_main.py @@ -379,10 +379,10 @@ def test_list__vertex_ai_dispatch(seams): def test_list__anthropic_dispatch(seams): result = bm.list_batches(custom_llm_provider="anthropic", after="cur", limit=5) - assert result is seams.anthropic.list_batches.return_value - _assert_only(seams.anthropic.list_batches, seams, "list_batches") + assert result is seams.base_http.list_batches.return_value + _assert_only(seams.base_http.list_batches, seams, "list_batches") - kw = seams.anthropic.list_batches.call_args.kwargs + kw = seams.base_http.list_batches.call_args.kwargs assert kw["after"] == "cur" assert kw["limit"] == 5 assert kw["_is_async"] is False @@ -433,10 +433,10 @@ def test_cancel__vertex_ai_dispatch(seams): def test_cancel__anthropic_dispatch(seams): result = bm.cancel_batch(batch_id="msgbatch_1", custom_llm_provider="anthropic") - assert result is seams.anthropic.cancel_batch.return_value - _assert_only(seams.anthropic.cancel_batch, seams, "cancel_batch") + assert result is seams.base_http.cancel_batch.return_value + _assert_only(seams.base_http.cancel_batch, seams, "cancel_batch") - kw = seams.anthropic.cancel_batch.call_args.kwargs + kw = seams.base_http.cancel_batch.call_args.kwargs assert kw["batch_id"] == "msgbatch_1" assert kw["_is_async"] is False