diff --git a/litellm/llms/anthropic/batches/handler.py b/litellm/llms/anthropic/batches/handler.py index 50047a1edfd..fe9afa6ce46 100644 --- a/litellm/llms/anthropic/batches/handler.py +++ b/litellm/llms/anthropic/batches/handler.py @@ -241,7 +241,7 @@ class AnthropicBatchesHandler: }, ) async_client: Final = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC) - response: Final = await async_client.get(url=list_url, headers=headers) + response: Final = await async_client.get(url=list_url, headers=headers, timeout=timeout) if response.status_code >= 400: raise self.provider_config.get_error_class( error_message=response.text, @@ -337,7 +337,7 @@ class AnthropicBatchesHandler: }, ) async_client: Final = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC) - response: Final = await async_client.post(url=cancel_url, headers=headers) + response: Final = await async_client.post(url=cancel_url, headers=headers, timeout=timeout) if response.status_code >= 400: raise self.provider_config.get_error_class( error_message=response.text, diff --git a/litellm/llms/anthropic/batches/transformation.py b/litellm/llms/anthropic/batches/transformation.py index d61af0cdd35..bdf96795857 100644 --- a/litellm/llms/anthropic/batches/transformation.py +++ b/litellm/llms/anthropic/batches/transformation.py @@ -92,10 +92,6 @@ class AnthropicMessageBatchList(TypedDict, total=False): _ANTHROPIC_MESSAGE_BATCH_ADAPTER: Final = TypeAdapter(AnthropicMessageBatch) _ANTHROPIC_MESSAGE_BATCH_LIST_ADAPTER: Final = TypeAdapter(AnthropicMessageBatchList) _OBJECT_DICT_ADAPTER: Final = TypeAdapter(dict[str, object]) -_EMPTY_PARAMS: Final[ - dict -] = {} # mutable-ok: shared empty mapping for signatures that declare dict; callees only read it -_EMPTY_METADATA: Final[dict] = {} # mutable-ok: shared empty LiteLLMBatch metadata; callers never mutate it _ANTHROPIC_PROCESSING_STATUS_TO_OPENAI_STATUS: Final[ @@ -150,7 +146,6 @@ def transform_anthropic_message_batch(response_data: AnthropicMessageBatch) -> L """Map an Anthropic MessageBatch object onto a LiteLLMBatch in OpenAI batch shape.""" batch_id: Final = response_data.get("id", "") processing_status: Final = response_data.get("processing_status", "in_progress") - openai_status: Final = _ANTHROPIC_PROCESSING_STATUS_TO_OPENAI_STATUS.get(processing_status, "in_progress") created_at: Final = _parse_anthropic_timestamp(response_data.get("created_at")) ended_at: Final = _parse_anthropic_timestamp(response_data.get("ended_at")) @@ -158,6 +153,12 @@ def transform_anthropic_message_batch(response_data: AnthropicMessageBatch) -> L cancel_initiated_at: Final = _parse_anthropic_timestamp(response_data.get("cancel_initiated_at")) archived_at: Final = _parse_anthropic_timestamp(response_data.get("archived_at")) + openai_status: Final = ( + "cancelled" + if processing_status == "ended" and cancel_initiated_at is not None + else _ANTHROPIC_PROCESSING_STATUS_TO_OPENAI_STATUS.get(processing_status, "in_progress") + ) + request_counts_data: Final = response_data.get("request_counts", EMPTY_MAPPING) from openai.types.batch import BatchRequestCounts @@ -186,16 +187,16 @@ def transform_anthropic_message_batch(response_data: AnthropicMessageBatch) -> L output_file_id=batch_id, error_file_id=None, created_at=created_at or int(time.time()), - in_progress_at=created_at if processing_status == "in_progress" else None, + in_progress_at=created_at if openai_status == "in_progress" else None, expires_at=expires_at, finalizing_at=None, - completed_at=ended_at if processing_status == "ended" else None, + completed_at=ended_at if openai_status == "completed" else None, failed_at=None, expired_at=archived_at if archived_at else None, - cancelling_at=(cancel_initiated_at if processing_status == "canceling" else None), - cancelled_at=(ended_at if processing_status == "canceling" and ended_at else None), + cancelling_at=(cancel_initiated_at if openai_status in ("cancelling", "cancelled") else None), + cancelled_at=(ended_at if openai_status == "cancelled" else None), request_counts=request_counts, - metadata=_EMPTY_METADATA, + metadata={}, # mutable-ok: LiteLLMBatch wants a fresh dict per instance ) @@ -227,7 +228,7 @@ def transform_openai_batch_lines_to_anthropic_requests( optional_params: Final = _OBJECT_DICT_ADAPTER.validate_python( anthropic_config.map_openai_params( non_default_params=non_default_params, - optional_params=_EMPTY_PARAMS, + optional_params={}, # mutable-ok: map_openai_params writes into this per line model=model, drop_params=True, ), @@ -237,8 +238,8 @@ def transform_openai_batch_lines_to_anthropic_requests( model=model, messages=messages, optional_params=optional_params, - litellm_params=_EMPTY_PARAMS, - headers=_EMPTY_PARAMS, + litellm_params={}, # mutable-ok: transform_request may write into this per line + headers={}, # mutable-ok: transform_request writes headers entries per line ), ) params["model"] = model diff --git a/tests/test_litellm/llms/anthropic/batches/test_handler.py b/tests/test_litellm/llms/anthropic/batches/test_handler.py index 8b2ffa59f2b..13c33772496 100644 --- a/tests/test_litellm/llms/anthropic/batches/test_handler.py +++ b/tests/test_litellm/llms/anthropic/batches/test_handler.py @@ -81,6 +81,7 @@ async def test_alist_batches_gets_list_url_with_pagination_and_headers(handler, assert headers["x-api-key"] == "sk-ant-test" assert headers["anthropic-version"] == "2023-06-01" assert headers["anthropic-beta"] == "message-batches-2024-09-24" + assert call_kwargs["timeout"] == 60.0 assert result["object"] == "list" assert result["has_more"] is False @@ -102,6 +103,7 @@ async def test_acancel_batch_posts_to_cancel_url_and_maps_response(handler, patc _, call_kwargs = fake_client.post.call_args assert call_kwargs["url"] == "https://api.anthropic.com/v1/messages/batches/msgbatch_abc/cancel" assert call_kwargs["headers"]["x-api-key"] == "sk-ant-test" + assert call_kwargs["timeout"] == 60.0 assert isinstance(batch, LiteLLMBatch) assert batch.id == "msgbatch_abc" diff --git a/tests/test_litellm/llms/anthropic/batches/test_transformation.py b/tests/test_litellm/llms/anthropic/batches/test_transformation.py index b5a673fac04..8cea774a303 100644 --- a/tests/test_litellm/llms/anthropic/batches/test_transformation.py +++ b/tests/test_litellm/llms/anthropic/batches/test_transformation.py @@ -111,8 +111,42 @@ def test_openai_lines_translate_to_anthropic_params(): assert first["params"]["system"] assert all(m["role"] != "system" for m in first["params"]["messages"]) # max_tokens is required by Anthropic and defaulted even when absent - assert second["params"]["max_tokens"] + assert second["params"]["max_tokens"] == 64000 assert second["params"]["model"] == "claude-sonnet-4-5" + # params are isolated per line: req-1's temperature must not leak into req-2 + assert "temperature" not in second["params"] + + +def test_openai_lines_params_do_not_leak_across_calls(): + transform_openai_batch_lines_to_anthropic_requests( + [ + { + "custom_id": "req-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": "ignored", + "messages": [{"role": "user", "content": "Hi"}], + "temperature": 0.2, + "max_tokens": 7, + }, + } + ], + model="claude-sonnet-4-5", + ) + (second_run,) = transform_openai_batch_lines_to_anthropic_requests( + [ + { + "custom_id": "req-2", + "method": "POST", + "url": "/v1/chat/completions", + "body": {"model": "ignored", "messages": [{"role": "user", "content": "Hello"}]}, + } + ], + model="claude-sonnet-4-5", + ) + assert "temperature" not in second_run["params"] + assert second_run["params"]["max_tokens"] == 64000 def test_openai_line_missing_custom_id_raises(): @@ -234,6 +268,44 @@ def test_cancel_batch_response_canceling_maps_to_cancelling(config): assert batch.cancelling_at == 1727173800 +def test_ended_after_cancel_initiated_maps_to_cancelled(config): + raw = _response( + { + "id": "msgbatch_abc", + "processing_status": "ended", + "created_at": "2024-09-24T10:00:00Z", + "cancel_initiated_at": "2024-09-24T10:30:00Z", + "ended_at": "2024-09-24T10:35:00Z", + "request_counts": {"canceled": 1, "succeeded": 0}, + } + ) + batch = config.transform_retrieve_batch_response( + model=None, raw_response=raw, logging_obj=MagicMock(), litellm_params={} + ) + assert batch.status == "cancelled" + assert batch.cancelled_at == 1727174100 + assert batch.cancelling_at == 1727173800 + assert batch.completed_at is None + + +def test_ended_without_cancel_maps_to_completed(config): + raw = _response( + { + "id": "msgbatch_abc", + "processing_status": "ended", + "created_at": "2024-09-24T10:00:00Z", + "ended_at": "2024-09-24T10:35:00Z", + "request_counts": {"succeeded": 2}, + } + ) + batch = config.transform_retrieve_batch_response( + model=None, raw_response=raw, logging_obj=MagicMock(), litellm_params={} + ) + assert batch.status == "completed" + assert batch.completed_at == 1727174100 + assert batch.cancelled_at is None + + # =========================================================================== # # transform_anthropic_batch_result_line # =========================================================================== # diff --git a/tests/unit/llms/anthropic/batches/test_transformation.py b/tests/unit/llms/anthropic/batches/test_transformation.py index 943d36f7ec4..e296515835f 100644 --- a/tests/unit/llms/anthropic/batches/test_transformation.py +++ b/tests/unit/llms/anthropic/batches/test_transformation.py @@ -376,8 +376,8 @@ def test_transform_retrieve_response_canceling_maps_to_cancelling(config): # "canceling" -> OpenAI "cancelling". assert batch.status == "cancelling" assert batch.cancelling_at == 1727173800 - # cancelled_at = ended_at when canceling and ended_at present. - assert batch.cancelled_at == 1727174700 + # cancelled_at stays unset while the batch is still cancelling + assert batch.cancelled_at is None assert batch.completed_at is None