fix(anthropic): isolate per-line batch params, map cancelled ended batches, honor request timeout

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Mubashir Osmani 2026-09-22 21:16:13 +00:00
parent 6e476ef734
commit c170682187
5 changed files with 93 additions and 18 deletions

View file

@ -241,7 +241,7 @@ class AnthropicBatchesHandler:
},
)
async_client: Final = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC)
response: Final = await async_client.get(url=list_url, headers=headers)
response: Final = await async_client.get(url=list_url, headers=headers, timeout=timeout)
if response.status_code >= 400:
raise self.provider_config.get_error_class(
error_message=response.text,
@ -337,7 +337,7 @@ class AnthropicBatchesHandler:
},
)
async_client: Final = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC)
response: Final = await async_client.post(url=cancel_url, headers=headers)
response: Final = await async_client.post(url=cancel_url, headers=headers, timeout=timeout)
if response.status_code >= 400:
raise self.provider_config.get_error_class(
error_message=response.text,

View file

@ -92,10 +92,6 @@ class AnthropicMessageBatchList(TypedDict, total=False):
_ANTHROPIC_MESSAGE_BATCH_ADAPTER: Final = TypeAdapter(AnthropicMessageBatch)
_ANTHROPIC_MESSAGE_BATCH_LIST_ADAPTER: Final = TypeAdapter(AnthropicMessageBatchList)
_OBJECT_DICT_ADAPTER: Final = TypeAdapter(dict[str, object])
_EMPTY_PARAMS: Final[
dict
] = {} # mutable-ok: shared empty mapping for signatures that declare dict; callees only read it
_EMPTY_METADATA: Final[dict] = {} # mutable-ok: shared empty LiteLLMBatch metadata; callers never mutate it
_ANTHROPIC_PROCESSING_STATUS_TO_OPENAI_STATUS: Final[
@ -150,7 +146,6 @@ def transform_anthropic_message_batch(response_data: AnthropicMessageBatch) -> L
"""Map an Anthropic MessageBatch object onto a LiteLLMBatch in OpenAI batch shape."""
batch_id: Final = response_data.get("id", "")
processing_status: Final = response_data.get("processing_status", "in_progress")
openai_status: Final = _ANTHROPIC_PROCESSING_STATUS_TO_OPENAI_STATUS.get(processing_status, "in_progress")
created_at: Final = _parse_anthropic_timestamp(response_data.get("created_at"))
ended_at: Final = _parse_anthropic_timestamp(response_data.get("ended_at"))
@ -158,6 +153,12 @@ def transform_anthropic_message_batch(response_data: AnthropicMessageBatch) -> L
cancel_initiated_at: Final = _parse_anthropic_timestamp(response_data.get("cancel_initiated_at"))
archived_at: Final = _parse_anthropic_timestamp(response_data.get("archived_at"))
openai_status: Final = (
"cancelled"
if processing_status == "ended" and cancel_initiated_at is not None
else _ANTHROPIC_PROCESSING_STATUS_TO_OPENAI_STATUS.get(processing_status, "in_progress")
)
request_counts_data: Final = response_data.get("request_counts", EMPTY_MAPPING)
from openai.types.batch import BatchRequestCounts
@ -186,16 +187,16 @@ def transform_anthropic_message_batch(response_data: AnthropicMessageBatch) -> L
output_file_id=batch_id,
error_file_id=None,
created_at=created_at or int(time.time()),
in_progress_at=created_at if processing_status == "in_progress" else None,
in_progress_at=created_at if openai_status == "in_progress" else None,
expires_at=expires_at,
finalizing_at=None,
completed_at=ended_at if processing_status == "ended" else None,
completed_at=ended_at if openai_status == "completed" else None,
failed_at=None,
expired_at=archived_at if archived_at else None,
cancelling_at=(cancel_initiated_at if processing_status == "canceling" else None),
cancelled_at=(ended_at if processing_status == "canceling" and ended_at else None),
cancelling_at=(cancel_initiated_at if openai_status in ("cancelling", "cancelled") else None),
cancelled_at=(ended_at if openai_status == "cancelled" else None),
request_counts=request_counts,
metadata=_EMPTY_METADATA,
metadata={}, # mutable-ok: LiteLLMBatch wants a fresh dict per instance
)
@ -227,7 +228,7 @@ def transform_openai_batch_lines_to_anthropic_requests(
optional_params: Final = _OBJECT_DICT_ADAPTER.validate_python(
anthropic_config.map_openai_params(
non_default_params=non_default_params,
optional_params=_EMPTY_PARAMS,
optional_params={}, # mutable-ok: map_openai_params writes into this per line
model=model,
drop_params=True,
),
@ -237,8 +238,8 @@ def transform_openai_batch_lines_to_anthropic_requests(
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=_EMPTY_PARAMS,
headers=_EMPTY_PARAMS,
litellm_params={}, # mutable-ok: transform_request may write into this per line
headers={}, # mutable-ok: transform_request writes headers entries per line
),
)
params["model"] = model

View file

@ -81,6 +81,7 @@ async def test_alist_batches_gets_list_url_with_pagination_and_headers(handler,
assert headers["x-api-key"] == "sk-ant-test"
assert headers["anthropic-version"] == "2023-06-01"
assert headers["anthropic-beta"] == "message-batches-2024-09-24"
assert call_kwargs["timeout"] == 60.0
assert result["object"] == "list"
assert result["has_more"] is False
@ -102,6 +103,7 @@ async def test_acancel_batch_posts_to_cancel_url_and_maps_response(handler, patc
_, call_kwargs = fake_client.post.call_args
assert call_kwargs["url"] == "https://api.anthropic.com/v1/messages/batches/msgbatch_abc/cancel"
assert call_kwargs["headers"]["x-api-key"] == "sk-ant-test"
assert call_kwargs["timeout"] == 60.0
assert isinstance(batch, LiteLLMBatch)
assert batch.id == "msgbatch_abc"

View file

@ -111,8 +111,42 @@ def test_openai_lines_translate_to_anthropic_params():
assert first["params"]["system"]
assert all(m["role"] != "system" for m in first["params"]["messages"])
# max_tokens is required by Anthropic and defaulted even when absent
assert second["params"]["max_tokens"]
assert second["params"]["max_tokens"] == 64000
assert second["params"]["model"] == "claude-sonnet-4-5"
# params are isolated per line: req-1's temperature must not leak into req-2
assert "temperature" not in second["params"]
def test_openai_lines_params_do_not_leak_across_calls():
transform_openai_batch_lines_to_anthropic_requests(
[
{
"custom_id": "req-1",
"method": "POST",
"url": "/v1/chat/completions",
"body": {
"model": "ignored",
"messages": [{"role": "user", "content": "Hi"}],
"temperature": 0.2,
"max_tokens": 7,
},
}
],
model="claude-sonnet-4-5",
)
(second_run,) = transform_openai_batch_lines_to_anthropic_requests(
[
{
"custom_id": "req-2",
"method": "POST",
"url": "/v1/chat/completions",
"body": {"model": "ignored", "messages": [{"role": "user", "content": "Hello"}]},
}
],
model="claude-sonnet-4-5",
)
assert "temperature" not in second_run["params"]
assert second_run["params"]["max_tokens"] == 64000
def test_openai_line_missing_custom_id_raises():
@ -234,6 +268,44 @@ def test_cancel_batch_response_canceling_maps_to_cancelling(config):
assert batch.cancelling_at == 1727173800
def test_ended_after_cancel_initiated_maps_to_cancelled(config):
raw = _response(
{
"id": "msgbatch_abc",
"processing_status": "ended",
"created_at": "2024-09-24T10:00:00Z",
"cancel_initiated_at": "2024-09-24T10:30:00Z",
"ended_at": "2024-09-24T10:35:00Z",
"request_counts": {"canceled": 1, "succeeded": 0},
}
)
batch = config.transform_retrieve_batch_response(
model=None, raw_response=raw, logging_obj=MagicMock(), litellm_params={}
)
assert batch.status == "cancelled"
assert batch.cancelled_at == 1727174100
assert batch.cancelling_at == 1727173800
assert batch.completed_at is None
def test_ended_without_cancel_maps_to_completed(config):
raw = _response(
{
"id": "msgbatch_abc",
"processing_status": "ended",
"created_at": "2024-09-24T10:00:00Z",
"ended_at": "2024-09-24T10:35:00Z",
"request_counts": {"succeeded": 2},
}
)
batch = config.transform_retrieve_batch_response(
model=None, raw_response=raw, logging_obj=MagicMock(), litellm_params={}
)
assert batch.status == "completed"
assert batch.completed_at == 1727174100
assert batch.cancelled_at is None
# =========================================================================== #
# transform_anthropic_batch_result_line
# =========================================================================== #

View file

@ -376,8 +376,8 @@ def test_transform_retrieve_response_canceling_maps_to_cancelling(config):
# "canceling" -> OpenAI "cancelling".
assert batch.status == "cancelling"
assert batch.cancelling_at == 1727173800
# cancelled_at = ended_at when canceling and ended_at present.
assert batch.cancelled_at == 1727174700
# cancelled_at stays unset while the batch is still cancelling
assert batch.cancelled_at is None
assert batch.completed_at is None