mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(anthropic): isolate per-line batch params, map cancelled ended batches, honor request timeout
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
6e476ef734
commit
c170682187
5 changed files with 93 additions and 18 deletions
|
|
@ -241,7 +241,7 @@ class AnthropicBatchesHandler:
|
|||
},
|
||||
)
|
||||
async_client: Final = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC)
|
||||
response: Final = await async_client.get(url=list_url, headers=headers)
|
||||
response: Final = await async_client.get(url=list_url, headers=headers, timeout=timeout)
|
||||
if response.status_code >= 400:
|
||||
raise self.provider_config.get_error_class(
|
||||
error_message=response.text,
|
||||
|
|
@ -337,7 +337,7 @@ class AnthropicBatchesHandler:
|
|||
},
|
||||
)
|
||||
async_client: Final = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC)
|
||||
response: Final = await async_client.post(url=cancel_url, headers=headers)
|
||||
response: Final = await async_client.post(url=cancel_url, headers=headers, timeout=timeout)
|
||||
if response.status_code >= 400:
|
||||
raise self.provider_config.get_error_class(
|
||||
error_message=response.text,
|
||||
|
|
|
|||
|
|
@ -92,10 +92,6 @@ class AnthropicMessageBatchList(TypedDict, total=False):
|
|||
_ANTHROPIC_MESSAGE_BATCH_ADAPTER: Final = TypeAdapter(AnthropicMessageBatch)
|
||||
_ANTHROPIC_MESSAGE_BATCH_LIST_ADAPTER: Final = TypeAdapter(AnthropicMessageBatchList)
|
||||
_OBJECT_DICT_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
_EMPTY_PARAMS: Final[
|
||||
dict
|
||||
] = {} # mutable-ok: shared empty mapping for signatures that declare dict; callees only read it
|
||||
_EMPTY_METADATA: Final[dict] = {} # mutable-ok: shared empty LiteLLMBatch metadata; callers never mutate it
|
||||
|
||||
|
||||
_ANTHROPIC_PROCESSING_STATUS_TO_OPENAI_STATUS: Final[
|
||||
|
|
@ -150,7 +146,6 @@ def transform_anthropic_message_batch(response_data: AnthropicMessageBatch) -> L
|
|||
"""Map an Anthropic MessageBatch object onto a LiteLLMBatch in OpenAI batch shape."""
|
||||
batch_id: Final = response_data.get("id", "")
|
||||
processing_status: Final = response_data.get("processing_status", "in_progress")
|
||||
openai_status: Final = _ANTHROPIC_PROCESSING_STATUS_TO_OPENAI_STATUS.get(processing_status, "in_progress")
|
||||
|
||||
created_at: Final = _parse_anthropic_timestamp(response_data.get("created_at"))
|
||||
ended_at: Final = _parse_anthropic_timestamp(response_data.get("ended_at"))
|
||||
|
|
@ -158,6 +153,12 @@ def transform_anthropic_message_batch(response_data: AnthropicMessageBatch) -> L
|
|||
cancel_initiated_at: Final = _parse_anthropic_timestamp(response_data.get("cancel_initiated_at"))
|
||||
archived_at: Final = _parse_anthropic_timestamp(response_data.get("archived_at"))
|
||||
|
||||
openai_status: Final = (
|
||||
"cancelled"
|
||||
if processing_status == "ended" and cancel_initiated_at is not None
|
||||
else _ANTHROPIC_PROCESSING_STATUS_TO_OPENAI_STATUS.get(processing_status, "in_progress")
|
||||
)
|
||||
|
||||
request_counts_data: Final = response_data.get("request_counts", EMPTY_MAPPING)
|
||||
from openai.types.batch import BatchRequestCounts
|
||||
|
||||
|
|
@ -186,16 +187,16 @@ def transform_anthropic_message_batch(response_data: AnthropicMessageBatch) -> L
|
|||
output_file_id=batch_id,
|
||||
error_file_id=None,
|
||||
created_at=created_at or int(time.time()),
|
||||
in_progress_at=created_at if processing_status == "in_progress" else None,
|
||||
in_progress_at=created_at if openai_status == "in_progress" else None,
|
||||
expires_at=expires_at,
|
||||
finalizing_at=None,
|
||||
completed_at=ended_at if processing_status == "ended" else None,
|
||||
completed_at=ended_at if openai_status == "completed" else None,
|
||||
failed_at=None,
|
||||
expired_at=archived_at if archived_at else None,
|
||||
cancelling_at=(cancel_initiated_at if processing_status == "canceling" else None),
|
||||
cancelled_at=(ended_at if processing_status == "canceling" and ended_at else None),
|
||||
cancelling_at=(cancel_initiated_at if openai_status in ("cancelling", "cancelled") else None),
|
||||
cancelled_at=(ended_at if openai_status == "cancelled" else None),
|
||||
request_counts=request_counts,
|
||||
metadata=_EMPTY_METADATA,
|
||||
metadata={}, # mutable-ok: LiteLLMBatch wants a fresh dict per instance
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -227,7 +228,7 @@ def transform_openai_batch_lines_to_anthropic_requests(
|
|||
optional_params: Final = _OBJECT_DICT_ADAPTER.validate_python(
|
||||
anthropic_config.map_openai_params(
|
||||
non_default_params=non_default_params,
|
||||
optional_params=_EMPTY_PARAMS,
|
||||
optional_params={}, # mutable-ok: map_openai_params writes into this per line
|
||||
model=model,
|
||||
drop_params=True,
|
||||
),
|
||||
|
|
@ -237,8 +238,8 @@ def transform_openai_batch_lines_to_anthropic_requests(
|
|||
model=model,
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params=_EMPTY_PARAMS,
|
||||
headers=_EMPTY_PARAMS,
|
||||
litellm_params={}, # mutable-ok: transform_request may write into this per line
|
||||
headers={}, # mutable-ok: transform_request writes headers entries per line
|
||||
),
|
||||
)
|
||||
params["model"] = model
|
||||
|
|
|
|||
|
|
@ -81,6 +81,7 @@ async def test_alist_batches_gets_list_url_with_pagination_and_headers(handler,
|
|||
assert headers["x-api-key"] == "sk-ant-test"
|
||||
assert headers["anthropic-version"] == "2023-06-01"
|
||||
assert headers["anthropic-beta"] == "message-batches-2024-09-24"
|
||||
assert call_kwargs["timeout"] == 60.0
|
||||
|
||||
assert result["object"] == "list"
|
||||
assert result["has_more"] is False
|
||||
|
|
@ -102,6 +103,7 @@ async def test_acancel_batch_posts_to_cancel_url_and_maps_response(handler, patc
|
|||
_, call_kwargs = fake_client.post.call_args
|
||||
assert call_kwargs["url"] == "https://api.anthropic.com/v1/messages/batches/msgbatch_abc/cancel"
|
||||
assert call_kwargs["headers"]["x-api-key"] == "sk-ant-test"
|
||||
assert call_kwargs["timeout"] == 60.0
|
||||
|
||||
assert isinstance(batch, LiteLLMBatch)
|
||||
assert batch.id == "msgbatch_abc"
|
||||
|
|
|
|||
|
|
@ -111,8 +111,42 @@ def test_openai_lines_translate_to_anthropic_params():
|
|||
assert first["params"]["system"]
|
||||
assert all(m["role"] != "system" for m in first["params"]["messages"])
|
||||
# max_tokens is required by Anthropic and defaulted even when absent
|
||||
assert second["params"]["max_tokens"]
|
||||
assert second["params"]["max_tokens"] == 64000
|
||||
assert second["params"]["model"] == "claude-sonnet-4-5"
|
||||
# params are isolated per line: req-1's temperature must not leak into req-2
|
||||
assert "temperature" not in second["params"]
|
||||
|
||||
|
||||
def test_openai_lines_params_do_not_leak_across_calls():
|
||||
transform_openai_batch_lines_to_anthropic_requests(
|
||||
[
|
||||
{
|
||||
"custom_id": "req-1",
|
||||
"method": "POST",
|
||||
"url": "/v1/chat/completions",
|
||||
"body": {
|
||||
"model": "ignored",
|
||||
"messages": [{"role": "user", "content": "Hi"}],
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 7,
|
||||
},
|
||||
}
|
||||
],
|
||||
model="claude-sonnet-4-5",
|
||||
)
|
||||
(second_run,) = transform_openai_batch_lines_to_anthropic_requests(
|
||||
[
|
||||
{
|
||||
"custom_id": "req-2",
|
||||
"method": "POST",
|
||||
"url": "/v1/chat/completions",
|
||||
"body": {"model": "ignored", "messages": [{"role": "user", "content": "Hello"}]},
|
||||
}
|
||||
],
|
||||
model="claude-sonnet-4-5",
|
||||
)
|
||||
assert "temperature" not in second_run["params"]
|
||||
assert second_run["params"]["max_tokens"] == 64000
|
||||
|
||||
|
||||
def test_openai_line_missing_custom_id_raises():
|
||||
|
|
@ -234,6 +268,44 @@ def test_cancel_batch_response_canceling_maps_to_cancelling(config):
|
|||
assert batch.cancelling_at == 1727173800
|
||||
|
||||
|
||||
def test_ended_after_cancel_initiated_maps_to_cancelled(config):
|
||||
raw = _response(
|
||||
{
|
||||
"id": "msgbatch_abc",
|
||||
"processing_status": "ended",
|
||||
"created_at": "2024-09-24T10:00:00Z",
|
||||
"cancel_initiated_at": "2024-09-24T10:30:00Z",
|
||||
"ended_at": "2024-09-24T10:35:00Z",
|
||||
"request_counts": {"canceled": 1, "succeeded": 0},
|
||||
}
|
||||
)
|
||||
batch = config.transform_retrieve_batch_response(
|
||||
model=None, raw_response=raw, logging_obj=MagicMock(), litellm_params={}
|
||||
)
|
||||
assert batch.status == "cancelled"
|
||||
assert batch.cancelled_at == 1727174100
|
||||
assert batch.cancelling_at == 1727173800
|
||||
assert batch.completed_at is None
|
||||
|
||||
|
||||
def test_ended_without_cancel_maps_to_completed(config):
|
||||
raw = _response(
|
||||
{
|
||||
"id": "msgbatch_abc",
|
||||
"processing_status": "ended",
|
||||
"created_at": "2024-09-24T10:00:00Z",
|
||||
"ended_at": "2024-09-24T10:35:00Z",
|
||||
"request_counts": {"succeeded": 2},
|
||||
}
|
||||
)
|
||||
batch = config.transform_retrieve_batch_response(
|
||||
model=None, raw_response=raw, logging_obj=MagicMock(), litellm_params={}
|
||||
)
|
||||
assert batch.status == "completed"
|
||||
assert batch.completed_at == 1727174100
|
||||
assert batch.cancelled_at is None
|
||||
|
||||
|
||||
# =========================================================================== #
|
||||
# transform_anthropic_batch_result_line
|
||||
# =========================================================================== #
|
||||
|
|
|
|||
|
|
@ -376,8 +376,8 @@ def test_transform_retrieve_response_canceling_maps_to_cancelling(config):
|
|||
# "canceling" -> OpenAI "cancelling".
|
||||
assert batch.status == "cancelling"
|
||||
assert batch.cancelling_at == 1727173800
|
||||
# cancelled_at = ended_at when canceling and ended_at present.
|
||||
assert batch.cancelled_at == 1727174700
|
||||
# cancelled_at stays unset while the batch is still cancelling
|
||||
assert batch.cancelled_at is None
|
||||
assert batch.completed_at is None
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue