diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index ee7745d0add..93b8634c74c 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -338,7 +338,7 @@ class CheckBatchCost: # Pass deployment model_info so custom batch pricing # (input_cost_per_token_batches etc.) is used for cost calc - deployment_model_info = deployment_info.model_info.model_dump() if deployment_info.model_info else {} + deployment_model_info = deployment_info.model_info.model_dump() if deployment_info.model_info else None batch_cost, batch_usage, batch_models = ( await calculate_batch_cost_and_usage( file_content_dictionary=file_content_as_dict, diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index 74e753b09ea..f0ffeadd9fa 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -25,6 +25,17 @@ async def calculate_batch_cost_and_usage( deployment-specific pricing (e.g. input_cost_per_token_batches) is used instead of the global cost map. """ + # Translate Anthropic JSONL format to OpenAI format so the shared + # cost/usage helpers work uniformly across providers. + # Anthropic uses {result: {type, message: {usage: {input_tokens, output_tokens}}}} + # but helpers expect {response: {status_code, body: {usage: {prompt_tokens, ...}}}} + # Ref: https://github.com/BerriAI/litellm/issues/27944 + if custom_llm_provider == "anthropic": + file_content_dictionary = [ + _translate_anthropic_batch_item_to_openai(item) + for item in file_content_dictionary + ] + batch_cost = _batch_cost_calculator( custom_llm_provider=custom_llm_provider, file_content_dictionary=file_content_dictionary, @@ -515,6 +526,51 @@ def _get_response_from_batch_job_output_file(batch_job_output_file: dict) -> Any return _response_body +def _translate_anthropic_batch_item_to_openai(item: dict) -> dict: + """ + Translate a single Anthropic batch result JSONL item to OpenAI format + so the shared cost/usage helpers can process it uniformly. + + Anthropic format: + {"custom_id": "x", "result": {"type": "succeeded", + "message": {"model": "claude-...", + "usage": {"input_tokens": N, "output_tokens": M}}}} + + OpenAI format expected by _get_response_from_batch_job_output_file: + {"custom_id": "x", "response": {"status_code": 200, + "body": {"model": "claude-...", + "usage": {"prompt_tokens": N, "completion_tokens": M, + "total_tokens": N+M}}}} + + Ref: https://github.com/BerriAI/litellm/issues/27944 + """ + result = item.get("result", {}) + if result.get("type") != "succeeded": + # failed/errored items return as-is so _batch_response_was_successful + # correctly treats them as non-200 + return item + + message = result.get("message", {}) + anthropic_usage = message.get("usage", {}) + prompt_tokens = anthropic_usage.get("input_tokens", 0) + completion_tokens = anthropic_usage.get("output_tokens", 0) + + return { + "custom_id": item.get("custom_id"), + "response": { + "status_code": 200, + "body": { + "model": message.get("model", ""), + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + }, + }, + }, + } + + def _batch_response_was_successful(batch_job_output_file: dict) -> bool: """ Check if the batch job response status == 200 diff --git a/litellm/llms/anthropic/files/transformation.py b/litellm/llms/anthropic/files/transformation.py index ea9bf00f505..e9ffd853692 100644 --- a/litellm/llms/anthropic/files/transformation.py +++ b/litellm/llms/anthropic/files/transformation.py @@ -271,6 +271,13 @@ class AnthropicFilesConfig(BaseFilesConfig): AnthropicModelInfo.get_api_base(litellm_params.get("api_base")) or ANTHROPIC_FILES_API_BASE ) + if file_id and file_id.startswith("msgbatch_"): + encoded_batch_id = encode_url_path_segment(file_id, field_name="file_id") + return ( + f"{api_base.rstrip('/')}/v1/messages/batches/{encoded_batch_id}/results", + {}, + ) + encoded_file_id = encode_url_path_segment(file_id, field_name="file_id") return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}/content", {} diff --git a/tests/test_litellm/llms/anthropic/files/test_anthropic_files_transformation.py b/tests/test_litellm/llms/anthropic/files/test_anthropic_files_transformation.py index 9fc4981510f..20a40e60dce 100644 --- a/tests/test_litellm/llms/anthropic/files/test_anthropic_files_transformation.py +++ b/tests/test_litellm/llms/anthropic/files/test_anthropic_files_transformation.py @@ -317,6 +317,29 @@ class TestAnthropicFilesConfig: litellm_params={}, ) + def test_transform_file_content_request_msgbatch_routes_to_batch_endpoint(self): + url, params = self.config.transform_file_content_request( + file_content_request={"file_id": "msgbatch_abc123"}, + optional_params={}, + litellm_params={}, + ) + assert url == f"{ANTHROPIC_FILES_API_BASE}/v1/messages/batches/msgbatch_abc123/results" + assert params == {} + + def test_transform_file_content_request_msgbatch_encodes_path_traversal(self): + url, params = self.config.transform_file_content_request( + file_content_request={ + "file_id": "msgbatch_x/../../../v1/messages/batches?limit=1#frag" + }, + optional_params={}, + litellm_params={}, + ) + assert ( + url + == f"{ANTHROPIC_FILES_API_BASE}/v1/messages/batches/msgbatch_x%2F..%2F..%2F..%2Fv1%2Fmessages%2Fbatches%3Flimit%3D1%23frag/results" + ) + assert params == {} + def test_transform_file_content_response(self): mock_response = Mock(spec=httpx.Response) result = self.config.transform_file_content_response( diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_batch_translation.py b/tests/test_litellm/llms/anthropic/test_anthropic_batch_translation.py new file mode 100644 index 00000000000..9e3f1cd4ff8 --- /dev/null +++ b/tests/test_litellm/llms/anthropic/test_anthropic_batch_translation.py @@ -0,0 +1,51 @@ +""" +Tests for Anthropic batch JSONL translation. +Ref: https://github.com/BerriAI/litellm/issues/27944 +""" +import pytest +from litellm.batches.batch_utils import _translate_anthropic_batch_item_to_openai + + +def test_translate_succeeded_item(): + item = { + "custom_id": "req-1", + "result": { + "type": "succeeded", + "message": { + "model": "claude-opus-4-7", + "usage": {"input_tokens": 100, "output_tokens": 50}, + }, + }, + } + result = _translate_anthropic_batch_item_to_openai(item) + assert result["response"]["status_code"] == 200 + body = result["response"]["body"] + assert body["model"] == "claude-opus-4-7" + assert body["usage"]["prompt_tokens"] == 100 + assert body["usage"]["completion_tokens"] == 50 + assert body["usage"]["total_tokens"] == 150 + assert result["custom_id"] == "req-1" + + +def test_translate_failed_item_passes_through(): + item = { + "custom_id": "req-2", + "result": {"type": "errored", "error": {"type": "server_error"}}, + } + result = _translate_anthropic_batch_item_to_openai(item) + # failed items pass through unchanged; no response.status_code == 200 + assert result == item + + +def test_translate_zero_usage(): + item = { + "custom_id": "req-3", + "result": { + "type": "succeeded", + "message": {"model": "claude-haiku-4-5", "usage": {}}, + }, + } + result = _translate_anthropic_batch_item_to_openai(item) + assert result["response"]["body"]["usage"]["prompt_tokens"] == 0 + assert result["response"]["body"]["usage"]["completion_tokens"] == 0 + assert result["response"]["body"]["usage"]["total_tokens"] == 0