fix(bedrock): bill Titan embedding batch output tokens

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Devin AI 2026-09-18 04:41:45 +00:00
parent db37977307
commit 73a2a838ca
2 changed files with 52 additions and 10 deletions

View file

@ -669,6 +669,10 @@ def _get_batch_job_usage_from_response_body(
"""
Get the tokens of a batch job from the response body
"""
if custom_llm_provider == "bedrock":
embedding_usage: Final = _bedrock_embedding_usage(response_body)
if embedding_usage is not None:
return embedding_usage
if custom_llm_provider in ("anthropic", "bedrock"):
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
@ -697,6 +701,13 @@ def _get_batch_job_usage_from_response_body(
return usage
def _bedrock_embedding_usage(response_body: Mapping[str, Any]) -> Usage | None:
token_count: Final = response_body.get("inputTextTokenCount")
if not isinstance(token_count, int):
return None
return Usage(prompt_tokens=token_count, completion_tokens=0, total_tokens=token_count)
def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[str, Any]) -> Mapping[str, Any]:
"""
Get the ``result`` object from a line of an Anthropic message batch results JSONL file.

View file

@ -23,7 +23,6 @@ import pytest
import respx
from openai.types.batch import BatchRequestCounts
import litellm
import litellm.batches.batch_utils as bu
from litellm.types.utils import LiteLLMBatch, Usage
@ -645,9 +644,7 @@ async def test_calculate_vertex_disable_transform_needs_model_name(monkeypatch):
lambda content, model: pytest.fail("raw vertex path should not run"),
)
result = await bu.calculate_batch_cost_and_usage(
file_content_dictionary=[], custom_llm_provider="vertex_ai"
)
result = await bu.calculate_batch_cost_and_usage(file_content_dictionary=[], custom_llm_provider="vertex_ai")
assert result.cost == 0.0
assert result.usage.total_tokens == 0
assert result.models == []
@ -1282,6 +1279,7 @@ async def test_handle_completed_batch_no_output_file_is_zero(monkeypatch):
result set - zero cost, zero usage, no models - instead of letting the file
fetch raise "Output file id is None" on every aretrieve_batch logging poll.
"""
# The output-file fetch must not even be attempted when there is no output file.
async def _must_not_fetch(*args, **kwargs):
pytest.fail("_fetch_batch_output_file_content should not be called")
@ -1408,7 +1406,10 @@ def test_anthropic_response_body_is_result_message():
def test_anthropic_usage_conversion_includes_cache_tokens():
body = {"model": "claude-sonnet-4-5-20250929", "usage": _anthropic_usage(1000, 200, cache_creation=2000, cache_read=8000)}
body = {
"model": "claude-sonnet-4-5-20250929",
"usage": _anthropic_usage(1000, 200, cache_creation=2000, cache_read=8000),
}
usage = bu._get_batch_job_usage_from_response_body(body, custom_llm_provider="anthropic")
assert usage.prompt_tokens == 11000
assert usage.completion_tokens == 200
@ -1423,7 +1424,9 @@ def test_bedrock_model_output_line_success_check():
"modelOutput": {"model": "claude-sonnet-4-6", "usage": {"input_tokens": 13, "output_tokens": 5}},
}
assert bu._batch_response_was_successful(row, custom_llm_provider="bedrock") is True
assert bu._get_response_from_batch_job_output_file(row, custom_llm_provider="bedrock")["model"] == "claude-sonnet-4-6"
assert (
bu._get_response_from_batch_job_output_file(row, custom_llm_provider="bedrock")["model"] == "claude-sonnet-4-6"
)
def test_bedrock_cost_uses_deployment_model_name():
@ -1477,7 +1480,13 @@ def test_total_usage_without_cache_tokens_has_no_prompt_details(monkeypatch):
rows = [
{
"custom_id": "req-1",
"response": {"status_code": 200, "body": {"model": "gpt-5.2", "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}}},
"response": {
"status_code": 200,
"body": {
"model": "gpt-5.2",
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
},
},
}
]
result = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai")
@ -1519,7 +1528,9 @@ def test_anthropic_cost_without_model_info_uses_batch_cost_calculator(monkeypatc
lambda **kw: pytest.fail("anthropic rows must not go through completion_cost"),
)
result = bu._aggregate_batch_cost_usage_models(entries=[_anthropic_succeeded_row()], custom_llm_provider="anthropic")
result = bu._aggregate_batch_cost_usage_models(
entries=[_anthropic_succeeded_row()], custom_llm_provider="anthropic"
)
assert result.cost == pytest.approx(0.3)
assert seen[0]["model"] == "claude-sonnet-4-5-20250929"
@ -1554,7 +1565,11 @@ async def test_calculate_batch_cost_and_usage_anthropic_end_to_end():
)
assert result.cost == pytest.approx(1000 * 3e-6 / 2 + 8000 * 3e-7 / 2 + 2000 * 3.75e-6 / 2 + 200 * 15e-6 / 2)
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (11000, 200, 11200)
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (
11000,
200,
11200,
)
assert result.models == ["claude-sonnet-4-5"]
@ -1719,7 +1734,10 @@ async def test_handle_completed_batch_honors_deployment_pricing(monkeypatch) ->
def test_bedrock_converse_shaped_batch_usage_is_parsed():
body = {"model": "us.amazon.nova-lite-v1:0", "usage": {"inputTokens": 2202, "outputTokens": 540, "totalTokens": 2742}}
body = {
"model": "us.amazon.nova-lite-v1:0",
"usage": {"inputTokens": 2202, "outputTokens": 540, "totalTokens": 2742},
}
usage = bu._get_batch_job_usage_from_response_body(body, custom_llm_provider="bedrock")
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (2202, 540, 2742)
@ -1755,6 +1773,18 @@ def test_bedrock_anthropic_shaped_batch_usage_still_parsed():
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (18, 10, 28)
def test_bedrock_embedding_batch_usage_is_parsed():
body = {"embedding": [0.1, 0.2], "inputTextTokenCount": 17}
usage = bu._get_batch_job_usage_from_response_body(body, custom_llm_provider="bedrock")
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (17, 0, 17)
def test_bedrock_batch_usage_without_usage_or_embedding_tokens_is_zero():
body = {"embedding": [0.1, 0.2]}
usage = bu._get_batch_job_usage_from_response_body(body, custom_llm_provider="bedrock")
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (0, 0, 0)
def test_unparsable_bedrock_batch_usage_warns(caplog):
"""An unrecognized usage shape must be visible, not a silent $0."""
body = {"model": "amazon.titan-text-lite-v1", "usage": {"inputTextTokenCount": 42}}
@ -1769,6 +1799,7 @@ def test_unparsable_bedrock_batch_usage_warns(caplog):
# batch_cost_is_final
# --------------------------------------------------------------------------- #
def _retrieved_batch(
status: str, output_file_id: str | None = None, counts: BatchRequestCounts | None = None
) -> LiteLLMBatch: