mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(bedrock): bill Titan embedding batch output tokens
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
db37977307
commit
73a2a838ca
2 changed files with 52 additions and 10 deletions
|
|
@ -669,6 +669,10 @@ def _get_batch_job_usage_from_response_body(
|
|||
"""
|
||||
Get the tokens of a batch job from the response body
|
||||
"""
|
||||
if custom_llm_provider == "bedrock":
|
||||
embedding_usage: Final = _bedrock_embedding_usage(response_body)
|
||||
if embedding_usage is not None:
|
||||
return embedding_usage
|
||||
if custom_llm_provider in ("anthropic", "bedrock"):
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
|
||||
|
|
@ -697,6 +701,13 @@ def _get_batch_job_usage_from_response_body(
|
|||
return usage
|
||||
|
||||
|
||||
def _bedrock_embedding_usage(response_body: Mapping[str, Any]) -> Usage | None:
|
||||
token_count: Final = response_body.get("inputTextTokenCount")
|
||||
if not isinstance(token_count, int):
|
||||
return None
|
||||
return Usage(prompt_tokens=token_count, completion_tokens=0, total_tokens=token_count)
|
||||
|
||||
|
||||
def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[str, Any]) -> Mapping[str, Any]:
|
||||
"""
|
||||
Get the ``result`` object from a line of an Anthropic message batch results JSONL file.
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ import pytest
|
|||
import respx
|
||||
from openai.types.batch import BatchRequestCounts
|
||||
|
||||
|
||||
import litellm
|
||||
import litellm.batches.batch_utils as bu
|
||||
from litellm.types.utils import LiteLLMBatch, Usage
|
||||
|
|
@ -645,9 +644,7 @@ async def test_calculate_vertex_disable_transform_needs_model_name(monkeypatch):
|
|||
lambda content, model: pytest.fail("raw vertex path should not run"),
|
||||
)
|
||||
|
||||
result = await bu.calculate_batch_cost_and_usage(
|
||||
file_content_dictionary=[], custom_llm_provider="vertex_ai"
|
||||
)
|
||||
result = await bu.calculate_batch_cost_and_usage(file_content_dictionary=[], custom_llm_provider="vertex_ai")
|
||||
assert result.cost == 0.0
|
||||
assert result.usage.total_tokens == 0
|
||||
assert result.models == []
|
||||
|
|
@ -1282,6 +1279,7 @@ async def test_handle_completed_batch_no_output_file_is_zero(monkeypatch):
|
|||
result set - zero cost, zero usage, no models - instead of letting the file
|
||||
fetch raise "Output file id is None" on every aretrieve_batch logging poll.
|
||||
"""
|
||||
|
||||
# The output-file fetch must not even be attempted when there is no output file.
|
||||
async def _must_not_fetch(*args, **kwargs):
|
||||
pytest.fail("_fetch_batch_output_file_content should not be called")
|
||||
|
|
@ -1408,7 +1406,10 @@ def test_anthropic_response_body_is_result_message():
|
|||
|
||||
|
||||
def test_anthropic_usage_conversion_includes_cache_tokens():
|
||||
body = {"model": "claude-sonnet-4-5-20250929", "usage": _anthropic_usage(1000, 200, cache_creation=2000, cache_read=8000)}
|
||||
body = {
|
||||
"model": "claude-sonnet-4-5-20250929",
|
||||
"usage": _anthropic_usage(1000, 200, cache_creation=2000, cache_read=8000),
|
||||
}
|
||||
usage = bu._get_batch_job_usage_from_response_body(body, custom_llm_provider="anthropic")
|
||||
assert usage.prompt_tokens == 11000
|
||||
assert usage.completion_tokens == 200
|
||||
|
|
@ -1423,7 +1424,9 @@ def test_bedrock_model_output_line_success_check():
|
|||
"modelOutput": {"model": "claude-sonnet-4-6", "usage": {"input_tokens": 13, "output_tokens": 5}},
|
||||
}
|
||||
assert bu._batch_response_was_successful(row, custom_llm_provider="bedrock") is True
|
||||
assert bu._get_response_from_batch_job_output_file(row, custom_llm_provider="bedrock")["model"] == "claude-sonnet-4-6"
|
||||
assert (
|
||||
bu._get_response_from_batch_job_output_file(row, custom_llm_provider="bedrock")["model"] == "claude-sonnet-4-6"
|
||||
)
|
||||
|
||||
|
||||
def test_bedrock_cost_uses_deployment_model_name():
|
||||
|
|
@ -1477,7 +1480,13 @@ def test_total_usage_without_cache_tokens_has_no_prompt_details(monkeypatch):
|
|||
rows = [
|
||||
{
|
||||
"custom_id": "req-1",
|
||||
"response": {"status_code": 200, "body": {"model": "gpt-5.2", "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}}},
|
||||
"response": {
|
||||
"status_code": 200,
|
||||
"body": {
|
||||
"model": "gpt-5.2",
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
result = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai")
|
||||
|
|
@ -1519,7 +1528,9 @@ def test_anthropic_cost_without_model_info_uses_batch_cost_calculator(monkeypatc
|
|||
lambda **kw: pytest.fail("anthropic rows must not go through completion_cost"),
|
||||
)
|
||||
|
||||
result = bu._aggregate_batch_cost_usage_models(entries=[_anthropic_succeeded_row()], custom_llm_provider="anthropic")
|
||||
result = bu._aggregate_batch_cost_usage_models(
|
||||
entries=[_anthropic_succeeded_row()], custom_llm_provider="anthropic"
|
||||
)
|
||||
|
||||
assert result.cost == pytest.approx(0.3)
|
||||
assert seen[0]["model"] == "claude-sonnet-4-5-20250929"
|
||||
|
|
@ -1554,7 +1565,11 @@ async def test_calculate_batch_cost_and_usage_anthropic_end_to_end():
|
|||
)
|
||||
|
||||
assert result.cost == pytest.approx(1000 * 3e-6 / 2 + 8000 * 3e-7 / 2 + 2000 * 3.75e-6 / 2 + 200 * 15e-6 / 2)
|
||||
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (11000, 200, 11200)
|
||||
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (
|
||||
11000,
|
||||
200,
|
||||
11200,
|
||||
)
|
||||
assert result.models == ["claude-sonnet-4-5"]
|
||||
|
||||
|
||||
|
|
@ -1719,7 +1734,10 @@ async def test_handle_completed_batch_honors_deployment_pricing(monkeypatch) ->
|
|||
|
||||
|
||||
def test_bedrock_converse_shaped_batch_usage_is_parsed():
|
||||
body = {"model": "us.amazon.nova-lite-v1:0", "usage": {"inputTokens": 2202, "outputTokens": 540, "totalTokens": 2742}}
|
||||
body = {
|
||||
"model": "us.amazon.nova-lite-v1:0",
|
||||
"usage": {"inputTokens": 2202, "outputTokens": 540, "totalTokens": 2742},
|
||||
}
|
||||
usage = bu._get_batch_job_usage_from_response_body(body, custom_llm_provider="bedrock")
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (2202, 540, 2742)
|
||||
|
||||
|
|
@ -1755,6 +1773,18 @@ def test_bedrock_anthropic_shaped_batch_usage_still_parsed():
|
|||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (18, 10, 28)
|
||||
|
||||
|
||||
def test_bedrock_embedding_batch_usage_is_parsed():
|
||||
body = {"embedding": [0.1, 0.2], "inputTextTokenCount": 17}
|
||||
usage = bu._get_batch_job_usage_from_response_body(body, custom_llm_provider="bedrock")
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (17, 0, 17)
|
||||
|
||||
|
||||
def test_bedrock_batch_usage_without_usage_or_embedding_tokens_is_zero():
|
||||
body = {"embedding": [0.1, 0.2]}
|
||||
usage = bu._get_batch_job_usage_from_response_body(body, custom_llm_provider="bedrock")
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (0, 0, 0)
|
||||
|
||||
|
||||
def test_unparsable_bedrock_batch_usage_warns(caplog):
|
||||
"""An unrecognized usage shape must be visible, not a silent $0."""
|
||||
body = {"model": "amazon.titan-text-lite-v1", "usage": {"inputTextTokenCount": 42}}
|
||||
|
|
@ -1769,6 +1799,7 @@ def test_unparsable_bedrock_batch_usage_warns(caplog):
|
|||
# batch_cost_is_final
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _retrieved_batch(
|
||||
status: str, output_file_id: str | None = None, counts: BatchRequestCounts | None = None
|
||||
) -> LiteLLMBatch:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue