scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-10 03:28:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 85

fix(anthropic): bill an uncostable partial pass-through stream at zero cost instead of dropping its usage

Browse source
This commit is contained in:
mateo-berri 2026-09-04 09:26:51 -07:00
parent 2f3d1ca575
commit 6c27754455
2 changed files with 29 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py
View file

@ -240,18 +240,28 @@ class AnthropicPassthroughLoggingHandler:
usage: Final = cast(Usage | None, getattr(partial_response, "usage", None))
if partial_response is None or usage is None:
return
litellm_logging_obj.record_partial_usage_for_failure(
usage=usage,
response_cost=AnthropicPassthroughLoggingHandler._cost_partial_stream_or_zero(
partial_response=partial_response, model=model, logging_obj=litellm_logging_obj
),
)
@staticmethod
def _cost_partial_stream_or_zero(
partial_response: ModelResponse | TextCompletionResponse, model: str, logging_obj: LiteLLMLoggingObj
) -> float:
try:
response_cost: Final = AnthropicPassthroughLoggingHandler._compute_response_cost(
return AnthropicPassthroughLoggingHandler._compute_response_cost(
litellm_model_response=partial_response,
model=AnthropicPassthroughLoggingHandler._resolve_costing_model(model, litellm_logging_obj),
logging_obj=litellm_logging_obj,
model=AnthropicPassthroughLoggingHandler._resolve_costing_model(model, logging_obj),
logging_obj=logging_obj,
)
except Exception as e: # noqa: BLE001 # an uncostable partial stream must still log as a failure
except Exception as e: # noqa: BLE001 # an uncostable partial stream still bills its tokens, at zero cost
verbose_proxy_logger.warning(
"Anthropic passthrough: could not cost the partial usage of a failed stream (model=%s): %s", model, e
)
return
litellm_logging_obj.record_partial_usage_for_failure(usage=usage, response_cost=response_cost)
return 0.0
@staticmethod
def _compute_response_cost(

13
tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py
View file

@ -2505,6 +2505,19 @@ class TestRecordPartialUsageForFailure:
assert usage.prompt_tokens == 52
assert logging_obj.model_call_details["response_cost"] > 0
def test_stashes_partial_usage_at_zero_cost_when_model_is_unpriced(self):
logging_obj = self._make_logging_obj()
AnthropicPassthroughLoggingHandler.record_partial_usage_for_failure(
litellm_logging_obj=logging_obj,
request_body={"model": "claude-unpriced-test-model", "stream": True},
all_chunks=self._interrupted_chunks(),
)
usage = logging_obj.model_call_details["combined_usage_object"]
assert usage.prompt_tokens == 52
assert logging_obj.model_call_details["response_cost"] == 0.0
def test_leaves_logging_obj_untouched_when_nothing_streamed(self):
logging_obj = self._make_logging_obj()