From a798aca9a0980793c9ca8df32671d826754e9696 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:19:26 -0700 Subject: [PATCH] test(interactions): cover usage transformation, cost calculation, and success-handler billing --- .../test_usage_object_transformation.py | 73 ++++++++++ .../test_litellm_logging.py | 137 ++++++++++++++++++ tests/test_litellm/test_cost_calculator.py | 41 ++++++ 3 files changed, 251 insertions(+) create mode 100644 tests/test_litellm/litellm_core_utils/llm_cost_calc/test_usage_object_transformation.py diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_usage_object_transformation.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_usage_object_transformation.py new file mode 100644 index 00000000000..3dd76c20ea3 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_usage_object_transformation.py @@ -0,0 +1,73 @@ +from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import ( + InteractionsUsageObjectTransformation, +) + + +def _omni_flash_video_usage() -> dict: + return { + "total_tokens": 18247, + "total_input_tokens": 16, + "input_tokens_by_modality": [{"modality": "text", "tokens": 16}], + "total_cached_tokens": 0, + "total_output_tokens": 17937, + "output_tokens_by_modality": [ + {"modality": "text", "tokens": 561}, + {"modality": "video", "tokens": 17376}, + ], + "total_tool_use_tokens": 0, + "total_thought_tokens": 294, + } + + +def test_is_interactions_usage_dict(): + assert InteractionsUsageObjectTransformation.is_interactions_usage_dict(_omni_flash_video_usage()) + assert InteractionsUsageObjectTransformation.is_interactions_usage_dict({"total_output_tokens": 5}) + assert not InteractionsUsageObjectTransformation.is_interactions_usage_dict( + {"prompt_tokens": 1, "completion_tokens": 2} + ) + assert not InteractionsUsageObjectTransformation.is_interactions_usage_dict( + {"input_tokens": 1, "output_tokens": 2} + ) + + +def test_transform_interactions_usage_maps_modalities(): + usage = InteractionsUsageObjectTransformation.transform_interactions_usage_to_chat_usage( + _omni_flash_video_usage() + ) + assert usage.prompt_tokens == 16 + assert usage.completion_tokens == 17937 + 294 + assert usage.total_tokens == 18247 + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.text_tokens == 16 + assert usage.prompt_tokens_details.cached_tokens == 0 + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 294 + assert usage.completion_tokens_details.text_tokens == 561 + assert usage.completion_tokens_details.video_tokens == 17376 + + +def test_transform_interactions_usage_reasoning_key_fallback(): + usage = InteractionsUsageObjectTransformation.transform_interactions_usage_to_chat_usage( + {"total_input_tokens": 10, "total_output_tokens": 20, "total_reasoning_tokens": 5} + ) + assert usage.prompt_tokens == 10 + assert usage.completion_tokens == 25 + assert usage.total_tokens == 35 + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 5 + + +def test_transform_interactions_usage_cached_tokens(): + usage = InteractionsUsageObjectTransformation.transform_interactions_usage_to_chat_usage( + {"total_input_tokens": 100, "total_cached_tokens": 40, "total_output_tokens": 10} + ) + assert usage.prompt_tokens == 100 + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.cached_tokens == 40 + + +def test_transform_interactions_usage_empty(): + usage = InteractionsUsageObjectTransformation.transform_interactions_usage_to_chat_usage(None) + assert usage.prompt_tokens == 0 + assert usage.completion_tokens == 0 + assert usage.total_tokens == 0 diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index ade2c677745..2152d6be983 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -3773,3 +3773,140 @@ def test_zero_token_video_usage_preserves_duration_seconds(logging_obj): assert payload["metadata"]["usage_object"]["duration_seconds"] == 4.0 assert payload["total_tokens"] == 0 assert payload["completion_tokens"] == 0 + + +def _register_fake_interactions_model(): + litellm.register_model( + model_cost={ + "gemini/fake-interactions-model": { + "litellm_provider": "gemini", + "mode": "chat", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 2.5e-6, + } + } + ) + + +def _interactions_usage_dict(): + return { + "total_tokens": 871, + "total_input_tokens": 16, + "input_tokens_by_modality": [{"modality": "text", "tokens": 16}], + "total_cached_tokens": 0, + "total_output_tokens": 561, + "output_tokens_by_modality": [{"modality": "text", "tokens": 561}], + "total_tool_use_tokens": 0, + "total_thought_tokens": 294, + } + + +_EXPECTED_INTERACTIONS_COST = 16 * 3e-7 + (561 + 294) * 2.5e-6 + + +def _interactions_logging_obj(call_type: str, stream: bool) -> LitellmLogging: + import datetime + + logging_obj = LitellmLogging( + model="gemini/fake-interactions-model", + messages=[{"role": "user", "content": "hi"}], + stream=stream, + call_type=call_type, + start_time=datetime.datetime.now(), + litellm_call_id="interactions-cost-test", + function_id="interactions-cost-test", + ) + logging_obj.update_from_kwargs( + kwargs={}, + model="gemini/fake-interactions-model", + optional_params={}, + litellm_params={"litellm_call_id": "interactions-cost-test"}, + custom_llm_provider="gemini", + ) + return logging_obj + + +def test_interactions_create_success_tracks_response_cost(): + import datetime + + from litellm.types.interactions import InteractionsAPIResponse + + _register_fake_interactions_model() + logging_obj = _interactions_logging_obj(call_type="acreate_interaction", stream=False) + result = InteractionsAPIResponse( + id="interaction_123", + model="fake-interactions-model", + status="completed", + steps=[], + usage=_interactions_usage_dict(), + ) + + logging_obj._success_handler_helper_fn( + result=result, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + cache_hit=False, + ) + + assert logging_obj.model_call_details["response_cost"] == pytest.approx(_EXPECTED_INTERACTIONS_COST) + standard_logging_object = logging_obj.model_call_details["standard_logging_object"] + assert standard_logging_object["prompt_tokens"] == 16 + assert standard_logging_object["completion_tokens"] == 561 + 294 + assert standard_logging_object["total_tokens"] == 871 + assert standard_logging_object["response_cost"] == pytest.approx(_EXPECTED_INTERACTIONS_COST) + + +def test_interactions_streaming_completed_event_tracks_response_cost(): + import datetime + + from litellm.types.interactions import InteractionsAPIStreamingResponse + + _register_fake_interactions_model() + logging_obj = _interactions_logging_obj(call_type="acreate_interaction", stream=True) + result = InteractionsAPIStreamingResponse( + event_type="interaction.completed", + interaction={ + "id": "interaction_123", + "model": "fake-interactions-model", + "status": "completed", + "steps": [], + "usage": _interactions_usage_dict(), + }, + ) + + logging_obj._success_handler_helper_fn( + result=result, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + cache_hit=False, + ) + + assert logging_obj.model_call_details["response_cost"] == pytest.approx(_EXPECTED_INTERACTIONS_COST) + standard_logging_object = logging_obj.model_call_details["standard_logging_object"] + assert standard_logging_object["prompt_tokens"] == 16 + assert standard_logging_object["completion_tokens"] == 561 + 294 + + +def test_interactions_get_poll_does_not_bill(): + import datetime + + from litellm.types.interactions import InteractionsAPIResponse + + _register_fake_interactions_model() + logging_obj = _interactions_logging_obj(call_type="aget_interaction", stream=False) + result = InteractionsAPIResponse( + id="interaction_123", + model="fake-interactions-model", + status="completed", + steps=[], + usage=_interactions_usage_dict(), + ) + + logging_obj._success_handler_helper_fn( + result=result, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + cache_hit=False, + ) + + assert logging_obj.model_call_details.get("response_cost") is None diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 9636db4f4cd..83e125b9b74 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -3479,3 +3479,44 @@ def test_batch_cost_calculator_cache_creation_falls_back_to_input_rate(): ) assert prompt_cost == pytest.approx((1000 * 3e-6 + 8000 * 3e-7 + 2000 * 3e-6) / 2) + + +def test_completion_cost_interactions_api_response(): + from litellm.types.interactions import InteractionsAPIResponse + + litellm.register_model( + model_cost={ + "gemini/fake-interactions-model": { + "litellm_provider": "gemini", + "mode": "chat", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 2.5e-6, + } + } + ) + + response = InteractionsAPIResponse( + id="interaction_123", + model="fake-interactions-model", + status="completed", + steps=[], + usage={ + "total_tokens": 871, + "total_input_tokens": 16, + "input_tokens_by_modality": [{"modality": "text", "tokens": 16}], + "total_cached_tokens": 0, + "total_output_tokens": 561, + "output_tokens_by_modality": [{"modality": "text", "tokens": 561}], + "total_tool_use_tokens": 0, + "total_thought_tokens": 294, + }, + ) + + cost = completion_cost( + completion_response=response, + model="gemini/fake-interactions-model", + custom_llm_provider="gemini", + call_type="acreate_interaction", + ) + + assert cost == pytest.approx(16 * 3e-7 + (561 + 294) * 2.5e-6)