diff --git a/litellm/llms/fireworks_ai/cost_calculator.py b/litellm/llms/fireworks_ai/cost_calculator.py index b704b599676..5eab965bae6 100644 --- a/litellm/llms/fireworks_ai/cost_calculator.py +++ b/litellm/llms/fireworks_ai/cost_calculator.py @@ -77,7 +77,7 @@ def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]: ) ## CALCULATE INPUT COST - input_cost_per_token = model_info["input_cost_per_token"] + input_cost_per_token: float = model_info["input_cost_per_token"] or 0.0 prompt_cost: float = usage["prompt_tokens"] * input_cost_per_token ## ADJUST FOR CACHE READ TOKENS diff --git a/tests/local_testing/test_completion_cost.py b/tests/local_testing/test_completion_cost.py index f4d8208ebd3..618287e1955 100644 --- a/tests/local_testing/test_completion_cost.py +++ b/tests/local_testing/test_completion_cost.py @@ -4,7 +4,9 @@ import traceback import litellm.cost_calculator -sys.path.insert(0, os.path.abspath("../..")) # Adds the parent directory to the system path +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path import asyncio import os import time @@ -213,17 +215,23 @@ def test_cost_ft_gpt_35(): usage=Usage(prompt_tokens=21, completion_tokens=17, total_tokens=38), ) - cost = litellm.completion_cost(completion_response=resp, custom_llm_provider="openai") + cost = litellm.completion_cost( + completion_response=resp, custom_llm_provider="openai" + ) print("\n Calculated Cost for ft:gpt-3.5", cost) input_cost = model_cost["ft:gpt-3.5-turbo"]["input_cost_per_token"] output_cost = model_cost["ft:gpt-3.5-turbo"]["output_cost_per_token"] print(input_cost, output_cost) - expected_cost = (input_cost * resp.usage.prompt_tokens) + (output_cost * resp.usage.completion_tokens) + expected_cost = (input_cost * resp.usage.prompt_tokens) + ( + output_cost * resp.usage.completion_tokens + ) print("\n Excpected cost", expected_cost) assert cost == expected_cost except Exception as e: print(f"Error: {e}") - pytest.fail(f"Cost Calc failed for ft:gpt-3.5. Expected {expected_cost}, Calculated cost {cost}") + pytest.fail( + f"Cost Calc failed for ft:gpt-3.5. Expected {expected_cost}, Calculated cost {cost}" + ) # test_cost_ft_gpt_35() @@ -252,11 +260,15 @@ def test_cost_azure_gpt_35(): usage=Usage(prompt_tokens=21, completion_tokens=17, total_tokens=38), ) - cost = litellm.completion_cost(completion_response=resp, model="azure/chatgpt-deployment-2") + cost = litellm.completion_cost( + completion_response=resp, model="azure/chatgpt-deployment-2" + ) print("\n Calculated Cost for azure/gpt-3.5-turbo", cost) input_cost = model_cost["azure/gpt-35-turbo"]["input_cost_per_token"] output_cost = model_cost["azure/gpt-35-turbo"]["output_cost_per_token"] - expected_cost = (input_cost * resp.usage.prompt_tokens) + (output_cost * resp.usage.completion_tokens) + expected_cost = (input_cost * resp.usage.prompt_tokens) + ( + output_cost * resp.usage.completion_tokens + ) print("\n Excpected cost", expected_cost) assert cost == expected_cost except Exception as e: @@ -291,7 +303,9 @@ def test_cost_azure_embedding(): assert cost == expected_cost except Exception as e: - pytest.fail(f"Cost Calc failed for azure/gpt-3.5-turbo. Expected {expected_cost}, Calculated cost {cost}") + pytest.fail( + f"Cost Calc failed for azure/gpt-3.5-turbo. Expected {expected_cost}, Calculated cost {cost}" + ) # test_cost_azure_embedding() @@ -301,7 +315,9 @@ def test_cost_bedrock_pricing_actual_calls(): litellm.set_verbose = True model = "anthropic.claude-3-5-sonnet-20240620-v1:0" messages = [{"role": "user", "content": "Hey, how's it going?"}] - response = litellm.completion(model=model, messages=messages, mock_response="hello cool one") + response = litellm.completion( + model=model, messages=messages, mock_response="hello cool one" + ) print("response", response) cost = litellm.completion_cost( @@ -332,7 +348,8 @@ def test_whisper_openai(): print(f"cost: {cost}") print(f"whisper dict: {litellm.model_cost['whisper-1']}") expected_cost = round( - litellm.model_cost["whisper-1"]["output_cost_per_second"] * _total_time_in_seconds, + litellm.model_cost["whisper-1"]["output_cost_per_second"] + * _total_time_in_seconds, 5, ) assert round(cost, 5) == round(expected_cost, 5) @@ -352,12 +369,15 @@ def test_whisper_azure(): _total_time_in_seconds = 3 setattr(transcription, "duration", _total_time_in_seconds) - cost = litellm.completion_cost(model="azure/azure-whisper", completion_response=transcription) + cost = litellm.completion_cost( + model="azure/azure-whisper", completion_response=transcription + ) print(f"cost: {cost}") print(f"whisper dict: {litellm.model_cost['whisper-1']}") expected_cost = round( - litellm.model_cost["whisper-1"]["output_cost_per_second"] * _total_time_in_seconds, + litellm.model_cost["whisper-1"]["output_cost_per_second"] + * _total_time_in_seconds, 5, ) assert round(cost, 5) == round(expected_cost, 5) @@ -388,7 +408,9 @@ def test_dalle_3_azure_cost_tracking(): response.usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} response._hidden_params = {"model": "dall-e-3", "model_id": None} print(f"response hidden params: {response._hidden_params}") - cost = litellm.completion_cost(completion_response=response, call_type="image_generation") + cost = litellm.completion_cost( + completion_response=response, call_type="image_generation" + ) assert cost > 0 @@ -420,7 +442,9 @@ def test_replicate_llama3_cost_tracking(): model="replicate/meta/meta-llama-3-8b-instruct", object="chat.completion", system_fingerprint=None, - usage=litellm.utils.Usage(prompt_tokens=48, completion_tokens=31, total_tokens=79), + usage=litellm.utils.Usage( + prompt_tokens=48, completion_tokens=31, total_tokens=79 + ), ) cost = litellm.completion_cost( completion_response=response, @@ -430,8 +454,14 @@ def test_replicate_llama3_cost_tracking(): print(f"cost: {cost}") cost = round(cost, 5) expected_cost = round( - litellm.model_cost["replicate/meta/meta-llama-3-8b-instruct"]["input_cost_per_token"] * 48 - + litellm.model_cost["replicate/meta/meta-llama-3-8b-instruct"]["output_cost_per_token"] * 31, + litellm.model_cost["replicate/meta/meta-llama-3-8b-instruct"][ + "input_cost_per_token" + ] + * 48 + + litellm.model_cost["replicate/meta/meta-llama-3-8b-instruct"][ + "output_cost_per_token" + ] + * 31, 5, ) assert cost == expected_cost @@ -602,14 +632,18 @@ def test_vertex_ai_medlm_completion_cost(): with pytest.raises(Exception) as e: model = "vertex_ai/medlm-medium" messages = [{"role": "user", "content": "Test MedLM completion cost."}] - predictive_cost = completion_cost(model=model, messages=messages, custom_llm_provider="vertex_ai") + predictive_cost = completion_cost( + model=model, messages=messages, custom_llm_provider="vertex_ai" + ) os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") model = "vertex_ai/medlm-medium" messages = [{"role": "user", "content": "Test MedLM completion cost."}] - predictive_cost = completion_cost(model=model, messages=messages, custom_llm_provider="vertex_ai") + predictive_cost = completion_cost( + model=model, messages=messages, custom_llm_provider="vertex_ai" + ) assert predictive_cost > 0 model = "vertex_ai/medlm-large" @@ -676,7 +710,9 @@ def test_vertex_ai_embedding_completion_cost(caplog): litellm.model_cost = litellm.get_model_cost_map(url="") text = "The quick brown fox jumps over the lazy dog." - input_tokens = litellm.token_counter(model="vertex_ai/text-embedding-004", text=text) + input_tokens = litellm.token_counter( + model="vertex_ai/text-embedding-004", text=text + ) model_info = litellm.get_model_info(model="vertex_ai/text-embedding-004") @@ -699,7 +735,10 @@ def test_vertex_ai_embedding_completion_cost(caplog): captured_logs = [rec.message for rec in caplog.records] for item in captured_logs: print("\nitem:{}\n".format(item)) - if "litellm.litellm_core_utils.llm_cost_calc.google.cost_per_character(): Exception occured " in item: + if ( + "litellm.litellm_core_utils.llm_cost_calc.google.cost_per_character(): Exception occured " + in item + ): raise Exception("Error log raised for calculating embedding cost") @@ -769,7 +808,9 @@ def test_vertex_ai_llama_predict_cost(): model = "meta/llama3-405b-instruct-maas" messages = [{"role": "user", "content": "Hey, hows it going???"}] custom_llm_provider = "vertex_ai" - predictive_cost = completion_cost(model=model, messages=messages, custom_llm_provider=custom_llm_provider) + predictive_cost = completion_cost( + model=model, messages=messages, custom_llm_provider=custom_llm_provider + ) assert predictive_cost == 0 @@ -783,7 +824,9 @@ def test_vertex_ai_mistral_predict_cost(usage): else: from openai.types.completion_usage import CompletionUsage - response_usage = CompletionUsage(prompt_tokens=32, completion_tokens=55, total_tokens=87) + response_usage = CompletionUsage( + prompt_tokens=32, completion_tokens=55, total_tokens=87 + ) response_object = ModelResponse( id="26c0ef045020429d9c5c9b078c01e564", choices=[ @@ -817,7 +860,9 @@ def test_vertex_ai_mistral_predict_cost(usage): assert predictive_cost > 0 -@pytest.mark.parametrize("model", ["openai/tts-1", "azure/tts-1", "openai/gpt-4o-mini-tts"]) +@pytest.mark.parametrize( + "model", ["openai/tts-1", "azure/tts-1", "openai/gpt-4o-mini-tts"] +) def test_completion_cost_tts(model): os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") @@ -919,7 +964,9 @@ def test_completion_cost_azure_common_deployment_name(): response._hidden_params["custom_llm_provider"] = "azure" print(response) - with patch.object(litellm.cost_calculator, "completion_cost", new=MagicMock()) as mock_client: + with patch.object( + litellm.cost_calculator, "completion_cost", new=MagicMock() + ) as mock_client: _ = litellm.response_cost_calculator( response_object=response, model="gpt-4-0314", @@ -979,7 +1026,9 @@ def test_completion_cost_prompt_caching(model, custom_llm_provider): cost_1 = completion_cost(model=model, completion_response=response_1) - _model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) + _model_info = litellm.get_model_info( + model=model, custom_llm_provider=custom_llm_provider + ) expected_cost = ( ( response_1.usage.prompt_tokens @@ -987,9 +1036,12 @@ def test_completion_cost_prompt_caching(model, custom_llm_provider): - response_1.usage.prompt_tokens_details.cache_creation_tokens ) * _model_info["input_cost_per_token"] - + (response_1.usage.prompt_tokens_details.cached_tokens or 0) * _model_info["cache_read_input_token_cost"] - + (response_1.usage.cache_creation_input_tokens or 0) * _model_info["cache_creation_input_token_cost"] - + (response_1.usage.completion_tokens or 0) * _model_info["output_cost_per_token"] + + (response_1.usage.prompt_tokens_details.cached_tokens or 0) + * _model_info["cache_read_input_token_cost"] + + (response_1.usage.cache_creation_input_tokens or 0) + * _model_info["cache_creation_input_token_cost"] + + (response_1.usage.completion_tokens or 0) + * _model_info["output_cost_per_token"] ) # Cost of processing (non-cache hit + cache hit) + Cost of cache-writing (cache writing) assert round(expected_cost, 5) == round(cost_1, 5) @@ -1105,7 +1157,9 @@ def test_completion_cost_databricks_embedding(model, monkeypatch): sync_handler = HTTPHandler() with patch.object(HTTPHandler, "post", return_value=mock_response): - resp = litellm.embedding(model=model, input=["hey, how's it going?"], client=sync_handler) + resp = litellm.embedding( + model=model, input=["hey, how's it going?"], client=sync_handler + ) print(resp) cost = completion_cost(completion_response=resp) @@ -1142,150 +1196,6 @@ def test_completion_cost_fireworks_ai(model): cost = completion_cost(completion_response=resp) -def test_fireworks_ai_cache_token_pricing(): - """Test that Fireworks AI cost calculator accounts for cache read/creation tokens. - - Regression test for https://github.com/BerriAI/litellm/issues/24774 - """ - from litellm.llms.fireworks_ai.cost_calculator import cost_per_token - from litellm.types.utils import Usage - - # Simulate usage with cache read tokens - usage_with_cache = Usage( - prompt_tokens=1000, - completion_tokens=500, - cache_read_input_tokens=800, - cache_creation_input_tokens=0, - ) - - # Simulate usage without cache tokens - usage_no_cache = Usage( - prompt_tokens=1000, - completion_tokens=500, - ) - - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - # Use kimi-k2p5 which has cache_read_input_token_cost in the pricing config - prompt_cost_cached, completion_cost_cached = cost_per_token(model="fireworks_ai/kimi-k2p5", usage=usage_with_cache) - prompt_cost_no_cache, completion_cost_no_cache = cost_per_token( - model="fireworks_ai/kimi-k2p5", usage=usage_no_cache - ) - - # Completion cost should be the same regardless of cache - assert completion_cost_cached == completion_cost_no_cache - - # kimi-k2p5 has cache_read_input_token_cost (1e-07) < input_cost_per_token (6e-07), - # so prompt cost with 800 cache-read tokens must be cheaper - assert prompt_cost_cached < prompt_cost_no_cache, ( - "Prompt cost with 800 cache-read tokens should be less than full-price for the same total prompt tokens" - ) - - -def test_fireworks_ai_cache_read_token_pricing(): - """Deterministic branch-coverage test for the cache_read_input_tokens adjustment path. - - Exercises lines 86-91 of litellm/llms/fireworks_ai/cost_calculator.py. - - kimi-k2p5 pricing (from model_cost map): - input_cost_per_token = 6e-7 - cache_read_input_token_cost = 1e-7 (cheaper — cache hit discount) - output_cost_per_token = 3e-6 - - With prompt_tokens=1000 and cache_read_input_tokens=100: - base = 1000 * 6e-7 = 6e-4 - cache adjustment = 100 * (1e-7 - 6e-7) = -5e-5 - expected prompt_cost = 5.5e-4 - """ - from litellm.llms.fireworks_ai.cost_calculator import cost_per_token - from litellm.types.utils import Usage - - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - INPUT_COST = 6e-7 - CACHE_READ_COST = 1e-7 - PROMPT_TOKENS = 1000 - CACHE_READ_TOKENS = 100 - COMPLETION_TOKENS = 50 - - usage = Usage( - prompt_tokens=PROMPT_TOKENS, - completion_tokens=COMPLETION_TOKENS, - cache_read_input_tokens=CACHE_READ_TOKENS, - ) - - prompt_cost, completion_cost_val = cost_per_token(model="fireworks_ai/kimi-k2p5", usage=usage) - - expected_prompt_cost = PROMPT_TOKENS * INPUT_COST + CACHE_READ_TOKENS * (CACHE_READ_COST - INPUT_COST) - expected_completion_cost = COMPLETION_TOKENS * 3e-6 - - assert abs(prompt_cost - expected_prompt_cost) < 1e-12, ( - f"Cache-read prompt cost {prompt_cost} != expected {expected_prompt_cost}" - ) - assert abs(completion_cost_val - expected_completion_cost) < 1e-12, ( - f"Completion cost {completion_cost_val} != expected {expected_completion_cost}" - ) - # Sanity: cheaper than paying full rate for all 1000 tokens - assert prompt_cost < PROMPT_TOKENS * INPUT_COST, "Cache-read should reduce prompt cost below full input rate" - - -def test_fireworks_ai_cache_creation_token_pricing(): - """Deterministic branch-coverage test for the cache_creation_input_tokens adjustment path. - - Exercises lines 94-99 of litellm/llms/fireworks_ai/cost_calculator.py. - - No live fireworks model currently has cache_creation_input_token_cost set, so this - test injects a synthetic value (8e-7) directly into litellm.model_cost to isolate - the branch. The injection is scoped to the local model_cost copy created by - get_model_cost_map and does not persist beyond this test. - - Injected pricing for fireworks_ai/kimi-k2p5: - input_cost_per_token = 6e-7 - cache_creation_input_token_cost = 8e-7 (premium — first-write penalty) - - With prompt_tokens=1000 and cache_creation_input_tokens=100: - base = 1000 * 6e-7 = 6e-4 - creation adjustment = 100 * (8e-7 - 6e-7) = +2e-5 - expected prompt_cost = 6.2e-4 - """ - from litellm.llms.fireworks_ai.cost_calculator import cost_per_token - from litellm.types.utils import Usage - - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - INPUT_COST = 6e-7 - CACHE_CREATION_COST = 8e-7 - PROMPT_TOKENS = 1000 - CACHE_CREATION_TOKENS = 100 - COMPLETION_TOKENS = 50 - - # Inject cache_creation_input_token_cost into the local model_cost copy - litellm.model_cost["fireworks_ai/kimi-k2p5"]["cache_creation_input_token_cost"] = CACHE_CREATION_COST - - usage = Usage( - prompt_tokens=PROMPT_TOKENS, - completion_tokens=COMPLETION_TOKENS, - cache_creation_input_tokens=CACHE_CREATION_TOKENS, - ) - - prompt_cost, completion_cost_val = cost_per_token(model="fireworks_ai/kimi-k2p5", usage=usage) - - expected_prompt_cost = PROMPT_TOKENS * INPUT_COST + CACHE_CREATION_TOKENS * (CACHE_CREATION_COST - INPUT_COST) - expected_completion_cost = COMPLETION_TOKENS * 3e-6 - - assert abs(prompt_cost - expected_prompt_cost) < 1e-12, ( - f"Cache-creation prompt cost {prompt_cost} != expected {expected_prompt_cost}" - ) - assert abs(completion_cost_val - expected_completion_cost) < 1e-12, ( - f"Completion cost {completion_cost_val} != expected {expected_completion_cost}" - ) - # Sanity: more expensive than paying the plain input rate for all 1000 tokens - assert prompt_cost > PROMPT_TOKENS * INPUT_COST, "Cache-creation should increase prompt cost above full input rate" - - def test_cost_azure_openai_prompt_caching(): from litellm.utils import Choices, Message, ModelResponse, Usage from litellm.types.utils import ( @@ -1322,7 +1232,9 @@ def test_cost_azure_openai_prompt_caching(): completion_tokens=10, prompt_tokens=14, total_tokens=24, - completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=2), + completion_tokens_details=CompletionTokensDetailsWrapper( + reasoning_tokens=2 + ), ), ) @@ -1352,7 +1264,9 @@ def test_cost_azure_openai_prompt_caching(): prompt_tokens_details=PromptTokensDetailsWrapper( cached_tokens=14, ), - completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=2), + completion_tokens_details=CompletionTokensDetailsWrapper( + reasoning_tokens=2 + ), ), ) @@ -1364,15 +1278,21 @@ def test_cost_azure_openai_prompt_caching(): usage = response_2.usage _expected_cost2 = ( - (usage.prompt_tokens - usage.prompt_tokens_details.cached_tokens) * model_info["input_cost_per_token"] + (usage.prompt_tokens - usage.prompt_tokens_details.cached_tokens) + * model_info["input_cost_per_token"] + (usage.completion_tokens * model_info["output_cost_per_token"]) - + (usage.prompt_tokens_details.cached_tokens * model_info["cache_read_input_token_cost"]) + + ( + usage.prompt_tokens_details.cached_tokens + * model_info["cache_read_input_token_cost"] + ) ) print("_expected_cost2", _expected_cost2) print("cost_2", cost_2) - assert abs(cost_2 - _expected_cost2) < 1e-5 # Allow for small floating-point differences + assert ( + abs(cost_2 - _expected_cost2) < 1e-5 + ) # Allow for small floating-point differences def test_completion_cost_vertex_llama3(): @@ -1485,9 +1405,11 @@ def test_cost_openai_prompt_caching(): usage = response_2.usage _expected_cost2 = ( - (usage.prompt_tokens - usage.prompt_tokens_details.cached_tokens) * model_info["input_cost_per_token"] + (usage.prompt_tokens - usage.prompt_tokens_details.cached_tokens) + * model_info["input_cost_per_token"] + usage.completion_tokens * model_info["output_cost_per_token"] - + usage.prompt_tokens_details.cached_tokens * model_info["cache_read_input_token_cost"] + + usage.prompt_tokens_details.cached_tokens + * model_info["cache_read_input_token_cost"] ) print("_expected_cost2", _expected_cost2) @@ -1529,7 +1451,9 @@ def test_completion_cost_azure_ai_rerank(model): ) print("response", response) model = model - cost = completion_cost(model=model, completion_response=response, call_type="arerank") + cost = completion_cost( + model=model, completion_response=response, call_type="arerank" + ) assert cost > 0 @@ -2482,7 +2406,9 @@ async def test_test_completion_cost_gpt4o_audio_output_from_model(stream): completion_tokens=34, prompt_tokens=16, total_tokens=50, - completion_tokens_details=CompletionTokensDetailsWrapper(audio_tokens=28, reasoning_tokens=0, text_tokens=6), + completion_tokens_details=CompletionTokensDetailsWrapper( + audio_tokens=28, reasoning_tokens=0, text_tokens=6 + ), prompt_tokens_details=PromptTokensDetailsWrapper( audio_tokens=0, cached_tokens=0, text_tokens=16, image_tokens=0 ), @@ -2521,15 +2447,27 @@ async def test_test_completion_cost_gpt4o_audio_output_from_model(stream): print(f"model_info: {model_info}") ## input cost - input_audio_cost = model_info["input_cost_per_audio_token"] * usage_object.prompt_tokens_details.audio_tokens - input_text_cost = model_info["input_cost_per_token"] * usage_object.prompt_tokens_details.text_tokens + input_audio_cost = ( + model_info["input_cost_per_audio_token"] + * usage_object.prompt_tokens_details.audio_tokens + ) + input_text_cost = ( + model_info["input_cost_per_token"] + * usage_object.prompt_tokens_details.text_tokens + ) total_input_cost = input_audio_cost + input_text_cost ## output cost - output_audio_cost = model_info["output_cost_per_audio_token"] * usage_object.completion_tokens_details.audio_tokens - output_text_cost = model_info["output_cost_per_token"] * usage_object.completion_tokens_details.text_tokens + output_audio_cost = ( + model_info["output_cost_per_audio_token"] + * usage_object.completion_tokens_details.audio_tokens + ) + output_text_cost = ( + model_info["output_cost_per_token"] + * usage_object.completion_tokens_details.text_tokens + ) total_output_cost = output_audio_cost + output_text_cost @@ -2655,7 +2593,9 @@ def test_moderations(): litellm.add_known_models() assert "omni-moderation-latest" in litellm.model_cost - print(f"litellm.model_cost['omni-moderation-latest']: {litellm.model_cost['omni-moderation-latest']}") + print( + f"litellm.model_cost['omni-moderation-latest']: {litellm.model_cost['omni-moderation-latest']}" + ) assert "omni-moderation-latest" in litellm.open_ai_chat_completion_models response = moderation("I am a bad person", model="omni-moderation-latest") @@ -2692,7 +2632,9 @@ def test_cost_calculator_azure_embedding(): def test_add_known_models(): litellm.add_known_models() - assert "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0" not in litellm.bedrock_models + assert ( + "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0" not in litellm.bedrock_models + ) @pytest.mark.skip(reason="flaky test") @@ -2808,7 +2750,9 @@ def test_cost_calculator_with_base_model_with_router(base_model_arg): } if base_model_arg == "litellm_param": - model_item["litellm_params"]["base_model"] = "bedrock/anthropic.claude-3-sonnet-20240229-v1:0" + model_item["litellm_params"][ + "base_model" + ] = "bedrock/anthropic.claude-3-sonnet-20240229-v1:0" elif base_model_arg == "model_info": model_item["model_info"] = { "base_model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", diff --git a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py new file mode 100644 index 00000000000..028d3f2053b --- /dev/null +++ b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py @@ -0,0 +1,155 @@ +"""Tests for Fireworks AI cost calculator cache token accounting. + +Regression coverage for https://github.com/BerriAI/litellm/issues/24774 — the +fireworks_ai cost calculator now subtracts the full-price portion of cache read +and cache creation tokens, then adds the discounted/premium rate. +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../../../../..")) + +import litellm +from litellm.llms.fireworks_ai.cost_calculator import cost_per_token +from litellm.types.utils import Usage + + +@pytest.fixture(autouse=True) +def _local_model_cost_map(monkeypatch): + """Load model prices from the bundled JSON so tests are deterministic and offline.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + litellm.model_cost = litellm.get_model_cost_map(url="") + yield + + +def test_cache_read_tokens_reduce_prompt_cost(): + """800 cache-read tokens should make the call cheaper than the full-price path. + + kimi-k2p5 pricing (model_cost map): + input_cost_per_token = 6e-7 + cache_read_input_token_cost = 1e-7 (cache hit discount) + """ + usage_with_cache = Usage( + prompt_tokens=1000, + completion_tokens=500, + cache_read_input_tokens=800, + cache_creation_input_tokens=0, + ) + usage_no_cache = Usage(prompt_tokens=1000, completion_tokens=500) + + prompt_cost_cached, completion_cost_cached = cost_per_token( + model="fireworks_ai/kimi-k2p5", usage=usage_with_cache + ) + prompt_cost_no_cache, completion_cost_no_cache = cost_per_token( + model="fireworks_ai/kimi-k2p5", usage=usage_no_cache + ) + + # Completion cost is unaffected by cache state. + assert completion_cost_cached == completion_cost_no_cache + # Cache hits are cheaper than full-price input tokens. + assert ( + prompt_cost_cached < prompt_cost_no_cache + ), "800 cache-read tokens at 1e-7 should be cheaper than 800 full-price tokens at 6e-7" + + +def test_cache_read_token_pricing_is_deterministic(): + """Exercises the cache_read_input_tokens adjustment branch with exact arithmetic. + + With prompt_tokens=1000 and cache_read_input_tokens=100: + base = 1000 * 6e-7 = 6e-4 + cache adjustment = 100 * (1e-7 - 6e-7) = -5e-5 + expected prompt_cost = 5.5e-4 + """ + INPUT_COST = 6e-7 + CACHE_READ_COST = 1e-7 + PROMPT_TOKENS = 1000 + CACHE_READ_TOKENS = 100 + COMPLETION_TOKENS = 50 + + usage = Usage( + prompt_tokens=PROMPT_TOKENS, + completion_tokens=COMPLETION_TOKENS, + cache_read_input_tokens=CACHE_READ_TOKENS, + ) + + prompt_cost, completion_cost_val = cost_per_token( + model="fireworks_ai/kimi-k2p5", usage=usage + ) + + expected_prompt_cost = PROMPT_TOKENS * INPUT_COST + CACHE_READ_TOKENS * ( + CACHE_READ_COST - INPUT_COST + ) + expected_completion_cost = COMPLETION_TOKENS * 3e-6 + + assert ( + abs(prompt_cost - expected_prompt_cost) < 1e-12 + ), f"Cache-read prompt cost {prompt_cost} != expected {expected_prompt_cost}" + assert abs(completion_cost_val - expected_completion_cost) < 1e-12 + # Sanity: cheaper than paying full rate for all 1000 tokens. + assert prompt_cost < PROMPT_TOKENS * INPUT_COST + + +def test_cache_creation_token_pricing_is_deterministic(): + """Exercises the cache_creation_input_tokens adjustment branch. + + No live fireworks model currently has cache_creation_input_token_cost set, + so this test injects a synthetic value into the local model_cost copy. + The fixture rebuilds model_cost per test, so the injection is scoped. + + With prompt_tokens=1000 and cache_creation_input_tokens=100: + base = 1000 * 6e-7 = 6e-4 + creation adjustment = 100 * (8e-7 - 6e-7) = +2e-5 + expected prompt_cost = 6.2e-4 + """ + INPUT_COST = 6e-7 + CACHE_CREATION_COST = 8e-7 + PROMPT_TOKENS = 1000 + CACHE_CREATION_TOKENS = 100 + COMPLETION_TOKENS = 50 + + litellm.model_cost["fireworks_ai/kimi-k2p5"][ + "cache_creation_input_token_cost" + ] = CACHE_CREATION_COST + + usage = Usage( + prompt_tokens=PROMPT_TOKENS, + completion_tokens=COMPLETION_TOKENS, + cache_creation_input_tokens=CACHE_CREATION_TOKENS, + ) + + prompt_cost, completion_cost_val = cost_per_token( + model="fireworks_ai/kimi-k2p5", usage=usage + ) + + expected_prompt_cost = PROMPT_TOKENS * INPUT_COST + CACHE_CREATION_TOKENS * ( + CACHE_CREATION_COST - INPUT_COST + ) + expected_completion_cost = COMPLETION_TOKENS * 3e-6 + + assert abs(prompt_cost - expected_prompt_cost) < 1e-12 + assert abs(completion_cost_val - expected_completion_cost) < 1e-12 + # Sanity: more expensive than full input rate. + assert prompt_cost > PROMPT_TOKENS * INPUT_COST + + +def test_no_cache_tokens_uses_plain_input_rate(): + """When neither cache_read nor cache_creation tokens are present, + prompt_cost is exactly prompt_tokens * input_cost_per_token. + + Catches a regression where the cache branches incorrectly fire on zero tokens. + """ + INPUT_COST = 6e-7 + PROMPT_TOKENS = 500 + COMPLETION_TOKENS = 10 + + usage = Usage(prompt_tokens=PROMPT_TOKENS, completion_tokens=COMPLETION_TOKENS) + + prompt_cost, completion_cost_val = cost_per_token( + model="fireworks_ai/kimi-k2p5", usage=usage + ) + + assert abs(prompt_cost - PROMPT_TOKENS * INPUT_COST) < 1e-12 + assert abs(completion_cost_val - COMPLETION_TOKENS * 3e-6) < 1e-12