diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 457baddd232..f8f9de7fbec 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -76,7 +76,10 @@ from litellm.llms.perplexity.cost_calculator import ( from litellm.llms.tencent.cost_calculator import ( cost_per_token as tencent_cost_per_token, ) -from litellm.llms.together_ai.cost_calculator import get_model_params_and_category +from litellm.llms.together_ai.cost_calculator import ( + get_model_params_and_category, + has_together_registry_pricing, +) from litellm.llms.vertex_ai.cost_calculator import ( cost_per_character as google_cost_per_character, ) @@ -1569,10 +1572,9 @@ def completion_cost( return MCPCostCalculator.calculate_mcp_tool_call_cost(litellm_logging_obj=litellm_logging_obj) # Calculate cost based on prompt_tokens, completion_tokens - if "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai": - # together ai prices based on size of llm - # get_model_params_and_category takes a model name and returns the category of LLM size it is in model_prices_and_context_window.json - + if ( + "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai" + ) and not has_together_registry_pricing(model, litellm.model_cost): model = get_model_params_and_category(model, call_type=CallTypes(call_type)) # replicate llms are calculate based on time for request running diff --git a/litellm/llms/together_ai/cost_calculator.py b/litellm/llms/together_ai/cost_calculator.py index 431e94f1442..6fc2c949fa6 100644 --- a/litellm/llms/together_ai/cost_calculator.py +++ b/litellm/llms/together_ai/cost_calculator.py @@ -3,6 +3,7 @@ Handles calculating cost for together ai models """ import re +from collections.abc import Mapping from typing import Final from litellm.constants import ( @@ -18,6 +19,12 @@ from litellm.constants import ( from litellm.types.utils import CallTypes +def has_together_registry_pricing(model: str, cost_map: Mapping[str, object]) -> bool: + stripped: Final = model.removeprefix("together_ai/") + entry: Final = cost_map.get(f"together_ai/{stripped}") + return isinstance(entry, Mapping) and "input_cost_per_token" in entry + + # Extract the number of billion parameters from the model name # only used for together_computer LLMs def get_model_params_and_category(model_name, call_type: CallTypes) -> str: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1ccaab56cdf..3864bd08964 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -38807,14 +38807,14 @@ "supports_reasoning": true }, "together_ai/Qwen/Qwen3.7-Max": { - "cache_read_input_token_cost": 1.3e-07, - "input_cost_per_token": 1.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 2.5e-06, "litellm_provider": "together_ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", - "output_cost_per_token": 3.75e-06, + "output_cost_per_token": 7.5e-06, "source": "https://docs.together.ai/docs/serverless-models", "supports_prompt_caching": true }, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1ccaab56cdf..3864bd08964 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -38807,14 +38807,14 @@ "supports_reasoning": true }, "together_ai/Qwen/Qwen3.7-Max": { - "cache_read_input_token_cost": 1.3e-07, - "input_cost_per_token": 1.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 2.5e-06, "litellm_provider": "together_ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", - "output_cost_per_token": 3.75e-06, + "output_cost_per_token": 7.5e-06, "source": "https://docs.together.ai/docs/serverless-models", "supports_prompt_caching": true }, diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 0c99d128e14..d42d83ce6d9 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -3977,6 +3977,74 @@ def test_completion_cost_prices_anthropic_shaped_cache_read_tokens(_local_model_ assert cost == pytest.approx(3 * 4e-6 + 4014 * 4e-7 + 5 * 2e-5, rel=1e-9) +def _together_chat_response(model: str, prompt_tokens: int, completion_tokens: int, cached_tokens: int) -> ModelResponse: + return ModelResponse( + id="chatcmpl-together-cache", + choices=[{"finish_reason": "stop", "index": 0, "message": {"content": "acknowledged", "role": "assistant"}}], + created=1756164000, + model=model, + object="chat.completion", + usage=Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), + ), + ) + + +def test_completion_cost_prices_together_cached_tokens_at_cache_read_rate(_local_model_cost_map): + """Regression: Together reports prompt_tokens_details.cached_tokens but no together_ai + registry entry carried cache_read_input_token_cost, so cache-hit tokens were priced at + 0.0 and spend on cache-heavy workloads was understated.""" + + cost = completion_cost( + completion_response=_together_chat_response( + model="deepseek-ai/DeepSeek-V4-Flash-0731", prompt_tokens=7864, completion_tokens=16, cached_tokens=7863 + ), + custom_llm_provider="together_ai", + ) + + assert cost == pytest.approx(1 * 1.4e-07 + 7863 * 3e-08 + 16 * 2.8e-07, rel=1e-9) + + +def test_completion_cost_together_mapped_model_skips_size_bucket(_local_model_cost_map): + """Regression: any together model whose name matches (\\d+b) was rewritten to a + together-ai-* size bucket before the registry lookup, so mapped models like + Muse-Glimmer-30B never used their per-model rates, cache fields included.""" + + cost = completion_cost( + completion_response=_together_chat_response( + model="meta-models/Muse-Glimmer-30B", prompt_tokens=63, completion_tokens=16, cached_tokens=0 + ), + custom_llm_provider="together_ai", + ) + + assert cost == pytest.approx(63 * 3.5e-07 + 16 * 1.5e-06, rel=1e-9) + + +def test_completion_cost_together_unmapped_model_still_uses_size_bucket(_local_model_cost_map): + cost = completion_cost( + completion_response=_together_chat_response( + model="qwen/Qwen2-72B-Instruct", prompt_tokens=23, completion_tokens=15, cached_tokens=0 + ), + custom_llm_provider="together_ai", + ) + + assert cost == pytest.approx((23 + 15) * 9e-07, rel=1e-9) + + +def test_completion_cost_together_metadata_only_model_still_uses_size_bucket(_local_model_cost_map): + assert "input_cost_per_token" not in litellm.model_cost["together_ai/togethercomputer/CodeLlama-34b-Instruct"] + + cost = completion_cost( + completion_response=_together_chat_response( + model="togethercomputer/CodeLlama-34b-Instruct", prompt_tokens=23, completion_tokens=15, cached_tokens=0 + ), + custom_llm_provider="together_ai", + ) + + assert cost == pytest.approx((23 + 15) * 8e-07, rel=1e-9) def test_select_model_name_strips_unregistered_alias_prefix(_local_model_cost_map): """A router-facing model_name alias containing "/" whose leading segment is NOT a registered provider must not be double-prefixed into a non-existent cost key. diff --git a/tests/test_litellm/test_together_ai_model_metadata.py b/tests/test_litellm/test_together_ai_model_metadata.py index 32d0769becc..e0379aead84 100644 --- a/tests/test_litellm/test_together_ai_model_metadata.py +++ b/tests/test_litellm/test_together_ai_model_metadata.py @@ -159,3 +159,51 @@ def test_together_backup_cost_map_in_sync(cost_map: CostMap): together_main = {k: v for k, v in cost_map.items() if k.startswith("together_ai/")} together_backup = {k: v for k, v in backup.items() if k.startswith("together_ai/")} assert together_backup == together_main + + +CACHED_INPUT_MODELS: Final = ( + "together_ai/moonshotai/Kimi-K3", + "together_ai/zai-org/GLM-5.2", + "together_ai/meta-models/Muse-Glimmer-30B", + "together_ai/Qwen/Qwen3.8-2.4T-A95B", + "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813", + "together_ai/deepseek-ai/DeepSeek-V4-Flash-0731", + "together_ai/thinkingmachines/Inkling", + "together_ai/MiniMaxAI/MiniMax-M3", + "together_ai/thinkingmachines/Inkling-Small", + "together_ai/moonshotai/Kimi-K2.7-Code", + "together_ai/deepseek-ai/DeepSeek-V4-Pro", + "together_ai/nvidia/nemotron-3-ultra-550b-a55b", + "together_ai/Qwen/Qwen3.7-Max", +) + + +@pytest.mark.parametrize("model", CACHED_INPUT_MODELS) +def test_together_cached_input_model_carries_cache_read_pricing(cost_map: CostMap, model: str): + info = cost_map.get(model) + assert info is not None, f"{model} missing from model_prices_and_context_window.json" + assert info.get("supports_prompt_caching") is True + cache_read = info.get("cache_read_input_token_cost") + assert isinstance(cache_read, float) + assert 0 < cache_read < info["input_cost_per_token"] + assert "cache_creation_input_token_cost" not in info + + +def test_together_prompt_caching_flag_implies_cache_read_rate(cost_map: CostMap): + for model, info in cost_map.items(): + if model.startswith("together_ai/") and info.get("supports_prompt_caching"): + assert "cache_read_input_token_cost" in info, f"{model} flags caching without a cache read rate" + + +def test_together_deepseek_v4_flash_cache_read_rate(cost_map: CostMap): + info = cost_map["together_ai/deepseek-ai/DeepSeek-V4-Flash-0731"] + assert info["input_cost_per_token"] == 1.4e-07 + assert info["cache_read_input_token_cost"] == 3e-08 + assert info["output_cost_per_token"] == 2.8e-07 + + +def test_together_qwen_37_max_repriced_to_current_together_rate(cost_map: CostMap): + info = cost_map["together_ai/Qwen/Qwen3.7-Max"] + assert info["input_cost_per_token"] == 2.5e-06 + assert info["output_cost_per_token"] == 7.5e-06 + assert info["cache_read_input_token_cost"] == 5e-07