Merge pull request #38280 from BerriAI/litellm_together_cache_pricing

fix(cost): apply Together AI cache read pricing and per-model registry rates

litellm/cost_calculator.py

@@ -76,7 +76,10 @@ from litellm.llms.perplexity.cost_calculator import (
 from litellm.llms.tencent.cost_calculator import (
     cost_per_token as tencent_cost_per_token,
 )
-from litellm.llms.together_ai.cost_calculator import get_model_params_and_category
+from litellm.llms.together_ai.cost_calculator import (
+    get_model_params_and_category,
+    has_together_registry_pricing,
+)
 from litellm.llms.vertex_ai.cost_calculator import (
     cost_per_character as google_cost_per_character,
 )
@@ -1569,10 +1572,9 @@ def completion_cost(
 
                     return MCPCostCalculator.calculate_mcp_tool_call_cost(litellm_logging_obj=litellm_logging_obj)
                 # Calculate cost based on prompt_tokens, completion_tokens
-                if "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai":
-                    # together ai prices based on size of llm
-                    # get_model_params_and_category takes a model name and returns the category of LLM size it is in model_prices_and_context_window.json
-
+                if (
+                    "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai"
+                ) and not has_together_registry_pricing(model, litellm.model_cost):
                     model = get_model_params_and_category(model, call_type=CallTypes(call_type))
 
                 # replicate llms are calculate based on time for request running

litellm/llms/together_ai/cost_calculator.py

@@ -3,6 +3,7 @@ Handles calculating cost for together ai models
 """
 
 import re
+from collections.abc import Mapping
 from typing import Final
 
 from litellm.constants import (
@@ -18,6 +19,12 @@ from litellm.constants import (
 from litellm.types.utils import CallTypes
 
 
+def has_together_registry_pricing(model: str, cost_map: Mapping[str, object]) -> bool:
+    stripped: Final = model.removeprefix("together_ai/")
+    entry: Final = cost_map.get(f"together_ai/{stripped}")
+    return isinstance(entry, Mapping) and "input_cost_per_token" in entry
+
+
 # Extract the number of billion parameters from the model name
 # only used for together_computer LLMs
 def get_model_params_and_category(model_name, call_type: CallTypes) -> str:

litellm/model_prices_and_context_window_backup.json

@@ -38807,14 +38807,14 @@
         "supports_reasoning": true
     },
     "together_ai/Qwen/Qwen3.7-Max": {
-        "cache_read_input_token_cost": 1.3e-07,
-        "input_cost_per_token": 1.25e-06,
+        "cache_read_input_token_cost": 5e-07,
+        "input_cost_per_token": 2.5e-06,
         "litellm_provider": "together_ai",
         "max_input_tokens": 1000000,
         "max_output_tokens": 1000000,
         "max_tokens": 1000000,
         "mode": "chat",
-        "output_cost_per_token": 3.75e-06,
+        "output_cost_per_token": 7.5e-06,
         "source": "https://docs.together.ai/docs/serverless-models",
         "supports_prompt_caching": true
     },

model_prices_and_context_window.json

@@ -38807,14 +38807,14 @@
         "supports_reasoning": true
     },
     "together_ai/Qwen/Qwen3.7-Max": {
-        "cache_read_input_token_cost": 1.3e-07,
-        "input_cost_per_token": 1.25e-06,
+        "cache_read_input_token_cost": 5e-07,
+        "input_cost_per_token": 2.5e-06,
         "litellm_provider": "together_ai",
         "max_input_tokens": 1000000,
         "max_output_tokens": 1000000,
         "max_tokens": 1000000,
         "mode": "chat",
-        "output_cost_per_token": 3.75e-06,
+        "output_cost_per_token": 7.5e-06,
         "source": "https://docs.together.ai/docs/serverless-models",
         "supports_prompt_caching": true
     },

tests/test_litellm/test_cost_calculator.py

@@ -3977,6 +3977,74 @@ def test_completion_cost_prices_anthropic_shaped_cache_read_tokens(_local_model_
     assert cost == pytest.approx(3 * 4e-6 + 4014 * 4e-7 + 5 * 2e-5, rel=1e-9)
 
 
+def _together_chat_response(model: str, prompt_tokens: int, completion_tokens: int, cached_tokens: int) -> ModelResponse:
+    return ModelResponse(
+        id="chatcmpl-together-cache",
+        choices=[{"finish_reason": "stop", "index": 0, "message": {"content": "acknowledged", "role": "assistant"}}],
+        created=1756164000,
+        model=model,
+        object="chat.completion",
+        usage=Usage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=prompt_tokens + completion_tokens,
+            prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens),
+        ),
+    )
+
+
+def test_completion_cost_prices_together_cached_tokens_at_cache_read_rate(_local_model_cost_map):
+    """Regression: Together reports prompt_tokens_details.cached_tokens but no together_ai
+    registry entry carried cache_read_input_token_cost, so cache-hit tokens were priced at
+    0.0 and spend on cache-heavy workloads was understated."""
+
+    cost = completion_cost(
+        completion_response=_together_chat_response(
+            model="deepseek-ai/DeepSeek-V4-Flash-0731", prompt_tokens=7864, completion_tokens=16, cached_tokens=7863
+        ),
+        custom_llm_provider="together_ai",
+    )
+
+    assert cost == pytest.approx(1 * 1.4e-07 + 7863 * 3e-08 + 16 * 2.8e-07, rel=1e-9)
+
+
+def test_completion_cost_together_mapped_model_skips_size_bucket(_local_model_cost_map):
+    """Regression: any together model whose name matches (\\d+b) was rewritten to a
+    together-ai-* size bucket before the registry lookup, so mapped models like
+    Muse-Glimmer-30B never used their per-model rates, cache fields included."""
+
+    cost = completion_cost(
+        completion_response=_together_chat_response(
+            model="meta-models/Muse-Glimmer-30B", prompt_tokens=63, completion_tokens=16, cached_tokens=0
+        ),
+        custom_llm_provider="together_ai",
+    )
+
+    assert cost == pytest.approx(63 * 3.5e-07 + 16 * 1.5e-06, rel=1e-9)
+
+
+def test_completion_cost_together_unmapped_model_still_uses_size_bucket(_local_model_cost_map):
+    cost = completion_cost(
+        completion_response=_together_chat_response(
+            model="qwen/Qwen2-72B-Instruct", prompt_tokens=23, completion_tokens=15, cached_tokens=0
+        ),
+        custom_llm_provider="together_ai",
+    )
+
+    assert cost == pytest.approx((23 + 15) * 9e-07, rel=1e-9)
+
+
+def test_completion_cost_together_metadata_only_model_still_uses_size_bucket(_local_model_cost_map):
+    assert "input_cost_per_token" not in litellm.model_cost["together_ai/togethercomputer/CodeLlama-34b-Instruct"]
+
+    cost = completion_cost(
+        completion_response=_together_chat_response(
+            model="togethercomputer/CodeLlama-34b-Instruct", prompt_tokens=23, completion_tokens=15, cached_tokens=0
+        ),
+        custom_llm_provider="together_ai",
+    )
+
+    assert cost == pytest.approx((23 + 15) * 8e-07, rel=1e-9)
 def test_select_model_name_strips_unregistered_alias_prefix(_local_model_cost_map):
     """A router-facing model_name alias containing "/" whose leading segment is NOT a
     registered provider must not be double-prefixed into a non-existent cost key.

tests/test_litellm/test_together_ai_model_metadata.py

@@ -159,3 +159,51 @@ def test_together_backup_cost_map_in_sync(cost_map: CostMap):
     together_main = {k: v for k, v in cost_map.items() if k.startswith("together_ai/")}
     together_backup = {k: v for k, v in backup.items() if k.startswith("together_ai/")}
     assert together_backup == together_main
+
+
+CACHED_INPUT_MODELS: Final = (
+    "together_ai/moonshotai/Kimi-K3",
+    "together_ai/zai-org/GLM-5.2",
+    "together_ai/meta-models/Muse-Glimmer-30B",
+    "together_ai/Qwen/Qwen3.8-2.4T-A95B",
+    "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813",
+    "together_ai/deepseek-ai/DeepSeek-V4-Flash-0731",
+    "together_ai/thinkingmachines/Inkling",
+    "together_ai/MiniMaxAI/MiniMax-M3",
+    "together_ai/thinkingmachines/Inkling-Small",
+    "together_ai/moonshotai/Kimi-K2.7-Code",
+    "together_ai/deepseek-ai/DeepSeek-V4-Pro",
+    "together_ai/nvidia/nemotron-3-ultra-550b-a55b",
+    "together_ai/Qwen/Qwen3.7-Max",
+)
+
+
+@pytest.mark.parametrize("model", CACHED_INPUT_MODELS)
+def test_together_cached_input_model_carries_cache_read_pricing(cost_map: CostMap, model: str):
+    info = cost_map.get(model)
+    assert info is not None, f"{model} missing from model_prices_and_context_window.json"
+    assert info.get("supports_prompt_caching") is True
+    cache_read = info.get("cache_read_input_token_cost")
+    assert isinstance(cache_read, float)
+    assert 0 < cache_read < info["input_cost_per_token"]
+    assert "cache_creation_input_token_cost" not in info
+
+
+def test_together_prompt_caching_flag_implies_cache_read_rate(cost_map: CostMap):
+    for model, info in cost_map.items():
+        if model.startswith("together_ai/") and info.get("supports_prompt_caching"):
+            assert "cache_read_input_token_cost" in info, f"{model} flags caching without a cache read rate"
+
+
+def test_together_deepseek_v4_flash_cache_read_rate(cost_map: CostMap):
+    info = cost_map["together_ai/deepseek-ai/DeepSeek-V4-Flash-0731"]
+    assert info["input_cost_per_token"] == 1.4e-07
+    assert info["cache_read_input_token_cost"] == 3e-08
+    assert info["output_cost_per_token"] == 2.8e-07
+
+
+def test_together_qwen_37_max_repriced_to_current_together_rate(cost_map: CostMap):
+    info = cost_map["together_ai/Qwen/Qwen3.7-Max"]
+    assert info["input_cost_per_token"] == 2.5e-06
+    assert info["output_cost_per_token"] == 7.5e-06
+    assert info["cache_read_input_token_cost"] == 5e-07