mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
fix(cost): apply Together AI cache read pricing and per-model registry rates
This commit is contained in:
mateo-berri
parent
75bf9f9452
commit
6fafb46731
6 changed files with 183 additions and 15 deletions
|
|
@ -75,7 +75,10 @@ from litellm.llms.perplexity.cost_calculator import (
|
|||
from litellm.llms.tencent.cost_calculator import (
|
||||
cost_per_token as tencent_cost_per_token,
|
||||
)
|
||||
from litellm.llms.together_ai.cost_calculator import get_model_params_and_category
|
||||
from litellm.llms.together_ai.cost_calculator import (
|
||||
get_model_params_and_category,
|
||||
has_together_registry_entry,
|
||||
)
|
||||
from litellm.llms.vertex_ai.cost_calculator import (
|
||||
cost_per_character as google_cost_per_character,
|
||||
)
|
||||
|
|
@ -1551,8 +1554,10 @@ def completion_cost(
|
|||
|
||||
return MCPCostCalculator.calculate_mcp_tool_call_cost(litellm_logging_obj=litellm_logging_obj)
|
||||
# Calculate cost based on prompt_tokens, completion_tokens
|
||||
if "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai":
|
||||
# together ai prices based on size of llm
|
||||
if (
|
||||
"togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai"
|
||||
) and not has_together_registry_entry(model, litellm.model_cost):
|
||||
# together ai prices unmapped models based on size of llm
|
||||
# get_model_params_and_category takes a model name and returns the category of LLM size it is in model_prices_and_context_window.json
|
||||
|
||||
model = get_model_params_and_category(model, call_type=CallTypes(call_type))
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Handles calculating cost for together ai models
|
|||
"""
|
||||
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from litellm.constants import (
|
||||
|
|
@ -18,6 +19,11 @@ from litellm.constants import (
|
|||
from litellm.types.utils import CallTypes
|
||||
|
||||
|
||||
def has_together_registry_entry(model: str, cost_map: Mapping[str, object]) -> bool:
|
||||
stripped: Final = model.removeprefix("together_ai/")
|
||||
return f"together_ai/{stripped}" in cost_map
|
||||
|
||||
|
||||
# Extract the number of billion parameters from the model name
|
||||
# only used for together_computer LLMs
|
||||
def get_model_params_and_category(model_name, call_type: CallTypes) -> str:
|
||||
|
|
|
|||
|
|
@ -38309,6 +38309,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"together_ai/MiniMaxAI/MiniMax-M3": {
|
||||
"cache_read_input_token_cost": 6e-08,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 524288,
|
||||
|
|
@ -38319,6 +38320,7 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -38362,14 +38364,16 @@
|
|||
"supports_reasoning": true
|
||||
},
|
||||
"together_ai/Qwen/Qwen3.7-Max": {
|
||||
"input_cost_per_token": 1.25e-06,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"input_cost_per_token": 2.5e-06,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 1000000,
|
||||
"max_tokens": 1000000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.75e-06,
|
||||
"source": "https://docs.together.ai/docs/serverless-models"
|
||||
"output_cost_per_token": 7.5e-06,
|
||||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_prompt_caching": true
|
||||
},
|
||||
"together_ai/Qwen/Qwen3.7-Plus": {
|
||||
"input_cost_per_token": 3.2e-07,
|
||||
|
|
@ -38382,6 +38386,7 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models"
|
||||
},
|
||||
"together_ai/Qwen/Qwen3.8-2.4T-A95B": {
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"input_cost_per_token": 2.5e-06,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 1010000,
|
||||
|
|
@ -38389,7 +38394,8 @@
|
|||
"max_tokens": 1010000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6.25e-06,
|
||||
"source": "https://docs.together.ai/docs/serverless-models"
|
||||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_prompt_caching": true
|
||||
},
|
||||
"together_ai/arize-ai/qwen-2-1.5b-instruct": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
|
|
@ -38402,6 +38408,7 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models"
|
||||
},
|
||||
"together_ai/deepseek-ai/DeepSeek-V4-Flash-0731": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"input_cost_per_token": 1.4e-07,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -38412,10 +38419,12 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"together_ai/deepseek-ai/DeepSeek-V4-Pro": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"input_cost_per_token": 1.74e-06,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 512000,
|
||||
|
|
@ -38426,11 +38435,13 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": {
|
||||
"cache_read_input_token_cost": 1.3e-07,
|
||||
"input_cost_per_token": 1.32e-06,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -38441,6 +38452,7 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
@ -38490,6 +38502,7 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models"
|
||||
},
|
||||
"together_ai/meta-models/Muse-Glimmer-30B": {
|
||||
"cache_read_input_token_cost": 4e-08,
|
||||
"input_cost_per_token": 3.5e-07,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -38497,9 +38510,11 @@
|
|||
"max_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.5e-06,
|
||||
"source": "https://docs.together.ai/docs/serverless-models"
|
||||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_prompt_caching": true
|
||||
},
|
||||
"together_ai/moonshotai/Kimi-K2.7-Code": {
|
||||
"cache_read_input_token_cost": 1.9e-07,
|
||||
"input_cost_per_token": 9.5e-07,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 262144,
|
||||
|
|
@ -38510,11 +38525,13 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"together_ai/moonshotai/Kimi-K3": {
|
||||
"cache_read_input_token_cost": 3e-07,
|
||||
"input_cost_per_token": 3e-06,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -38525,12 +38542,14 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"together_ai/nvidia/nemotron-3-ultra-550b-a55b": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 512288,
|
||||
|
|
@ -38541,6 +38560,7 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -38556,6 +38576,7 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models"
|
||||
},
|
||||
"together_ai/thinkingmachines/Inkling": {
|
||||
"cache_read_input_token_cost": 1.7e-07,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 524288,
|
||||
|
|
@ -38566,10 +38587,12 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"together_ai/thinkingmachines/Inkling-Small": {
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
"input_cost_per_token": 5e-07,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 524288,
|
||||
|
|
@ -38577,9 +38600,11 @@
|
|||
"max_tokens": 524288,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"source": "https://docs.together.ai/docs/serverless-models"
|
||||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_prompt_caching": true
|
||||
},
|
||||
"together_ai/zai-org/GLM-5.2": {
|
||||
"cache_read_input_token_cost": 2.6e-07,
|
||||
"input_cost_per_token": 1.4e-06,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 1048575,
|
||||
|
|
@ -38590,6 +38615,7 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
|
|||
|
|
@ -38309,6 +38309,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"together_ai/MiniMaxAI/MiniMax-M3": {
|
||||
"cache_read_input_token_cost": 6e-08,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 524288,
|
||||
|
|
@ -38319,6 +38320,7 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -38362,14 +38364,16 @@
|
|||
"supports_reasoning": true
|
||||
},
|
||||
"together_ai/Qwen/Qwen3.7-Max": {
|
||||
"input_cost_per_token": 1.25e-06,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"input_cost_per_token": 2.5e-06,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 1000000,
|
||||
"max_tokens": 1000000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.75e-06,
|
||||
"source": "https://docs.together.ai/docs/serverless-models"
|
||||
"output_cost_per_token": 7.5e-06,
|
||||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_prompt_caching": true
|
||||
},
|
||||
"together_ai/Qwen/Qwen3.7-Plus": {
|
||||
"input_cost_per_token": 3.2e-07,
|
||||
|
|
@ -38382,6 +38386,7 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models"
|
||||
},
|
||||
"together_ai/Qwen/Qwen3.8-2.4T-A95B": {
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"input_cost_per_token": 2.5e-06,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 1010000,
|
||||
|
|
@ -38389,7 +38394,8 @@
|
|||
"max_tokens": 1010000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6.25e-06,
|
||||
"source": "https://docs.together.ai/docs/serverless-models"
|
||||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_prompt_caching": true
|
||||
},
|
||||
"together_ai/arize-ai/qwen-2-1.5b-instruct": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
|
|
@ -38402,6 +38408,7 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models"
|
||||
},
|
||||
"together_ai/deepseek-ai/DeepSeek-V4-Flash-0731": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"input_cost_per_token": 1.4e-07,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -38412,10 +38419,12 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"together_ai/deepseek-ai/DeepSeek-V4-Pro": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"input_cost_per_token": 1.74e-06,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 512000,
|
||||
|
|
@ -38426,11 +38435,13 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": {
|
||||
"cache_read_input_token_cost": 1.3e-07,
|
||||
"input_cost_per_token": 1.32e-06,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -38441,6 +38452,7 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
@ -38490,6 +38502,7 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models"
|
||||
},
|
||||
"together_ai/meta-models/Muse-Glimmer-30B": {
|
||||
"cache_read_input_token_cost": 4e-08,
|
||||
"input_cost_per_token": 3.5e-07,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -38497,9 +38510,11 @@
|
|||
"max_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.5e-06,
|
||||
"source": "https://docs.together.ai/docs/serverless-models"
|
||||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_prompt_caching": true
|
||||
},
|
||||
"together_ai/moonshotai/Kimi-K2.7-Code": {
|
||||
"cache_read_input_token_cost": 1.9e-07,
|
||||
"input_cost_per_token": 9.5e-07,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 262144,
|
||||
|
|
@ -38510,11 +38525,13 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"together_ai/moonshotai/Kimi-K3": {
|
||||
"cache_read_input_token_cost": 3e-07,
|
||||
"input_cost_per_token": 3e-06,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -38525,12 +38542,14 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"together_ai/nvidia/nemotron-3-ultra-550b-a55b": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 512288,
|
||||
|
|
@ -38541,6 +38560,7 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -38556,6 +38576,7 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models"
|
||||
},
|
||||
"together_ai/thinkingmachines/Inkling": {
|
||||
"cache_read_input_token_cost": 1.7e-07,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 524288,
|
||||
|
|
@ -38566,10 +38587,12 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"together_ai/thinkingmachines/Inkling-Small": {
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
"input_cost_per_token": 5e-07,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 524288,
|
||||
|
|
@ -38577,9 +38600,11 @@
|
|||
"max_tokens": 524288,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"source": "https://docs.together.ai/docs/serverless-models"
|
||||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_prompt_caching": true
|
||||
},
|
||||
"together_ai/zai-org/GLM-5.2": {
|
||||
"cache_read_input_token_cost": 2.6e-07,
|
||||
"input_cost_per_token": 1.4e-06,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 1048575,
|
||||
|
|
@ -38590,6 +38615,7 @@
|
|||
"source": "https://docs.together.ai/docs/serverless-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
|
|||
|
|
@ -3781,3 +3781,60 @@ def test_completion_cost_prices_anthropic_shaped_cache_read_tokens(_local_model_
|
|||
)
|
||||
|
||||
assert cost == pytest.approx(3 * 4e-6 + 4014 * 4e-7 + 5 * 2e-5, rel=1e-9)
|
||||
|
||||
|
||||
def _together_chat_response(model: str, prompt_tokens: int, completion_tokens: int, cached_tokens: int) -> ModelResponse:
|
||||
return ModelResponse(
|
||||
id="chatcmpl-together-cache",
|
||||
choices=[{"finish_reason": "stop", "index": 0, "message": {"content": "acknowledged", "role": "assistant"}}],
|
||||
created=1756164000,
|
||||
model=model,
|
||||
object="chat.completion",
|
||||
usage=Usage(
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=prompt_tokens + completion_tokens,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_completion_cost_prices_together_cached_tokens_at_cache_read_rate(_local_model_cost_map):
|
||||
"""Regression: Together reports prompt_tokens_details.cached_tokens but no together_ai
|
||||
registry entry carried cache_read_input_token_cost, so cache-hit tokens were priced at
|
||||
0.0 and spend on cache-heavy workloads was understated."""
|
||||
|
||||
cost = completion_cost(
|
||||
completion_response=_together_chat_response(
|
||||
model="deepseek-ai/DeepSeek-V4-Flash-0731", prompt_tokens=7864, completion_tokens=16, cached_tokens=7863
|
||||
),
|
||||
custom_llm_provider="together_ai",
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(1 * 1.4e-07 + 7863 * 3e-08 + 16 * 2.8e-07, rel=1e-9)
|
||||
|
||||
|
||||
def test_completion_cost_together_mapped_model_skips_size_bucket(_local_model_cost_map):
|
||||
"""Regression: any together model whose name matches (\\d+b) was rewritten to a
|
||||
together-ai-* size bucket before the registry lookup, so mapped models like
|
||||
Muse-Glimmer-30B never used their per-model rates, cache fields included."""
|
||||
|
||||
cost = completion_cost(
|
||||
completion_response=_together_chat_response(
|
||||
model="meta-models/Muse-Glimmer-30B", prompt_tokens=63, completion_tokens=16, cached_tokens=0
|
||||
),
|
||||
custom_llm_provider="together_ai",
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(63 * 3.5e-07 + 16 * 1.5e-06, rel=1e-9)
|
||||
|
||||
|
||||
def test_completion_cost_together_unmapped_model_still_uses_size_bucket(_local_model_cost_map):
|
||||
cost = completion_cost(
|
||||
completion_response=_together_chat_response(
|
||||
model="qwen/Qwen2-72B-Instruct", prompt_tokens=23, completion_tokens=15, cached_tokens=0
|
||||
),
|
||||
custom_llm_provider="together_ai",
|
||||
)
|
||||
|
||||
assert cost == pytest.approx((23 + 15) * 9e-07, rel=1e-9)
|
||||
|
|
|
|||
|
|
@ -159,3 +159,51 @@ def test_together_backup_cost_map_in_sync(cost_map: CostMap):
|
|||
together_main = {k: v for k, v in cost_map.items() if k.startswith("together_ai/")}
|
||||
together_backup = {k: v for k, v in backup.items() if k.startswith("together_ai/")}
|
||||
assert together_backup == together_main
|
||||
|
||||
|
||||
CACHED_INPUT_MODELS: Final = (
|
||||
"together_ai/moonshotai/Kimi-K3",
|
||||
"together_ai/zai-org/GLM-5.2",
|
||||
"together_ai/meta-models/Muse-Glimmer-30B",
|
||||
"together_ai/Qwen/Qwen3.8-2.4T-A95B",
|
||||
"together_ai/deepseek-ai/DeepSeek-V4-Pro-0813",
|
||||
"together_ai/deepseek-ai/DeepSeek-V4-Flash-0731",
|
||||
"together_ai/thinkingmachines/Inkling",
|
||||
"together_ai/MiniMaxAI/MiniMax-M3",
|
||||
"together_ai/thinkingmachines/Inkling-Small",
|
||||
"together_ai/moonshotai/Kimi-K2.7-Code",
|
||||
"together_ai/deepseek-ai/DeepSeek-V4-Pro",
|
||||
"together_ai/nvidia/nemotron-3-ultra-550b-a55b",
|
||||
"together_ai/Qwen/Qwen3.7-Max",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", CACHED_INPUT_MODELS)
|
||||
def test_together_cached_input_model_carries_cache_read_pricing(cost_map: CostMap, model: str):
|
||||
info = cost_map.get(model)
|
||||
assert info is not None, f"{model} missing from model_prices_and_context_window.json"
|
||||
assert info.get("supports_prompt_caching") is True
|
||||
cache_read = info.get("cache_read_input_token_cost")
|
||||
assert isinstance(cache_read, float)
|
||||
assert 0 < cache_read < info["input_cost_per_token"]
|
||||
assert "cache_creation_input_token_cost" not in info
|
||||
|
||||
|
||||
def test_together_prompt_caching_flag_implies_cache_read_rate(cost_map: CostMap):
|
||||
for model, info in cost_map.items():
|
||||
if model.startswith("together_ai/") and info.get("supports_prompt_caching"):
|
||||
assert "cache_read_input_token_cost" in info, f"{model} flags caching without a cache read rate"
|
||||
|
||||
|
||||
def test_together_deepseek_v4_flash_cache_read_rate(cost_map: CostMap):
|
||||
info = cost_map["together_ai/deepseek-ai/DeepSeek-V4-Flash-0731"]
|
||||
assert info["input_cost_per_token"] == 1.4e-07
|
||||
assert info["cache_read_input_token_cost"] == 3e-08
|
||||
assert info["output_cost_per_token"] == 2.8e-07
|
||||
|
||||
|
||||
def test_together_qwen_37_max_repriced_to_current_together_rate(cost_map: CostMap):
|
||||
info = cost_map["together_ai/Qwen/Qwen3.7-Max"]
|
||||
assert info["input_cost_per_token"] == 2.5e-06
|
||||
assert info["output_cost_per_token"] == 7.5e-06
|
||||
assert info["cache_read_input_token_cost"] == 5e-07
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue