From 3183a53922f43c8a2c35e2560c02acb5f34e0db7 Mon Sep 17 00:00:00 2001 From: kerry Date: Sun, 11 Oct 2026 00:10:04 +0000 Subject: [PATCH] feat(decisions): bill decisions and systemone from /decisions/ cost map rows Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/__init__.py | 3 + litellm/decisions/call.py | 7 +- .../litellm_core_utils/decisions_pricing.py | 19 ++ .../pass_through/typesafe_logging_handler.py | 9 +- ...odel_prices_and_context_window_backup.json | 276 ++++++++++++++++-- model_prices_and_context_window.json | 276 ++++++++++++++++-- .../cost_calculation/cases/openai.py | 74 ++++- tests/integration/cost_calculation/catalog.py | 10 +- .../integration/cost_calculation/cost_map.py | 38 ++- .../cost_calculation/cost_tracking_case.py | 5 + .../cost_calculation/test_cost_tracking.py | 37 ++- tests/unit/decisions/test_main.py | 108 ++++++- .../test_decisions_pricing.py | 33 +++ ...st_typesafe_passthrough_logging_handler.py | 133 +++++++-- tests/unit/test_model_prices_schema.py | 31 ++ tests/unit/test_utils.py | 13 +- 16 files changed, 965 insertions(+), 107 deletions(-) create mode 100644 litellm/litellm_core_utils/decisions_pricing.py create mode 100644 tests/unit/litellm_core_utils/test_decisions_pricing.py diff --git a/litellm/__init__.py b/litellm/__init__.py index a611b90bc37..0b304919727 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -49,6 +49,7 @@ from collections.abc import Mapping from litellm.types.integrations.datadog import DatadogInitParams from litellm.types.integrations.newrelic import NewRelicInitParams from litellm.litellm_core_utils.core_helpers import drop_params_env_flag +from litellm.litellm_core_utils.decisions_pricing import is_decisions_pricing_key from litellm.types.integrations.pointfive import PointFiveInitParams from litellm.types.integrations.zerobus import ZerobusInitParams from litellm._logging import ( @@ -759,6 +760,8 @@ def is_openai_finetune_model(key: str) -> bool: def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None: for key, value in model_cost_map.items(): + if isinstance((provider := value.get("litellm_provider")), str) and is_decisions_pricing_key(key, provider): + continue if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key): open_ai_chat_completion_models.add(key) elif value.get("litellm_provider") == "text-completion-openai": diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 1ad2f006736..856e647a6e9 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -8,6 +8,7 @@ from typing_extensions import assert_never import litellm from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params +from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_model from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.decisions.transformation import ( @@ -263,7 +264,11 @@ def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> Deci { "model": f"{call.custom_llm_provider}/{call.model}", "custom_llm_provider": call.custom_llm_provider, - "provider_response_model": f"{call.custom_llm_provider}/{call.model}", + "provider_response_model": decisions_pricing_model( + call.custom_llm_provider, + call.model, + litellm.model_cost, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded + ), **({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}), } ) diff --git a/litellm/litellm_core_utils/decisions_pricing.py b/litellm/litellm_core_utils/decisions_pricing.py new file mode 100644 index 00000000000..e371c2aa2c9 --- /dev/null +++ b/litellm/litellm_core_utils/decisions_pricing.py @@ -0,0 +1,19 @@ +from collections.abc import Mapping +from typing import Final + + +def decisions_pricing_key(custom_llm_provider: str, model: str) -> str: + return f"{custom_llm_provider}/decisions/{model}" + + +def is_decisions_pricing_key(key: str, custom_llm_provider: str) -> bool: + return key.startswith(decisions_pricing_key(custom_llm_provider, "")) + + +def decisions_pricing_model( + custom_llm_provider: str, + model: str, + model_cost: Mapping[str, object], +) -> str: + decisions_model: Final = decisions_pricing_key(custom_llm_provider, model) + return decisions_model if decisions_model in model_cost else f"{custom_llm_provider}/{model}" diff --git a/litellm/llms/pass_through/typesafe_logging_handler.py b/litellm/llms/pass_through/typesafe_logging_handler.py index c24915c7126..3637db6e1c4 100644 --- a/litellm/llms/pass_through/typesafe_logging_handler.py +++ b/litellm/llms/pass_through/typesafe_logging_handler.py @@ -6,6 +6,7 @@ import httpx from pydantic import TypeAdapter, ValidationError import litellm +from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_model from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, # pyright: ignore[reportUnknownVariableType] # legacy helper has an untyped signature @@ -85,7 +86,13 @@ class TypeSafePassthroughLoggingHandler: input_tokens: Final = usage.input_tokens output_tokens: Final = usage.output_tokens candidate_model_keys: Final = tuple( - f"{custom_llm_provider}/{model}" for model in (response_model, request_model) if model is not None + decisions_pricing_model( + custom_llm_provider, + model, + litellm.model_cost, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded + ) + for model in (response_model, request_model) + if model is not None ) pricing: Final = _pricing_for(candidate_model_keys) response_cost: Final = ( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index bf7f1db7a31..a28bffa8c3e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -8538,7 +8538,11 @@ "output_cost_per_token_above_272k_tokens": 1.5e-05, "output_cost_per_token_above_272k_tokens_priority": 3e-05, "output_cost_per_token_priority": 2e-05, - "search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01}, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "source": "https://azure.microsoft.com/en-us/blog/gpt-6-astra-sol-and-luna-for-production-agents-in-microsoft-foundry/", "supported_endpoints": [ "/v1/chat/completions", @@ -8591,7 +8595,11 @@ "output_cost_per_token_above_272k_tokens": 1.5e-05, "output_cost_per_token_above_272k_tokens_priority": 3e-05, "output_cost_per_token_priority": 2e-05, - "search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01}, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "source": "https://azure.microsoft.com/en-us/blog/gpt-6-astra-sol-and-luna-for-production-agents-in-microsoft-foundry/", "supported_endpoints": [ "/v1/chat/completions", @@ -9283,7 +9291,11 @@ "output_cost_per_token_above_272k_tokens": 1.65e-05, "output_cost_per_token_above_272k_tokens_priority": 3.3e-05, "output_cost_per_token_priority": 2.2e-05, - "search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01}, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "source": "https://azure.microsoft.com/en-us/blog/gpt-6-astra-sol-and-luna-for-production-agents-in-microsoft-foundry/", "supported_endpoints": [ "/v1/chat/completions", @@ -15879,7 +15891,10 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "provider_specific_entry": {"us":1.1,"fast":2}, + "provider_specific_entry": { + "us": 1.1, + "fast": 2 + }, "supports_output_config": true, "supports_speed": true, "supports_fast_mode": true, @@ -16006,6 +16021,14 @@ "output_cost_per_token": 0.0, "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" }, + "cloudflare/decisions/@cf/cloudflare/clef": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + }, "cloudflare/@cf/cloudflare/clef-flash": { "input_cost_per_token": 9e-08, "litellm_provider": "cloudflare", @@ -16014,6 +16037,14 @@ "output_cost_per_token": 0.0, "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" }, + "cloudflare/decisions/@cf/cloudflare/clef-flash": { + "input_cost_per_token": 9e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + }, "cloudflare/@cf/meta/llama-2-7b-chat-fp16": { "input_cost_per_token": 1.923e-06, "litellm_provider": "cloudflare", @@ -21177,6 +21208,18 @@ "output_cost_per_token": 0.0, "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models" }, + "databricks/decisions/databricks-openjev-qwen35-4b": { + "cache_creation_input_token_cost": 0.0, + "cache_read_input_token_cost": 0.0, + "input_cost_per_token": 0.0, + "litellm_provider": "databricks", + "metadata": { + "notes": "Pay-per-token Foundation Model API decision endpoint (OpenJev, Qwen 3.5 4B) queried through the System One API. Databricks published no DBU rate for it as of 2026-10-07, so a deployment's model_info prices set the billed cost." + }, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models" + }, "dataforseo/search": { "input_cost_per_query": 0.003, "litellm_provider": "dataforseo", @@ -29603,7 +29646,11 @@ "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, "rpm": 1000.0, - "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -29680,7 +29727,11 @@ "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, "rpm": 1000.0, - "search_context_cost_per_query": {"search_context_size_low":0.035,"search_context_size_medium":0.035,"search_context_size_high":0.035}, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + }, "source": "https://ai.google.dev/gemini-api/docs/models/deep-research-pro-preview-12-2025", "supported_endpoints": [ "/v1/chat/completions", @@ -34963,6 +35014,17 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "openai/decisions/gpt-6-luna": { + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, + "litellm_provider": "openai", + "max_input_tokens": 922000, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, + "source": "https://developers.openai.com/api/docs/guides/decisions#pricing-and-availability" + }, "gpt-5.6": { "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, @@ -38761,7 +38823,7 @@ "supports_xhigh_reasoning_effort": true }, "meta/muse-voice-transcribe-1.0": { - "input_cost_per_second": 0.00005, + "input_cost_per_second": 5e-05, "litellm_provider": "meta", "mode": "audio_transcription", "source": "https://dev.meta.ai/docs/speech-to-text", @@ -42944,6 +43006,14 @@ "output_cost_per_token": 0.0, "source": "https://docs.perplexity.ai/api-reference/decisions-post" }, + "perplexity/decisions/pplx-decider-v1-27b": { + "input_cost_per_token": 4e-08, + "litellm_provider": "perplexity", + "max_input_tokens": 262144, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://docs.perplexity.ai/api-reference/decisions-post" + }, "perplexity/sonar": { "input_cost_per_token": 1e-06, "litellm_provider": "perplexity", @@ -48929,7 +48999,6 @@ "/v1/images/edits", "/v1/batch" ] - }, "vertex_ai/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -48963,7 +49032,6 @@ "/v1/images/edits", "/v1/batch" ] - }, "vertex_ai/gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, @@ -48996,7 +49064,6 @@ "/v1/images/edits", "/v1/batch" ] - }, "vertex_ai/gemini-3.1-flash-image-preview": { "input_cost_per_image": 0.00056, @@ -49021,7 +49088,6 @@ "/v1/images/edits", "/v1/batch" ] - }, "vertex_ai/gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, @@ -59585,7 +59651,13 @@ { "name": "claude-adaptive-thinking", "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"], + "fill_missing_for_providers": [ + "anthropic", + "azure_ai", + "bedrock", + "bedrock_converse", + "vertex_ai-anthropic_models" + ], "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Turns on adaptive thinking for new versions and new families with no code change.", "model_info": { "supports_adaptive_thinking": true @@ -59594,7 +59666,13 @@ { "name": "claude-legacy-thinking", "pattern": "claude-[a-z]+-4[-._]6(?!\\d)", - "fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"], + "fill_missing_for_providers": [ + "anthropic", + "azure_ai", + "bedrock", + "bedrock_converse", + "vertex_ai-anthropic_models" + ], "description": "Claude at version 4.6 exactly, in any id shape that contains claude--4-6 (dotted and underscored minors included, dated releases such as claude-sonnet-4-6-20260219 too). The 4.6 family is adaptive-thinking yet still accepts legacy thinking.type=enabled with budget_tokens, so the caller's hard budget cap is forwarded verbatim instead of being rewritten to an uncapped output_config.effort. The lookahead keeps two-digit minors such as 4-60 from matching. 4.7+ and 5+ majors reject the legacy shape and stay on the adaptive translation.", "model_info": { "supports_legacy_thinking": true @@ -59611,7 +59689,13 @@ { "name": "claude-mid-conversation-system", "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"], + "fill_missing_for_providers": [ + "anthropic", + "azure_ai", + "bedrock", + "bedrock_converse", + "vertex_ai-anthropic_models" + ], "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", "model_info": { "supports_mid_conversation_system": true @@ -59620,7 +59704,12 @@ { "name": "claude-tool-search", "pattern": "claude-[a-z]+-(?:4[-._](?:[5-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "fill_missing_for_providers": ["anthropic", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"], + "fill_missing_for_providers": [ + "anthropic", + "bedrock", + "bedrock_converse", + "vertex_ai-anthropic_models" + ], "description": "Claude at version 4.5 or higher, in any id shape that contains claude--: minors 4.5 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Anthropic's tool search docs list every Claude 4.5 and newer model as supported and Opus 4.1 and earlier as unsupported, so the flag follows the version instead of a per-model list. azure_ai is left out on purpose: Anthropic documents tool search as unavailable on Azure-hosted Foundry deployments, and the azure_ai/ key cannot tell those from Anthropic-hosted ones.", "model_info": { "supports_tool_search": true @@ -59637,7 +59726,11 @@ { "name": "openai-reasoning-family-baseline", "pattern": "^(?!.*search-api)(?:[a-z0-9_.-]+/)*(?:ft:)?(?:o[1-9]\\d*(?![a-z0-9])|gpt-[5-9](?:\\.\\d+)?(?![0-9.])|(?:gpt-\\d+(?:\\.\\d+)?(?:-[a-z0-9]+)*-)?(?:codex|deep-research|chat-latest)(?![a-z0-9]))", - "fill_missing_for_providers": ["azure", "azure_ai", "openai"], + "fill_missing_for_providers": [ + "azure", + "azure_ai", + "openai" + ], "description": "OpenAI reasoning families by id shape, under any provider namespace and with an optional ft: prefix: the o-series (o1, o3-pro, o4-mini), gpt-5 through gpt-9 majors including dotted minors and suffixed variants (gpt-5.5-cyber, gpt-6-astra), and the codex, deep-research and chat-latest lines when standalone or on a gpt base. gpt-5-search-api is excluded because it is a search-only surface. Every model here is a reasoning model, and the Responses API drops the caller's reasoning param for any mapped OpenAI model whose info lacks supports_reasoning, so an id the registry has not named yet keeps its reasoning settings instead of silently losing them. Rules lose to exact entries. Carries no mode and no pricing, so cost stays on the standard unpriced behavior.", "model_info": { "supports_reasoning": true @@ -68033,6 +68126,20 @@ "notes": "Self-hosted decision model; infrastructure costs are paid separately" } }, + "bespoke/decisions/nimble-latest": { + "input_cost_per_token": 0.0, + "litellm_provider": "bespoke", + "max_input_tokens": 8192, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://github.com/bespokelabsai/nimble", + "supported_endpoints": [ + "/v1/systemone" + ], + "metadata": { + "notes": "Self-hosted decision model; infrastructure costs are paid separately" + } + }, "bespoke/nimble": { "input_cost_per_token": 0.0, "litellm_provider": "bespoke", @@ -68047,6 +68154,20 @@ "notes": "Self-hosted decision model under the name Ollama serves it as; infrastructure costs are paid separately" } }, + "bespoke/decisions/nimble": { + "input_cost_per_token": 0.0, + "litellm_provider": "bespoke", + "max_input_tokens": 8192, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://ollama.com/library/nimble", + "supported_endpoints": [ + "/v1/systemone" + ], + "metadata": { + "notes": "Self-hosted decision model under the name Ollama serves it as; infrastructure costs are paid separately" + } + }, "bespoke/bespokelabs/Bespoke-Nimble-9B": { "input_cost_per_token": 0.0, "litellm_provider": "bespoke", @@ -68061,6 +68182,20 @@ "notes": "Self-hosted decision model; infrastructure costs are paid separately" } }, + "bespoke/decisions/bespokelabs/Bespoke-Nimble-9B": { + "input_cost_per_token": 0.0, + "litellm_provider": "bespoke", + "max_input_tokens": 8192, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://github.com/bespokelabsai/nimble", + "supported_endpoints": [ + "/v1/systemone" + ], + "metadata": { + "notes": "Self-hosted decision model; infrastructure costs are paid separately" + } + }, "laya/english": { "input_cost_per_token": 0.0, "litellm_provider": "laya", @@ -68074,6 +68209,19 @@ "notes": "Self-hosted decision model; infrastructure costs are paid separately" } }, + "laya/decisions/english": { + "input_cost_per_token": 0.0, + "litellm_provider": "laya", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://github.com/NandhaKishorM/laya", + "supported_endpoints": [ + "/v1/systemone" + ], + "metadata": { + "notes": "Self-hosted decision model; infrastructure costs are paid separately" + } + }, "laya/multilingual": { "input_cost_per_token": 0.0, "litellm_provider": "laya", @@ -68087,6 +68235,19 @@ "notes": "Self-hosted decision model; infrastructure costs are paid separately" } }, + "laya/decisions/multilingual": { + "input_cost_per_token": 0.0, + "litellm_provider": "laya", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://github.com/NandhaKishorM/laya", + "supported_endpoints": [ + "/v1/systemone" + ], + "metadata": { + "notes": "Self-hosted decision model; infrastructure costs are paid separately" + } + }, "laya/typed-decisions": { "input_cost_per_token": 0.0, "litellm_provider": "laya", @@ -68100,6 +68261,19 @@ "notes": "Self-hosted decision model; infrastructure costs are paid separately" } }, + "laya/decisions/typed-decisions": { + "input_cost_per_token": 0.0, + "litellm_provider": "laya", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://github.com/NandhaKishorM/laya", + "supported_endpoints": [ + "/v1/systemone" + ], + "metadata": { + "notes": "Self-hosted decision model; infrastructure costs are paid separately" + } + }, "strands_decider/strands-decider-2B-hobson-v19": { "input_cost_per_token": 0.0, "litellm_provider": "strands_decider", @@ -68110,6 +68284,16 @@ "notes": "Self-hosted decision model; infrastructure costs are paid separately" } }, + "strands_decider/decisions/strands-decider-2B-hobson-v19": { + "input_cost_per_token": 0.0, + "litellm_provider": "strands_decider", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://huggingface.co/StrandsAgents/strands-decider-2B-hobson-v19", + "metadata": { + "notes": "Self-hosted decision model; infrastructure costs are paid separately" + } + }, "typesafe/jev-1.13.0": { "input_cost_per_token": 4.2e-08, "litellm_provider": "typesafe", @@ -68117,6 +68301,13 @@ "output_cost_per_token": 0.0, "source": "https://docs.typesafe.ai/models" }, + "typesafe/decisions/jev-1.13.0": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "typesafe", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://docs.typesafe.ai/models" + }, "typesafe/jev-latest": { "input_cost_per_token": 4.2e-08, "litellm_provider": "typesafe", @@ -68124,6 +68315,13 @@ "output_cost_per_token": 0.0, "source": "https://docs.typesafe.ai/models" }, + "typesafe/decisions/jev-latest": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "typesafe", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://docs.typesafe.ai/models" + }, "typesafe/jev-preview": { "input_cost_per_token": 4.2e-08, "litellm_provider": "typesafe", @@ -68131,6 +68329,13 @@ "output_cost_per_token": 0.0, "source": "https://docs.typesafe.ai/models" }, + "typesafe/decisions/jev-preview": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "typesafe", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://docs.typesafe.ai/models" + }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 2e-07, @@ -69110,7 +69315,9 @@ "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", - "supported_regions": ["global"], + "supported_regions": [ + "global" + ], "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -70374,7 +70581,9 @@ }, "global.xai.grok-4.7": { "supports_regex_lookaround": false, - "supported_endpoints": ["/v1/chat/completions"], + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_bedrock_runtime_chat_completions_response_format": true, "cache_read_input_token_cost": 5e-07, @@ -70400,7 +70609,9 @@ }, "us.xai.grok-4.7": { "supports_regex_lookaround": false, - "supported_endpoints": ["/v1/chat/completions"], + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_bedrock_runtime_chat_completions_response_format": true, "cache_read_input_token_cost": 5.5e-07, @@ -70848,7 +71059,11 @@ "output_cost_per_token": 7.5e-06, "output_cost_per_token_batches": 3.75e-06, "rpm": 1000.0, - "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -70893,7 +71108,11 @@ "output_cost_per_token": 7.5e-06, "output_cost_per_token_batches": 3.75e-06, "output_cost_per_token_priority": 1.35e-05, - "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -70939,7 +71158,11 @@ "output_cost_per_token": 7.5e-06, "output_cost_per_token_batches": 3.75e-06, "output_cost_per_token_priority": 1.35e-05, - "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -71019,7 +71242,7 @@ "supports_tool_choice": true, "supports_vision": true }, -"claude-haiku-5-5": { + "claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens_batches": 2.5e-07, "output_cost_per_token_above_100k_tokens_batches": 1.25e-06, "cache_creation_input_token_cost_above_100k_tokens_batches": 3.125e-07, @@ -71787,6 +72010,13 @@ "output_cost_per_token": 0.0, "source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'" }, + "azure_ai/decisions/Microsoft-Decision-1": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "azure_ai", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'" + }, "gpt-rosalind-discovery": { "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index bf7f1db7a31..a28bffa8c3e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -8538,7 +8538,11 @@ "output_cost_per_token_above_272k_tokens": 1.5e-05, "output_cost_per_token_above_272k_tokens_priority": 3e-05, "output_cost_per_token_priority": 2e-05, - "search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01}, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "source": "https://azure.microsoft.com/en-us/blog/gpt-6-astra-sol-and-luna-for-production-agents-in-microsoft-foundry/", "supported_endpoints": [ "/v1/chat/completions", @@ -8591,7 +8595,11 @@ "output_cost_per_token_above_272k_tokens": 1.5e-05, "output_cost_per_token_above_272k_tokens_priority": 3e-05, "output_cost_per_token_priority": 2e-05, - "search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01}, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "source": "https://azure.microsoft.com/en-us/blog/gpt-6-astra-sol-and-luna-for-production-agents-in-microsoft-foundry/", "supported_endpoints": [ "/v1/chat/completions", @@ -9283,7 +9291,11 @@ "output_cost_per_token_above_272k_tokens": 1.65e-05, "output_cost_per_token_above_272k_tokens_priority": 3.3e-05, "output_cost_per_token_priority": 2.2e-05, - "search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01}, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "source": "https://azure.microsoft.com/en-us/blog/gpt-6-astra-sol-and-luna-for-production-agents-in-microsoft-foundry/", "supported_endpoints": [ "/v1/chat/completions", @@ -15879,7 +15891,10 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "provider_specific_entry": {"us":1.1,"fast":2}, + "provider_specific_entry": { + "us": 1.1, + "fast": 2 + }, "supports_output_config": true, "supports_speed": true, "supports_fast_mode": true, @@ -16006,6 +16021,14 @@ "output_cost_per_token": 0.0, "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" }, + "cloudflare/decisions/@cf/cloudflare/clef": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + }, "cloudflare/@cf/cloudflare/clef-flash": { "input_cost_per_token": 9e-08, "litellm_provider": "cloudflare", @@ -16014,6 +16037,14 @@ "output_cost_per_token": 0.0, "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" }, + "cloudflare/decisions/@cf/cloudflare/clef-flash": { + "input_cost_per_token": 9e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/" + }, "cloudflare/@cf/meta/llama-2-7b-chat-fp16": { "input_cost_per_token": 1.923e-06, "litellm_provider": "cloudflare", @@ -21177,6 +21208,18 @@ "output_cost_per_token": 0.0, "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models" }, + "databricks/decisions/databricks-openjev-qwen35-4b": { + "cache_creation_input_token_cost": 0.0, + "cache_read_input_token_cost": 0.0, + "input_cost_per_token": 0.0, + "litellm_provider": "databricks", + "metadata": { + "notes": "Pay-per-token Foundation Model API decision endpoint (OpenJev, Qwen 3.5 4B) queried through the System One API. Databricks published no DBU rate for it as of 2026-10-07, so a deployment's model_info prices set the billed cost." + }, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models" + }, "dataforseo/search": { "input_cost_per_query": 0.003, "litellm_provider": "dataforseo", @@ -29603,7 +29646,11 @@ "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, "rpm": 1000.0, - "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -29680,7 +29727,11 @@ "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, "rpm": 1000.0, - "search_context_cost_per_query": {"search_context_size_low":0.035,"search_context_size_medium":0.035,"search_context_size_high":0.035}, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + }, "source": "https://ai.google.dev/gemini-api/docs/models/deep-research-pro-preview-12-2025", "supported_endpoints": [ "/v1/chat/completions", @@ -34963,6 +35014,17 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "openai/decisions/gpt-6-luna": { + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, + "litellm_provider": "openai", + "max_input_tokens": 922000, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, + "source": "https://developers.openai.com/api/docs/guides/decisions#pricing-and-availability" + }, "gpt-5.6": { "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, @@ -38761,7 +38823,7 @@ "supports_xhigh_reasoning_effort": true }, "meta/muse-voice-transcribe-1.0": { - "input_cost_per_second": 0.00005, + "input_cost_per_second": 5e-05, "litellm_provider": "meta", "mode": "audio_transcription", "source": "https://dev.meta.ai/docs/speech-to-text", @@ -42944,6 +43006,14 @@ "output_cost_per_token": 0.0, "source": "https://docs.perplexity.ai/api-reference/decisions-post" }, + "perplexity/decisions/pplx-decider-v1-27b": { + "input_cost_per_token": 4e-08, + "litellm_provider": "perplexity", + "max_input_tokens": 262144, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://docs.perplexity.ai/api-reference/decisions-post" + }, "perplexity/sonar": { "input_cost_per_token": 1e-06, "litellm_provider": "perplexity", @@ -48929,7 +48999,6 @@ "/v1/images/edits", "/v1/batch" ] - }, "vertex_ai/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -48963,7 +49032,6 @@ "/v1/images/edits", "/v1/batch" ] - }, "vertex_ai/gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, @@ -48996,7 +49064,6 @@ "/v1/images/edits", "/v1/batch" ] - }, "vertex_ai/gemini-3.1-flash-image-preview": { "input_cost_per_image": 0.00056, @@ -49021,7 +49088,6 @@ "/v1/images/edits", "/v1/batch" ] - }, "vertex_ai/gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, @@ -59585,7 +59651,13 @@ { "name": "claude-adaptive-thinking", "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"], + "fill_missing_for_providers": [ + "anthropic", + "azure_ai", + "bedrock", + "bedrock_converse", + "vertex_ai-anthropic_models" + ], "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Turns on adaptive thinking for new versions and new families with no code change.", "model_info": { "supports_adaptive_thinking": true @@ -59594,7 +59666,13 @@ { "name": "claude-legacy-thinking", "pattern": "claude-[a-z]+-4[-._]6(?!\\d)", - "fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"], + "fill_missing_for_providers": [ + "anthropic", + "azure_ai", + "bedrock", + "bedrock_converse", + "vertex_ai-anthropic_models" + ], "description": "Claude at version 4.6 exactly, in any id shape that contains claude--4-6 (dotted and underscored minors included, dated releases such as claude-sonnet-4-6-20260219 too). The 4.6 family is adaptive-thinking yet still accepts legacy thinking.type=enabled with budget_tokens, so the caller's hard budget cap is forwarded verbatim instead of being rewritten to an uncapped output_config.effort. The lookahead keeps two-digit minors such as 4-60 from matching. 4.7+ and 5+ majors reject the legacy shape and stay on the adaptive translation.", "model_info": { "supports_legacy_thinking": true @@ -59611,7 +59689,13 @@ { "name": "claude-mid-conversation-system", "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"], + "fill_missing_for_providers": [ + "anthropic", + "azure_ai", + "bedrock", + "bedrock_converse", + "vertex_ai-anthropic_models" + ], "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", "model_info": { "supports_mid_conversation_system": true @@ -59620,7 +59704,12 @@ { "name": "claude-tool-search", "pattern": "claude-[a-z]+-(?:4[-._](?:[5-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "fill_missing_for_providers": ["anthropic", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"], + "fill_missing_for_providers": [ + "anthropic", + "bedrock", + "bedrock_converse", + "vertex_ai-anthropic_models" + ], "description": "Claude at version 4.5 or higher, in any id shape that contains claude--: minors 4.5 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Anthropic's tool search docs list every Claude 4.5 and newer model as supported and Opus 4.1 and earlier as unsupported, so the flag follows the version instead of a per-model list. azure_ai is left out on purpose: Anthropic documents tool search as unavailable on Azure-hosted Foundry deployments, and the azure_ai/ key cannot tell those from Anthropic-hosted ones.", "model_info": { "supports_tool_search": true @@ -59637,7 +59726,11 @@ { "name": "openai-reasoning-family-baseline", "pattern": "^(?!.*search-api)(?:[a-z0-9_.-]+/)*(?:ft:)?(?:o[1-9]\\d*(?![a-z0-9])|gpt-[5-9](?:\\.\\d+)?(?![0-9.])|(?:gpt-\\d+(?:\\.\\d+)?(?:-[a-z0-9]+)*-)?(?:codex|deep-research|chat-latest)(?![a-z0-9]))", - "fill_missing_for_providers": ["azure", "azure_ai", "openai"], + "fill_missing_for_providers": [ + "azure", + "azure_ai", + "openai" + ], "description": "OpenAI reasoning families by id shape, under any provider namespace and with an optional ft: prefix: the o-series (o1, o3-pro, o4-mini), gpt-5 through gpt-9 majors including dotted minors and suffixed variants (gpt-5.5-cyber, gpt-6-astra), and the codex, deep-research and chat-latest lines when standalone or on a gpt base. gpt-5-search-api is excluded because it is a search-only surface. Every model here is a reasoning model, and the Responses API drops the caller's reasoning param for any mapped OpenAI model whose info lacks supports_reasoning, so an id the registry has not named yet keeps its reasoning settings instead of silently losing them. Rules lose to exact entries. Carries no mode and no pricing, so cost stays on the standard unpriced behavior.", "model_info": { "supports_reasoning": true @@ -68033,6 +68126,20 @@ "notes": "Self-hosted decision model; infrastructure costs are paid separately" } }, + "bespoke/decisions/nimble-latest": { + "input_cost_per_token": 0.0, + "litellm_provider": "bespoke", + "max_input_tokens": 8192, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://github.com/bespokelabsai/nimble", + "supported_endpoints": [ + "/v1/systemone" + ], + "metadata": { + "notes": "Self-hosted decision model; infrastructure costs are paid separately" + } + }, "bespoke/nimble": { "input_cost_per_token": 0.0, "litellm_provider": "bespoke", @@ -68047,6 +68154,20 @@ "notes": "Self-hosted decision model under the name Ollama serves it as; infrastructure costs are paid separately" } }, + "bespoke/decisions/nimble": { + "input_cost_per_token": 0.0, + "litellm_provider": "bespoke", + "max_input_tokens": 8192, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://ollama.com/library/nimble", + "supported_endpoints": [ + "/v1/systemone" + ], + "metadata": { + "notes": "Self-hosted decision model under the name Ollama serves it as; infrastructure costs are paid separately" + } + }, "bespoke/bespokelabs/Bespoke-Nimble-9B": { "input_cost_per_token": 0.0, "litellm_provider": "bespoke", @@ -68061,6 +68182,20 @@ "notes": "Self-hosted decision model; infrastructure costs are paid separately" } }, + "bespoke/decisions/bespokelabs/Bespoke-Nimble-9B": { + "input_cost_per_token": 0.0, + "litellm_provider": "bespoke", + "max_input_tokens": 8192, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://github.com/bespokelabsai/nimble", + "supported_endpoints": [ + "/v1/systemone" + ], + "metadata": { + "notes": "Self-hosted decision model; infrastructure costs are paid separately" + } + }, "laya/english": { "input_cost_per_token": 0.0, "litellm_provider": "laya", @@ -68074,6 +68209,19 @@ "notes": "Self-hosted decision model; infrastructure costs are paid separately" } }, + "laya/decisions/english": { + "input_cost_per_token": 0.0, + "litellm_provider": "laya", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://github.com/NandhaKishorM/laya", + "supported_endpoints": [ + "/v1/systemone" + ], + "metadata": { + "notes": "Self-hosted decision model; infrastructure costs are paid separately" + } + }, "laya/multilingual": { "input_cost_per_token": 0.0, "litellm_provider": "laya", @@ -68087,6 +68235,19 @@ "notes": "Self-hosted decision model; infrastructure costs are paid separately" } }, + "laya/decisions/multilingual": { + "input_cost_per_token": 0.0, + "litellm_provider": "laya", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://github.com/NandhaKishorM/laya", + "supported_endpoints": [ + "/v1/systemone" + ], + "metadata": { + "notes": "Self-hosted decision model; infrastructure costs are paid separately" + } + }, "laya/typed-decisions": { "input_cost_per_token": 0.0, "litellm_provider": "laya", @@ -68100,6 +68261,19 @@ "notes": "Self-hosted decision model; infrastructure costs are paid separately" } }, + "laya/decisions/typed-decisions": { + "input_cost_per_token": 0.0, + "litellm_provider": "laya", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://github.com/NandhaKishorM/laya", + "supported_endpoints": [ + "/v1/systemone" + ], + "metadata": { + "notes": "Self-hosted decision model; infrastructure costs are paid separately" + } + }, "strands_decider/strands-decider-2B-hobson-v19": { "input_cost_per_token": 0.0, "litellm_provider": "strands_decider", @@ -68110,6 +68284,16 @@ "notes": "Self-hosted decision model; infrastructure costs are paid separately" } }, + "strands_decider/decisions/strands-decider-2B-hobson-v19": { + "input_cost_per_token": 0.0, + "litellm_provider": "strands_decider", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://huggingface.co/StrandsAgents/strands-decider-2B-hobson-v19", + "metadata": { + "notes": "Self-hosted decision model; infrastructure costs are paid separately" + } + }, "typesafe/jev-1.13.0": { "input_cost_per_token": 4.2e-08, "litellm_provider": "typesafe", @@ -68117,6 +68301,13 @@ "output_cost_per_token": 0.0, "source": "https://docs.typesafe.ai/models" }, + "typesafe/decisions/jev-1.13.0": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "typesafe", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://docs.typesafe.ai/models" + }, "typesafe/jev-latest": { "input_cost_per_token": 4.2e-08, "litellm_provider": "typesafe", @@ -68124,6 +68315,13 @@ "output_cost_per_token": 0.0, "source": "https://docs.typesafe.ai/models" }, + "typesafe/decisions/jev-latest": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "typesafe", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://docs.typesafe.ai/models" + }, "typesafe/jev-preview": { "input_cost_per_token": 4.2e-08, "litellm_provider": "typesafe", @@ -68131,6 +68329,13 @@ "output_cost_per_token": 0.0, "source": "https://docs.typesafe.ai/models" }, + "typesafe/decisions/jev-preview": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "typesafe", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://docs.typesafe.ai/models" + }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 2e-07, @@ -69110,7 +69315,9 @@ "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", - "supported_regions": ["global"], + "supported_regions": [ + "global" + ], "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -70374,7 +70581,9 @@ }, "global.xai.grok-4.7": { "supports_regex_lookaround": false, - "supported_endpoints": ["/v1/chat/completions"], + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_bedrock_runtime_chat_completions_response_format": true, "cache_read_input_token_cost": 5e-07, @@ -70400,7 +70609,9 @@ }, "us.xai.grok-4.7": { "supports_regex_lookaround": false, - "supported_endpoints": ["/v1/chat/completions"], + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_bedrock_runtime_chat_completions_response_format": true, "cache_read_input_token_cost": 5.5e-07, @@ -70848,7 +71059,11 @@ "output_cost_per_token": 7.5e-06, "output_cost_per_token_batches": 3.75e-06, "rpm": 1000.0, - "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -70893,7 +71108,11 @@ "output_cost_per_token": 7.5e-06, "output_cost_per_token_batches": 3.75e-06, "output_cost_per_token_priority": 1.35e-05, - "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -70939,7 +71158,11 @@ "output_cost_per_token": 7.5e-06, "output_cost_per_token_batches": 3.75e-06, "output_cost_per_token_priority": 1.35e-05, - "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -71019,7 +71242,7 @@ "supports_tool_choice": true, "supports_vision": true }, -"claude-haiku-5-5": { + "claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens_batches": 2.5e-07, "output_cost_per_token_above_100k_tokens_batches": 1.25e-06, "cache_creation_input_token_cost_above_100k_tokens_batches": 3.125e-07, @@ -71787,6 +72010,13 @@ "output_cost_per_token": 0.0, "source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'" }, + "azure_ai/decisions/Microsoft-Decision-1": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "azure_ai", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'" + }, "gpt-rosalind-discovery": { "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index c4e88166388..25a578f7afa 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -2,9 +2,10 @@ PARITY pairs the streamed case with its plain twin where both bill the same row.""" -from typing import Final +from typing import Final, cast from integration.cost_calculation.cost_tracking_case import ( + COST_MAP, BinaryResponse, CostTrackingTestCase, Deployment, @@ -18,6 +19,7 @@ from integration.cost_calculation.cost_tracking_case import ( SseResponse, WavUpload, ) +from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_key from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames @@ -7848,6 +7850,74 @@ GPT_6_1_SOL_INPUT_TEXT: Final = CostTrackingTestCase( ) +_OPENAI_GPT_6_LUNA_DECISIONS_MODEL: Final = decisions_pricing_key("openai", "gpt-6-luna") +_OPENAI_GPT_6_LUNA_DECISIONS_INPUT_TOKENS: Final = 367 +_OPENAI_GPT_6_LUNA_DECISIONS_OUTPUT_TOKENS: Final = 3 +_OPENAI_GPT_6_LUNA_DECISIONS_INPUT_RATE: Final = cast( + float, + COST_MAP[_OPENAI_GPT_6_LUNA_DECISIONS_MODEL].input_cost_per_token, +) +_OPENAI_GPT_6_LUNA_DECISIONS_SPEND: Final = ( + _OPENAI_GPT_6_LUNA_DECISIONS_INPUT_TOKENS * _OPENAI_GPT_6_LUNA_DECISIONS_INPUT_RATE +) +_OPENAI_GPT_6_LUNA_DECISIONS_RESPONSE: Final = JsonResponse( + content_type="application/json", + body={ + "model": "gpt-6-luna", + "answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}], + "usage": { + "input_tokens": _OPENAI_GPT_6_LUNA_DECISIONS_INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": 256, "cache_write_tokens": 64}, + "output_tokens": _OPENAI_GPT_6_LUNA_DECISIONS_OUTPUT_TOKENS, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": _OPENAI_GPT_6_LUNA_DECISIONS_INPUT_TOKENS + _OPENAI_GPT_6_LUNA_DECISIONS_OUTPUT_TOKENS, + }, + }, +) + +OPENAI_GPT_6_LUNA_DECISIONS: Final = CostTrackingTestCase( + name="openai-gpt-6-luna-decisions-evaluation-pricing", + covers="quota_management.spend_tracking.decisions_costs", + model="gpt-6-luna", + endpoint="/v1/decisions", + deployment=Deployment(model="openai/gpt-6-luna"), + request={ + "model": "$MODEL", + "input": "The package arrived broken.", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + }, + response=_OPENAI_GPT_6_LUNA_DECISIONS_RESPONSE, + expected=ExactExpected( + spend=_OPENAI_GPT_6_LUNA_DECISIONS_SPEND, + input_cost=_OPENAI_GPT_6_LUNA_DECISIONS_SPEND, + output_cost=0.0, + prompt_tokens=_OPENAI_GPT_6_LUNA_DECISIONS_INPUT_TOKENS, + completion_tokens=_OPENAI_GPT_6_LUNA_DECISIONS_OUTPUT_TOKENS, + ), +) + +OPENAI_GPT_6_LUNA_SYSTEMONE: Final = CostTrackingTestCase( + name="openai-gpt-6-luna-systemone-evaluation-pricing", + covers="quota_management.spend_tracking.decisions_costs", + model=_OPENAI_GPT_6_LUNA_DECISIONS_MODEL, + endpoint="/v1/systemone", + deployment=Deployment(model="openai/gpt-6-luna"), + request={ + "model": "$MODEL", + "state": "The package arrived broken.", + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + }, + response=_OPENAI_GPT_6_LUNA_DECISIONS_RESPONSE, + expected=ExactExpected( + spend=_OPENAI_GPT_6_LUNA_DECISIONS_SPEND, + input_cost=_OPENAI_GPT_6_LUNA_DECISIONS_SPEND, + output_cost=0.0, + prompt_tokens=_OPENAI_GPT_6_LUNA_DECISIONS_INPUT_TOKENS, + completion_tokens=_OPENAI_GPT_6_LUNA_DECISIONS_OUTPUT_TOKENS, + ), +) + + CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GPT_5_3_CODEX_INPUT_TEXT, GPT_5_3_CODEX_CACHE_READ, @@ -7980,6 +8050,8 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GPT_5_6_CHAT_REQUEST_TO_EMBEDDING_ENTRY, GPT_5_6_CLIENT_DISCONNECT_MID_STREAM, GPT_6_1_SOL_INPUT_TEXT, + OPENAI_GPT_6_LUNA_DECISIONS, + OPENAI_GPT_6_LUNA_SYSTEMONE, ) PARITY: Final[tuple[StreamParityTestCase, ...]] = ( diff --git a/tests/integration/cost_calculation/catalog.py b/tests/integration/cost_calculation/catalog.py index fe7367aed8a..da16922f52d 100644 --- a/tests/integration/cost_calculation/catalog.py +++ b/tests/integration/cost_calculation/catalog.py @@ -5,6 +5,8 @@ from __future__ import annotations import json from typing import Final +from litellm.litellm_core_utils.decisions_pricing import is_decisions_pricing_key + from integration.cost_calculation.cases import ( anthropic, azure, @@ -61,14 +63,18 @@ def data_errors() -> tuple[str, ...]: case.session_model for case in REALTIME_CASES if case.session_model is not None ) unknown_models: Final = sorted(model for model in case_models if model not in COST_MAP) - missing_cases: Final = sorted(model for model in COST_MAP if model not in case_models) + missing_cases: Final = sorted( + model + for model, entry in COST_MAP.items() + if model not in case_models and not is_decisions_pricing_key(model, entry.litellm_provider) + ) duplicate_names: Final = sorted( name for name in {case.name for case in _ALL_CASES} if sum(case.name == name for case in _ALL_CASES) > 1 ) input_rates: Final = tuple( (entry.litellm_provider, entry.input_cost_per_token, model) for model, entry in COST_MAP.items() - if entry.mode != "realtime" + if entry.mode != "realtime" and not is_decisions_pricing_key(model, entry.litellm_provider) ) shared_input_rate_details: Final = tuple( ( diff --git a/tests/integration/cost_calculation/cost_map.py b/tests/integration/cost_calculation/cost_map.py index 2b597aaadb8..10f1507c9b0 100644 --- a/tests/integration/cost_calculation/cost_map.py +++ b/tests/integration/cost_calculation/cost_map.py @@ -4,8 +4,17 @@ Plain JSON-shaped data: cost_tracking_case.py validates it into CostMapEntry and from typing import Final +from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_key from pydantic import JsonValue +_PERPLEXITY_DECISION_COST: Final[dict[str, JsonValue]] = { + "input_cost_per_token": 4e-08, + "litellm_provider": "perplexity", + "max_input_tokens": 262144, + "mode": "evaluation", + "output_cost_per_token": 0.0, +} + COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "dashscope/qwen4-max": { "litellm_provider": "dashscope", @@ -32,6 +41,26 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "input_cost_per_token": 3.2e-06, "output_cost_per_token": 1.6e-05, }, + "gpt-6-luna": { + "cache_creation_input_token_cost": 1.25e-07, + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 1e-07, + "litellm_provider": "openai", + "max_input_tokens": 922000, + "mode": "chat", + "output_cost_per_token": 5e-07, + }, + "openai/decisions/gpt-6-luna": { + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, + "litellm_provider": "openai", + "max_input_tokens": 922000, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, + "source": "https://developers.openai.com/api/docs/guides/decisions#pricing-and-availability", + }, "perplexity/sonar-next": { "litellm_provider": "perplexity", "mode": "chat", @@ -43,13 +72,8 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "search_context_size_high": 0.012, }, }, - "perplexity/pplx-decider-v1-27b": { - "input_cost_per_token": 4e-08, - "litellm_provider": "perplexity", - "max_input_tokens": 262144, - "mode": "evaluation", - "output_cost_per_token": 0.0, - }, + "perplexity/pplx-decider-v1-27b": _PERPLEXITY_DECISION_COST, + decisions_pricing_key("perplexity", "pplx-decider-v1-27b"): _PERPLEXITY_DECISION_COST, "deepseek/deepseek-v4-chat": { "litellm_provider": "deepseek", "mode": "chat", diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py index a146401f3ae..dda3ea700d2 100644 --- a/tests/integration/cost_calculation/cost_tracking_case.py +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -44,8 +44,12 @@ class CostMapEntry(BaseModel): max_output_tokens: int | None = None supports_function_calling: bool | None = None input_cost_per_token: float | None = None + input_cost_per_token_above_272k_tokens: float | None = None input_cost_per_query: float | None = None output_cost_per_token: float | None = None + regional_processing_uplift_multiplier_eu: float | None = None + regional_processing_uplift_multiplier_us: float | None = None + source: str | None = None input_cost_per_token_batches: float | None = None output_cost_per_token_batches: float | None = None output_cost_per_image_token_batches: float | None = None @@ -249,6 +253,7 @@ class CostTrackingTestCase(BaseModel): "/v1/audio/speech", "/v1/images/generations", "/v1/images/edits", + "/v1/decisions", "/v1/systemone", ] | Annotated[str, Field(pattern=r"^/(gemini|anthropic|bedrock)/")] diff --git a/tests/integration/cost_calculation/test_cost_tracking.py b/tests/integration/cost_calculation/test_cost_tracking.py index a84c31e810d..d8a20c642f0 100644 --- a/tests/integration/cost_calculation/test_cost_tracking.py +++ b/tests/integration/cost_calculation/test_cost_tracking.py @@ -252,7 +252,7 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) assert row.spend == 0, f"{case.name}: failure spend was {row.spend}" return assert response.is_success, f"{case.name}: proxy returned {response.status_code}: {response.text[:400]}" - if case.endpoint == "/v1/systemone": + if case.endpoint in {"/v1/decisions", "/v1/systemone"}: observed: Final = JSON_OBJECT.validate_json( httpx.get(f"{gateway.upstream_url}/__observations", timeout=5, trust_env=False).content ) @@ -261,22 +261,37 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) for value in observed["requests"] if isinstance(value, dict) and value.get("path") == f"/{scenario_id}/v1/decisions" ) + expected_body: Final = ( + { + "model": "pplx-decider-v1-27b", + "state": {"source": "cost-tracking"}, + "questions": { + "is_defect": { + "type": "noul", + "instructions": "Is this a defect?", + } + }, + } + if case.rates.litellm_provider == "perplexity" + else { + "model": "gpt-6-luna", + "input": case.request["input"] if case.endpoint == "/v1/decisions" else case.request["state"], + "questions": [ + { + "type": "predicate", + "name": "is_defect", + "instructions": "Is this a defect?", + } + ], + } + ) assert decision_observations == ( { "path": f"/{scenario_id}/v1/decisions", "authorization": "Bearer sk-scripted-provider", "method": "POST", "api_key": "", - "body": { - "model": "pplx-decider-v1-27b", - "state": {"source": "cost-tracking"}, - "questions": { - "is_defect": { - "type": "noul", - "instructions": "Is this a defect?", - } - }, - }, + "body": expected_body, }, ) if case.response.content_type == "text/event-stream": diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 81f5b60a036..5ed96e9b616 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -19,6 +19,7 @@ import litellm from litellm.cost_calculator import get_response_cost_from_hidden_params from litellm.constants import AWS_SIGNING_MAX_THREADS from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_key from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.bedrock.base_aws_llm import AWS_SIGNING_EXECUTOR from litellm.types.decisions import ( @@ -307,6 +308,82 @@ def test_decisions_cost_uses_litellm_token_pricing() -> None: assert cost == pytest.approx(expected_cost) +@pytest.mark.asyncio +async def test_openai_decisions_uses_evaluation_pricing_and_explicit_base_model( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE) + response: Final = await litellm.adecisions( + model="openai/gpt-6-luna", + input="The package arrived broken.", + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + api_key="caller-key", + api_base="https://api.openai.com/v1", + ) + decisions_key: Final = decisions_pricing_key("openai", "gpt-6-luna") + evaluation_cost: Final = litellm.model_cost[decisions_key] + chat_cost: Final = litellm.model_cost["gpt-6-luna"] + expected_chat_cost: Final = ( + (_INPUT_TOKENS - _CACHED_TOKENS - _CACHE_WRITE_TOKENS) * float(chat_cost["input_cost_per_token"]) + + _CACHED_TOKENS * float(chat_cost["cache_read_input_token_cost"]) + + _CACHE_WRITE_TOKENS * float(chat_cost["cache_creation_input_token_cost"]) + + _OUTPUT_TOKENS * float(chat_cost["output_cost_per_token"]) + ) + + assert route.called + assert response._hidden_params["model"] == "openai/gpt-6-luna" + assert response._hidden_params["provider_response_model"] == decisions_key + assert litellm.completion_cost(completion_response=response) == pytest.approx( + _INPUT_TOKENS * float(evaluation_cost["input_cost_per_token"]) + ) + assert litellm.completion_cost(completion_response=response, base_model="gpt-6-luna") == pytest.approx( + expected_chat_cost + ) + + +@pytest.mark.asyncio +async def test_systemone_uses_evaluation_pricing(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE) + response: Final = await litellm.asystemone( + model="openai/gpt-6-luna", + state="The package arrived broken.", + questions=_QUESTIONS, + api_key="caller-key", + api_base="https://api.openai.com/v1", + ) + decisions_key: Final = decisions_pricing_key("openai", "gpt-6-luna") + evaluation_cost: Final = litellm.model_cost[decisions_key] + + assert route.called + assert response._hidden_params["model"] == "openai/gpt-6-luna" + assert response._hidden_params["provider_response_model"] == decisions_key + assert litellm.completion_cost(completion_response=response) == pytest.approx( + _INPUT_TOKENS * float(evaluation_cost["input_cost_per_token"]) + ) + + +@pytest.mark.asyncio +async def test_systemone_falls_back_to_plain_pricing_without_decisions_row( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post("https://decisions.example/v1/decisions").respond(json=_RESPONSE) + response: Final = await litellm.asystemone( + model="perplexity/sonar", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + api_base="https://decisions.example/v1", + ) + plain_cost: Final = litellm.model_cost["perplexity/sonar"] + + assert route.called + assert response._hidden_params["provider_response_model"] == "perplexity/sonar" + assert litellm.completion_cost(completion_response=response) == pytest.approx( + _INPUT_TOKENS * float(plain_cost["input_cost_per_token"]) + + _OUTPUT_TOKENS * float(plain_cost["output_cost_per_token"]) + ) + + @pytest.mark.parametrize( "response", ( @@ -1293,13 +1370,8 @@ async def test_openai_decisions_translate_systemone_to_the_openai_wire_contract_ } assert response.answers == {"is_defect": NoulAnswer(type="noul", noul=0.9)} assert response.hidden_params["custom_llm_provider"] == "openai" - luna_cost: Final = litellm.model_cost["gpt-6-luna"] - expected_cost: Final = ( - (_INPUT_TOKENS - _CACHED_TOKENS - _CACHE_WRITE_TOKENS) * float(luna_cost["input_cost_per_token"]) - + _CACHED_TOKENS * float(luna_cost["cache_read_input_token_cost"]) - + _CACHE_WRITE_TOKENS * float(luna_cost["cache_creation_input_token_cost"]) - + _OUTPUT_TOKENS * float(luna_cost["output_cost_per_token"]) - ) + evaluation_cost: Final = litellm.model_cost[decisions_pricing_key("openai", "gpt-6-luna")] + expected_cost: Final = _INPUT_TOKENS * float(evaluation_cost["input_cost_per_token"]) assert expected_cost > 0 assert litellm.completion_cost(completion_response=response) == pytest.approx(expected_cost) @@ -1638,6 +1710,28 @@ async def test_azure_ai_decision_posts_deployment_to_foundry_systemone_route( assert response._hidden_params["model"] == "azure_ai/decision-1" +@pytest.mark.asyncio +async def test_azure_ai_systemone_uses_its_decisions_pricing_row(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://res.services.ai.azure.com/providers/microsoft/v1/systemone").respond( + json={**_RESPONSE, "model": "microsoft-decision-1"} + ) + response: Final = await litellm.asystemone( + model="azure_ai/Microsoft-Decision-1", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_base="https://res.services.ai.azure.com", + api_key="foundry-key", + ) + pricing_key: Final = decisions_pricing_key("azure_ai", "Microsoft-Decision-1") + evaluation_cost: Final = litellm.model_cost[pricing_key] + + assert route.called + assert response._hidden_params["provider_response_model"] == pricing_key + assert litellm.completion_cost(completion_response=response) == pytest.approx( + _INPUT_TOKENS * float(evaluation_cost["input_cost_per_token"]) + ) + + @pytest.mark.asyncio async def test_azure_ai_decision_reads_foundry_env_and_keeps_gateway_path_prefix( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/litellm_core_utils/test_decisions_pricing.py b/tests/unit/litellm_core_utils/test_decisions_pricing.py new file mode 100644 index 00000000000..01cadcf764b --- /dev/null +++ b/tests/unit/litellm_core_utils/test_decisions_pricing.py @@ -0,0 +1,33 @@ +from collections.abc import Mapping +from typing import Final + +from litellm.litellm_core_utils.decisions_pricing import ( + decisions_pricing_key, + decisions_pricing_model, + is_decisions_pricing_key, +) + + +def test_decisions_pricing_key_uses_provider_decisions_model_order() -> None: + pricing_key: Final = decisions_pricing_key("openai", "gpt-6-luna") + + assert pricing_key == "/".join(("openai", "decisions", "gpt-6-luna")) + + +def test_is_decisions_pricing_key_matches_only_the_provider_prefix() -> None: + decision_key: Final = decisions_pricing_key("openai", "gpt-6-luna") + plain_key: Final = "openai/gpt-6-luna" + other_provider_key: Final = decisions_pricing_key("perplexity", "gpt-6-luna") + + assert is_decisions_pricing_key(decision_key, "openai") + assert not is_decisions_pricing_key(plain_key, "openai") + assert not is_decisions_pricing_key(other_provider_key, "openai") + + +def test_decisions_pricing_model_uses_variant_or_falls_back_to_plain_model() -> None: + pricing_key: Final = decisions_pricing_key("openai", "gpt-6-luna") + variant_cost_map: Final[Mapping[str, object]] = {pricing_key: {}} + plain_cost_map: Final[Mapping[str, object]] = {"openai/gpt-6-luna": {}} + + assert decisions_pricing_model("openai", "gpt-6-luna", variant_cost_map) == pricing_key + assert decisions_pricing_model("openai", "gpt-6-luna", plain_cost_map) == "openai/gpt-6-luna" diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py index 07b52711dc3..2ad866b567c 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py @@ -6,6 +6,7 @@ import httpx import pytest import litellm +from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_key from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy.pass_through_endpoints.llm_provider_handlers.typesafe_passthrough_logging_handler import ( TypeSafePassthroughLoggingHandler, @@ -72,6 +73,40 @@ def test_uses_registry_pricing_and_standard_usage(): assert response["kwargs"]["combined_usage_object"].total_tokens == 360 +def test_uses_decisions_pricing_row_when_available(monkeypatch: pytest.MonkeyPatch) -> None: + model: Final = "nimble-latest" + plain_key: Final = f"bespoke/{model}" + decisions_key: Final = decisions_pricing_key("bespoke", model) + plain_pricing: Final = { + "input_cost_per_token": 1.0, + "output_cost_per_token": 1.0, + "litellm_provider": "bespoke", + "mode": "chat", + } + decisions_pricing: Final = { + "input_cost_per_token": 2.0, + "output_cost_per_token": 3.0, + "litellm_provider": "bespoke", + "mode": "evaluation", + } + monkeypatch.setitem(litellm.model_cost, plain_key, plain_pricing) + monkeypatch.setitem(litellm.model_cost, decisions_key, decisions_pricing) + result: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + httpx_response=_response(), + response_body={"model": model, "usage": {"input_tokens": 2, "output_tokens": 1}}, + logging_obj=_logging_obj(), + url_route="https://api.bespoke.example/v1/systemone", + result='{"answers": {}}', + start_time=datetime(2000, 1, 1), + end_time=datetime(2000, 1, 1), + cache_hit=False, + request_body={"model": model}, + custom_llm_provider="bespoke", + ) + + assert result["kwargs"]["response_cost"] == pytest.approx(7.0) + + def test_falls_back_to_request_model_when_response_model_is_missing(): result = _handler_result( {"usage": {"input_tokens": 10, "output_tokens": 2}}, @@ -142,68 +177,110 @@ def test_success_handler_dispatches_to_typesafe_handler(): @pytest.mark.asyncio @pytest.mark.parametrize("guardrail_cost", [0.0, 0.25]) @pytest.mark.parametrize("metadata_slot", ["metadata", "litellm_metadata"]) -@pytest.mark.parametrize("provider,requested,routing_model", [ - ("laya", "english", "multilingual"), ("laya", "english", None), - ("bespoke", "nimble-latest", None), - ("bespoke", "bespokelabs/Bespoke-Nimble-9B", None), -]) +@pytest.mark.parametrize( + "provider,requested,routing_model", + [ + ("laya", "english", "multilingual"), + ("laya", "english", None), + ("bespoke", "nimble-latest", None), + ("bespoke", "bespokelabs/Bespoke-Nimble-9B", None), + ], +) async def test_oss_gateway_accounts_for_checkpoint_usage_and_registered_cost( - monkeypatch: pytest.MonkeyPatch, routing_model: str | None, metadata_slot: str, guardrail_cost: float, - provider: str, requested: str + monkeypatch: pytest.MonkeyPatch, + routing_model: str | None, + metadata_slot: str, + guardrail_cost: float, + provider: str, + requested: str, ) -> None: checkpoint: Final = routing_model or requested model: Final = f"{provider}/{checkpoint}" input_rate: Final = 0.002 output_rate: Final = 0.005 - monkeypatch.setitem(litellm.model_cost, model, { - "input_cost_per_token": input_rate, "output_cost_per_token": output_rate, - "litellm_provider": provider, "mode": "evaluation", - }) + pricing_row: Final = { + "input_cost_per_token": input_rate, + "output_cost_per_token": output_rate, + "litellm_provider": provider, + "mode": "evaluation", + } + monkeypatch.setitem(litellm.model_cost, model, pricing_row) + monkeypatch.setitem(litellm.model_cost, decisions_pricing_key(provider, checkpoint), pricing_row) start: Final = datetime.now() logging_obj: Final = Logging( - model=requested, messages=[], stream=False, call_type="pass_through_endpoint", - start_time=start, litellm_call_id="oss-accounting", function_id="oss-accounting", kwargs={}, + model=requested, + messages=[], + stream=False, + call_type="pass_through_endpoint", + start_time=start, + litellm_call_id="oss-accounting", + function_id="oss-accounting", + kwargs={}, ) from fastapi import Request from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.pass_through_endpoints.pass_through_endpoints import HttpPassThroughEndpointHelpers - request: Final = Request({ - "type": "http", "method": "POST", "path": f"/{provider}/v1/systemone", - "headers": [], "query_string": b"", - }) + request: Final = Request( + { + "type": "http", + "method": "POST", + "path": f"/{provider}/v1/systemone", + "headers": [], + "query_string": b"", + } + ) auth: Final = UserAPIKeyAuth( - api_key="oss-budget-key", token="oss-budget-key", + api_key="oss-budget-key", + token="oss-budget-key", model_max_budget={f"{provider}/{requested}": {"budget_limit": 0.01, "time_period": "1d"}}, ) request_body: Final = {"model": requested, metadata_slot: {"model_group": "unbounded-client-choice"}} logging_kwargs: Final = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( - request=request, user_api_key_dict=auth, logging_obj=logging_obj, - passthrough_logging_payload={"url": f"https://{provider}.test/v1/systemone"}, _parsed_body=request_body, + request=request, + user_api_key_dict=auth, + logging_obj=logging_obj, + passthrough_logging_payload={"url": f"https://{provider}.test/v1/systemone"}, + _parsed_body=request_body, ) logging_kwargs["litellm_params"]["metadata"]["standard_logging_guardrail_information"] = [ {"guardrail_name": "trusted-hook", "guardrail_cost": guardrail_cost}, ] logging_obj.update_environment_variables( - model=requested, user="unknown", optional_params={}, - litellm_params=logging_kwargs["litellm_params"], call_type="pass_through_endpoint", + model=requested, + user="unknown", + optional_params={}, + litellm_params=logging_kwargs["litellm_params"], + call_type="pass_through_endpoint", ) body: Final = { - "model": "laya-rl-agent" if provider == "laya" else requested, "usage": {"input_tokens": 10, "output_tokens": 3}, + "model": "laya-rl-agent" if provider == "laya" else requested, + "usage": {"input_tokens": 10, "output_tokens": 3}, **({"routing": {"model": routing_model}} if routing_model else {}), } normalized: Final = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( - httpx_response=httpx.Response(200, request=httpx.Request("POST", f"https://{provider}.test/v1/systemone"), json=body), - response_body=body, request_body={"model": requested}, logging_obj=logging_obj, - url_route=f"https://{provider}.test/v1/systemone", result="{}", start_time=start, - end_time=datetime.now(), cache_hit=False, custom_llm_provider=provider, **logging_kwargs, + httpx_response=httpx.Response( + 200, request=httpx.Request("POST", f"https://{provider}.test/v1/systemone"), json=body + ), + response_body=body, + request_body={"model": requested}, + logging_obj=logging_obj, + url_route=f"https://{provider}.test/v1/systemone", + result="{}", + start_time=start, + end_time=datetime.now(), + cache_hit=False, + custom_llm_provider=provider, + **logging_kwargs, ) logged: Final = normalized["kwargs"] expected_cost: Final = 10 * input_rate + 3 * output_rate assert (logged["model"], logged["custom_llm_provider"]) == (model, provider) assert logged["response_cost"] == pytest.approx(expected_cost) assert logged["combined_usage_object"].model_dump(exclude_none=True) == { - "prompt_tokens": 10, "completion_tokens": 3, "total_tokens": 13, + "prompt_tokens": 10, + "completion_tokens": 3, + "total_tokens": 13, } assert logging_obj.model_call_details["model"] == model assert logging_obj.model_call_details["response_cost"] == pytest.approx(expected_cost) diff --git a/tests/unit/test_model_prices_schema.py b/tests/unit/test_model_prices_schema.py index 37df91d22af..b0ea1f12bf7 100644 --- a/tests/unit/test_model_prices_schema.py +++ b/tests/unit/test_model_prices_schema.py @@ -13,6 +13,7 @@ import pytest from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter import litellm +from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_key, is_decisions_pricing_key from litellm.litellm_core_utils.fallback_generalizations import match_capability_generalizations from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name from litellm.llms.openai_like.json_loader import JSONProviderRegistry @@ -25,6 +26,23 @@ BACKUP_PRICES_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_ba SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json" +def _evaluation_decisions_row_pair( + key: str, + value: object, + prices: Mapping[str, object], +) -> tuple[str, Mapping[str, object], Mapping[str, object]] | None: + if not isinstance(value, Mapping): + return None + provider: Final = value.get("litellm_provider") + if not isinstance(provider, str) or not is_decisions_pricing_key(key, provider): + return None + model: Final = key.removeprefix(decisions_pricing_key(provider, "")) + plain_row: Final = prices.get(f"{provider}/{model}") + if not isinstance(plain_row, Mapping) or plain_row.get("mode") != "evaluation": + return None + return key, value, plain_row + + def build_validator(schema: dict) -> jsonschema.Draft202012Validator: return jsonschema.Draft202012Validator(schema, format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER) @@ -430,6 +448,7 @@ def test_openai_reasoning_family_entries_carry_supports_reasoning(prices: dict): if isinstance(entry, dict) and entry.get("litellm_provider") == "openai" and is_openai_reasoning_family(name) + and not is_decisions_pricing_key(name, entry["litellm_provider"]) and entry.get("supports_reasoning") is not True ] assert unflagged == [], ( @@ -439,6 +458,18 @@ def test_openai_reasoning_family_entries_carry_supports_reasoning(prices: dict): ) +def test_decisions_pricing_rows_match_their_plain_evaluation_rows(prices: dict) -> None: + matching_rows: Final = tuple( + row_pair + for key, value in prices.items() + if (row_pair := _evaluation_decisions_row_pair(key, value, prices)) is not None + ) + + assert matching_rows + for key, decisions_row, evaluation_row in matching_rows: + assert decisions_row == evaluation_row, f"{key} must copy its plain evaluation row" + + _ABSENT: Final = object() REASONING_ANNOTATION_KEYS: Final = ( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 6b63f69fd2e..eddf037a36b 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -40,6 +40,7 @@ from litellm.constants import DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.humanloop import HumanloopLogger +from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_key from litellm.llms.base_llm.audio_transcription.transformation import BaseAudioTranscriptionConfig from litellm.integrations.langfuse.langfuse_prompt_management import LangfusePromptManagement from litellm.litellm_core_utils import litellm_logging @@ -2212,12 +2213,15 @@ def test_completion_registers_test_owned_model_prices() -> None: @pytest.mark.usefixtures("isolated_openai_model_sets") -def test_add_known_models_skips_region_scoped_bedrock_pricing_keys(monkeypatch: pytest.MonkeyPatch) -> None: +def test_add_known_models_skips_non_model_pricing_keys(monkeypatch: pytest.MonkeyPatch) -> None: price: Final = {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002, "mode": "chat"} + decisions_model: Final = "migration-openai-model" + decisions_pricing_model: Final = decisions_pricing_key("openai", decisions_model) model_cost_map: Final = { "migration.bedrock-model-v1:0": {**price, "litellm_provider": "bedrock"}, "bedrock/us-west-1/migration.bedrock-model-v1:0": {**price, "litellm_provider": "bedrock"}, - "migration-openai-model": {**price, "litellm_provider": "openai"}, + decisions_model: {**price, "litellm_provider": "openai"}, + decisions_pricing_model: {**price, "litellm_provider": "openai", "mode": "evaluation"}, } monkeypatch.setattr(litellm, "bedrock_models", set(litellm.bedrock_models)) monkeypatch.setattr(litellm, "models_by_provider", dict(litellm.models_by_provider)) @@ -2226,7 +2230,10 @@ def test_add_known_models_skips_region_scoped_bedrock_pricing_keys(monkeypatch: assert "migration.bedrock-model-v1:0" in litellm.bedrock_models assert "bedrock/us-west-1/migration.bedrock-model-v1:0" not in litellm.bedrock_models - assert "migration-openai-model" in litellm.models_by_provider["openai"] + assert decisions_model in litellm.open_ai_chat_completion_models + assert decisions_pricing_model not in litellm.open_ai_chat_completion_models + assert decisions_model in litellm.models_by_provider["openai"] + assert decisions_pricing_model not in litellm.models_by_provider["openai"] def test_utils_defines_tests_without_invoking_them_at_module_scope() -> None: