mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-16 04:29:31 +00:00
feat(decisions): bill decisions and systemone from <provider>/decisions/<model> cost map rows
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kerry
parent
f6bb87739d
commit
3183a53922
16 changed files with 965 additions and 107 deletions
|
|
@ -49,6 +49,7 @@ from collections.abc import Mapping
|
|||
from litellm.types.integrations.datadog import DatadogInitParams
|
||||
from litellm.types.integrations.newrelic import NewRelicInitParams
|
||||
from litellm.litellm_core_utils.core_helpers import drop_params_env_flag
|
||||
from litellm.litellm_core_utils.decisions_pricing import is_decisions_pricing_key
|
||||
from litellm.types.integrations.pointfive import PointFiveInitParams
|
||||
from litellm.types.integrations.zerobus import ZerobusInitParams
|
||||
from litellm._logging import (
|
||||
|
|
@ -759,6 +760,8 @@ def is_openai_finetune_model(key: str) -> bool:
|
|||
|
||||
def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None:
|
||||
for key, value in model_cost_map.items():
|
||||
if isinstance((provider := value.get("litellm_provider")), str) and is_decisions_pricing_key(key, provider):
|
||||
continue
|
||||
if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key):
|
||||
open_ai_chat_completion_models.add(key)
|
||||
elif value.get("litellm_provider") == "text-completion-openai":
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from typing_extensions import assert_never
|
|||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params
|
||||
from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_model
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.decisions.transformation import (
|
||||
|
|
@ -263,7 +264,11 @@ def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> Deci
|
|||
{
|
||||
"model": f"{call.custom_llm_provider}/{call.model}",
|
||||
"custom_llm_provider": call.custom_llm_provider,
|
||||
"provider_response_model": f"{call.custom_llm_provider}/{call.model}",
|
||||
"provider_response_model": decisions_pricing_model(
|
||||
call.custom_llm_provider,
|
||||
call.model,
|
||||
litellm.model_cost, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded
|
||||
),
|
||||
**({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}),
|
||||
}
|
||||
)
|
||||
|
|
|
|||
19
litellm/litellm_core_utils/decisions_pricing.py
Normal file
19
litellm/litellm_core_utils/decisions_pricing.py
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
|
||||
def decisions_pricing_key(custom_llm_provider: str, model: str) -> str:
|
||||
return f"{custom_llm_provider}/decisions/{model}"
|
||||
|
||||
|
||||
def is_decisions_pricing_key(key: str, custom_llm_provider: str) -> bool:
|
||||
return key.startswith(decisions_pricing_key(custom_llm_provider, ""))
|
||||
|
||||
|
||||
def decisions_pricing_model(
|
||||
custom_llm_provider: str,
|
||||
model: str,
|
||||
model_cost: Mapping[str, object],
|
||||
) -> str:
|
||||
decisions_model: Final = decisions_pricing_key(custom_llm_provider, model)
|
||||
return decisions_model if decisions_model in model_cost else f"{custom_llm_provider}/{model}"
|
||||
|
|
@ -6,6 +6,7 @@ import httpx
|
|||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_model
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
get_standard_logging_object_payload, # pyright: ignore[reportUnknownVariableType] # legacy helper has an untyped signature
|
||||
|
|
@ -85,7 +86,13 @@ class TypeSafePassthroughLoggingHandler:
|
|||
input_tokens: Final = usage.input_tokens
|
||||
output_tokens: Final = usage.output_tokens
|
||||
candidate_model_keys: Final = tuple(
|
||||
f"{custom_llm_provider}/{model}" for model in (response_model, request_model) if model is not None
|
||||
decisions_pricing_model(
|
||||
custom_llm_provider,
|
||||
model,
|
||||
litellm.model_cost, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded
|
||||
)
|
||||
for model in (response_model, request_model)
|
||||
if model is not None
|
||||
)
|
||||
pricing: Final = _pricing_for(candidate_model_keys)
|
||||
response_cost: Final = (
|
||||
|
|
|
|||
|
|
@ -8538,7 +8538,11 @@
|
|||
"output_cost_per_token_above_272k_tokens": 1.5e-05,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 3e-05,
|
||||
"output_cost_per_token_priority": 2e-05,
|
||||
"search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01},
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"source": "https://azure.microsoft.com/en-us/blog/gpt-6-astra-sol-and-luna-for-production-agents-in-microsoft-foundry/",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -8591,7 +8595,11 @@
|
|||
"output_cost_per_token_above_272k_tokens": 1.5e-05,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 3e-05,
|
||||
"output_cost_per_token_priority": 2e-05,
|
||||
"search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01},
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"source": "https://azure.microsoft.com/en-us/blog/gpt-6-astra-sol-and-luna-for-production-agents-in-microsoft-foundry/",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -9283,7 +9291,11 @@
|
|||
"output_cost_per_token_above_272k_tokens": 1.65e-05,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 3.3e-05,
|
||||
"output_cost_per_token_priority": 2.2e-05,
|
||||
"search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01},
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"source": "https://azure.microsoft.com/en-us/blog/gpt-6-astra-sol-and-luna-for-production-agents-in-microsoft-foundry/",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -15879,7 +15891,10 @@
|
|||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"provider_specific_entry": {"us":1.1,"fast":2},
|
||||
"provider_specific_entry": {
|
||||
"us": 1.1,
|
||||
"fast": 2
|
||||
},
|
||||
"supports_output_config": true,
|
||||
"supports_speed": true,
|
||||
"supports_fast_mode": true,
|
||||
|
|
@ -16006,6 +16021,14 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
},
|
||||
"cloudflare/decisions/@cf/cloudflare/clef": {
|
||||
"input_cost_per_token": 2.4e-07,
|
||||
"litellm_provider": "cloudflare",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
},
|
||||
"cloudflare/@cf/cloudflare/clef-flash": {
|
||||
"input_cost_per_token": 9e-08,
|
||||
"litellm_provider": "cloudflare",
|
||||
|
|
@ -16014,6 +16037,14 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
},
|
||||
"cloudflare/decisions/@cf/cloudflare/clef-flash": {
|
||||
"input_cost_per_token": 9e-08,
|
||||
"litellm_provider": "cloudflare",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
},
|
||||
"cloudflare/@cf/meta/llama-2-7b-chat-fp16": {
|
||||
"input_cost_per_token": 1.923e-06,
|
||||
"litellm_provider": "cloudflare",
|
||||
|
|
@ -21177,6 +21208,18 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models"
|
||||
},
|
||||
"databricks/decisions/databricks-openjev-qwen35-4b": {
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
"cache_read_input_token_cost": 0.0,
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "databricks",
|
||||
"metadata": {
|
||||
"notes": "Pay-per-token Foundation Model API decision endpoint (OpenJev, Qwen 3.5 4B) queried through the System One API. Databricks published no DBU rate for it as of 2026-10-07, so a deployment's model_info prices set the billed cost."
|
||||
},
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models"
|
||||
},
|
||||
"dataforseo/search": {
|
||||
"input_cost_per_query": 0.003,
|
||||
"litellm_provider": "dataforseo",
|
||||
|
|
@ -29603,7 +29646,11 @@
|
|||
"output_cost_per_token": 3e-06,
|
||||
"output_cost_per_token_batches": 1.5e-06,
|
||||
"rpm": 1000.0,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_low": 0.014,
|
||||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"source": "https://ai.google.dev/gemini-api/docs/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -29680,7 +29727,11 @@
|
|||
"output_cost_per_token": 1.2e-05,
|
||||
"output_cost_per_token_batches": 6e-06,
|
||||
"rpm": 1000.0,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.035,"search_context_size_medium":0.035,"search_context_size_high":0.035},
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_low": 0.035,
|
||||
"search_context_size_medium": 0.035,
|
||||
"search_context_size_high": 0.035
|
||||
},
|
||||
"source": "https://ai.google.dev/gemini-api/docs/models/deep-research-pro-preview-12-2025",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -34963,6 +35014,17 @@
|
|||
"supports_web_search": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
},
|
||||
"openai/decisions/gpt-6-luna": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
"input_cost_per_token_above_272k_tokens": 2e-07,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 922000,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
"regional_processing_uplift_multiplier_us": 1.1,
|
||||
"source": "https://developers.openai.com/api/docs/guides/decisions#pricing-and-availability"
|
||||
},
|
||||
"gpt-5.6": {
|
||||
"cache_creation_input_token_cost": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
|
||||
|
|
@ -38761,7 +38823,7 @@
|
|||
"supports_xhigh_reasoning_effort": true
|
||||
},
|
||||
"meta/muse-voice-transcribe-1.0": {
|
||||
"input_cost_per_second": 0.00005,
|
||||
"input_cost_per_second": 5e-05,
|
||||
"litellm_provider": "meta",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://dev.meta.ai/docs/speech-to-text",
|
||||
|
|
@ -42944,6 +43006,14 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.perplexity.ai/api-reference/decisions-post"
|
||||
},
|
||||
"perplexity/decisions/pplx-decider-v1-27b": {
|
||||
"input_cost_per_token": 4e-08,
|
||||
"litellm_provider": "perplexity",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.perplexity.ai/api-reference/decisions-post"
|
||||
},
|
||||
"perplexity/sonar": {
|
||||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "perplexity",
|
||||
|
|
@ -48929,7 +48999,6 @@
|
|||
"/v1/images/edits",
|
||||
"/v1/batch"
|
||||
]
|
||||
|
||||
},
|
||||
"vertex_ai/gemini-3-pro-image-preview": {
|
||||
"input_cost_per_image": 0.0011,
|
||||
|
|
@ -48963,7 +49032,6 @@
|
|||
"/v1/images/edits",
|
||||
"/v1/batch"
|
||||
]
|
||||
|
||||
},
|
||||
"vertex_ai/gemini-3.1-flash-image": {
|
||||
"cache_read_input_token_cost": 5e-08,
|
||||
|
|
@ -48996,7 +49064,6 @@
|
|||
"/v1/images/edits",
|
||||
"/v1/batch"
|
||||
]
|
||||
|
||||
},
|
||||
"vertex_ai/gemini-3.1-flash-image-preview": {
|
||||
"input_cost_per_image": 0.00056,
|
||||
|
|
@ -49021,7 +49088,6 @@
|
|||
"/v1/images/edits",
|
||||
"/v1/batch"
|
||||
]
|
||||
|
||||
},
|
||||
"vertex_ai/gemini-3.1-flash-lite-image": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
|
|
@ -59585,7 +59651,13 @@
|
|||
{
|
||||
"name": "claude-adaptive-thinking",
|
||||
"pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",
|
||||
"fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"],
|
||||
"fill_missing_for_providers": [
|
||||
"anthropic",
|
||||
"azure_ai",
|
||||
"bedrock",
|
||||
"bedrock_converse",
|
||||
"vertex_ai-anthropic_models"
|
||||
],
|
||||
"description": "Claude at version 4.6 or higher, in any id shape that contains claude-<family>-: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Turns on adaptive thinking for new versions and new families with no code change.",
|
||||
"model_info": {
|
||||
"supports_adaptive_thinking": true
|
||||
|
|
@ -59594,7 +59666,13 @@
|
|||
{
|
||||
"name": "claude-legacy-thinking",
|
||||
"pattern": "claude-[a-z]+-4[-._]6(?!\\d)",
|
||||
"fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"],
|
||||
"fill_missing_for_providers": [
|
||||
"anthropic",
|
||||
"azure_ai",
|
||||
"bedrock",
|
||||
"bedrock_converse",
|
||||
"vertex_ai-anthropic_models"
|
||||
],
|
||||
"description": "Claude at version 4.6 exactly, in any id shape that contains claude-<family>-4-6 (dotted and underscored minors included, dated releases such as claude-sonnet-4-6-20260219 too). The 4.6 family is adaptive-thinking yet still accepts legacy thinking.type=enabled with budget_tokens, so the caller's hard budget cap is forwarded verbatim instead of being rewritten to an uncapped output_config.effort. The lookahead keeps two-digit minors such as 4-60 from matching. 4.7+ and 5+ majors reject the legacy shape and stay on the adaptive translation.",
|
||||
"model_info": {
|
||||
"supports_legacy_thinking": true
|
||||
|
|
@ -59611,7 +59689,13 @@
|
|||
{
|
||||
"name": "claude-mid-conversation-system",
|
||||
"pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",
|
||||
"fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"],
|
||||
"fill_missing_for_providers": [
|
||||
"anthropic",
|
||||
"azure_ai",
|
||||
"bedrock",
|
||||
"bedrock_converse",
|
||||
"vertex_ai-anthropic_models"
|
||||
],
|
||||
"description": "Claude at version 4.8 or higher, in any id shape that contains claude-<family>-: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.",
|
||||
"model_info": {
|
||||
"supports_mid_conversation_system": true
|
||||
|
|
@ -59620,7 +59704,12 @@
|
|||
{
|
||||
"name": "claude-tool-search",
|
||||
"pattern": "claude-[a-z]+-(?:4[-._](?:[5-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",
|
||||
"fill_missing_for_providers": ["anthropic", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"],
|
||||
"fill_missing_for_providers": [
|
||||
"anthropic",
|
||||
"bedrock",
|
||||
"bedrock_converse",
|
||||
"vertex_ai-anthropic_models"
|
||||
],
|
||||
"description": "Claude at version 4.5 or higher, in any id shape that contains claude-<family>-: minors 4.5 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Anthropic's tool search docs list every Claude 4.5 and newer model as supported and Opus 4.1 and earlier as unsupported, so the flag follows the version instead of a per-model list. azure_ai is left out on purpose: Anthropic documents tool search as unavailable on Azure-hosted Foundry deployments, and the azure_ai/ key cannot tell those from Anthropic-hosted ones.",
|
||||
"model_info": {
|
||||
"supports_tool_search": true
|
||||
|
|
@ -59637,7 +59726,11 @@
|
|||
{
|
||||
"name": "openai-reasoning-family-baseline",
|
||||
"pattern": "^(?!.*search-api)(?:[a-z0-9_.-]+/)*(?:ft:)?(?:o[1-9]\\d*(?![a-z0-9])|gpt-[5-9](?:\\.\\d+)?(?![0-9.])|(?:gpt-\\d+(?:\\.\\d+)?(?:-[a-z0-9]+)*-)?(?:codex|deep-research|chat-latest)(?![a-z0-9]))",
|
||||
"fill_missing_for_providers": ["azure", "azure_ai", "openai"],
|
||||
"fill_missing_for_providers": [
|
||||
"azure",
|
||||
"azure_ai",
|
||||
"openai"
|
||||
],
|
||||
"description": "OpenAI reasoning families by id shape, under any provider namespace and with an optional ft: prefix: the o-series (o1, o3-pro, o4-mini), gpt-5 through gpt-9 majors including dotted minors and suffixed variants (gpt-5.5-cyber, gpt-6-astra), and the codex, deep-research and chat-latest lines when standalone or on a gpt base. gpt-5-search-api is excluded because it is a search-only surface. Every model here is a reasoning model, and the Responses API drops the caller's reasoning param for any mapped OpenAI model whose info lacks supports_reasoning, so an id the registry has not named yet keeps its reasoning settings instead of silently losing them. Rules lose to exact entries. Carries no mode and no pricing, so cost stays on the standard unpriced behavior.",
|
||||
"model_info": {
|
||||
"supports_reasoning": true
|
||||
|
|
@ -68033,6 +68126,20 @@
|
|||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"bespoke/decisions/nimble-latest": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "bespoke",
|
||||
"max_input_tokens": 8192,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://github.com/bespokelabsai/nimble",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
],
|
||||
"metadata": {
|
||||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"bespoke/nimble": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "bespoke",
|
||||
|
|
@ -68047,6 +68154,20 @@
|
|||
"notes": "Self-hosted decision model under the name Ollama serves it as; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"bespoke/decisions/nimble": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "bespoke",
|
||||
"max_input_tokens": 8192,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://ollama.com/library/nimble",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
],
|
||||
"metadata": {
|
||||
"notes": "Self-hosted decision model under the name Ollama serves it as; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"bespoke/bespokelabs/Bespoke-Nimble-9B": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "bespoke",
|
||||
|
|
@ -68061,6 +68182,20 @@
|
|||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"bespoke/decisions/bespokelabs/Bespoke-Nimble-9B": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "bespoke",
|
||||
"max_input_tokens": 8192,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://github.com/bespokelabsai/nimble",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
],
|
||||
"metadata": {
|
||||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"laya/english": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "laya",
|
||||
|
|
@ -68074,6 +68209,19 @@
|
|||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"laya/decisions/english": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "laya",
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://github.com/NandhaKishorM/laya",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
],
|
||||
"metadata": {
|
||||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"laya/multilingual": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "laya",
|
||||
|
|
@ -68087,6 +68235,19 @@
|
|||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"laya/decisions/multilingual": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "laya",
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://github.com/NandhaKishorM/laya",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
],
|
||||
"metadata": {
|
||||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"laya/typed-decisions": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "laya",
|
||||
|
|
@ -68100,6 +68261,19 @@
|
|||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"laya/decisions/typed-decisions": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "laya",
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://github.com/NandhaKishorM/laya",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
],
|
||||
"metadata": {
|
||||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"strands_decider/strands-decider-2B-hobson-v19": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "strands_decider",
|
||||
|
|
@ -68110,6 +68284,16 @@
|
|||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"strands_decider/decisions/strands-decider-2B-hobson-v19": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "strands_decider",
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://huggingface.co/StrandsAgents/strands-decider-2B-hobson-v19",
|
||||
"metadata": {
|
||||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"typesafe/jev-1.13.0": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
"litellm_provider": "typesafe",
|
||||
|
|
@ -68117,6 +68301,13 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.typesafe.ai/models"
|
||||
},
|
||||
"typesafe/decisions/jev-1.13.0": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
"litellm_provider": "typesafe",
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.typesafe.ai/models"
|
||||
},
|
||||
"typesafe/jev-latest": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
"litellm_provider": "typesafe",
|
||||
|
|
@ -68124,6 +68315,13 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.typesafe.ai/models"
|
||||
},
|
||||
"typesafe/decisions/jev-latest": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
"litellm_provider": "typesafe",
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.typesafe.ai/models"
|
||||
},
|
||||
"typesafe/jev-preview": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
"litellm_provider": "typesafe",
|
||||
|
|
@ -68131,6 +68329,13 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.typesafe.ai/models"
|
||||
},
|
||||
"typesafe/decisions/jev-preview": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
"litellm_provider": "typesafe",
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.typesafe.ai/models"
|
||||
},
|
||||
"wandb/deepseek-ai/DeepSeek-V4.1-Flash": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"input_cost_per_token": 2e-07,
|
||||
|
|
@ -69110,7 +69315,9 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 4.4e-06,
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supported_regions": ["global"],
|
||||
"supported_regions": [
|
||||
"global"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
|
|
@ -70374,7 +70581,9 @@
|
|||
},
|
||||
"global.xai.grok-4.7": {
|
||||
"supports_regex_lookaround": false,
|
||||
"supported_endpoints": ["/v1/chat/completions"],
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions"
|
||||
],
|
||||
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -70400,7 +70609,9 @@
|
|||
},
|
||||
"us.xai.grok-4.7": {
|
||||
"supports_regex_lookaround": false,
|
||||
"supported_endpoints": ["/v1/chat/completions"],
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions"
|
||||
],
|
||||
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"cache_read_input_token_cost": 5.5e-07,
|
||||
|
|
@ -70848,7 +71059,11 @@
|
|||
"output_cost_per_token": 7.5e-06,
|
||||
"output_cost_per_token_batches": 3.75e-06,
|
||||
"rpm": 1000.0,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_low": 0.014,
|
||||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"source": "https://ai.google.dev/gemini-api/docs/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -70893,7 +71108,11 @@
|
|||
"output_cost_per_token": 7.5e-06,
|
||||
"output_cost_per_token_batches": 3.75e-06,
|
||||
"output_cost_per_token_priority": 1.35e-05,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_low": 0.014,
|
||||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -70939,7 +71158,11 @@
|
|||
"output_cost_per_token": 7.5e-06,
|
||||
"output_cost_per_token_batches": 3.75e-06,
|
||||
"output_cost_per_token_priority": 1.35e-05,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_low": 0.014,
|
||||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -71019,7 +71242,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"claude-haiku-5-5": {
|
||||
"claude-haiku-5-5": {
|
||||
"input_cost_per_token_above_100k_tokens_batches": 2.5e-07,
|
||||
"output_cost_per_token_above_100k_tokens_batches": 1.25e-06,
|
||||
"cache_creation_input_token_cost_above_100k_tokens_batches": 3.125e-07,
|
||||
|
|
@ -71787,6 +72010,13 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'"
|
||||
},
|
||||
"azure_ai/decisions/Microsoft-Decision-1": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'"
|
||||
},
|
||||
"gpt-rosalind-discovery": {
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"input_cost_per_token": 5e-06,
|
||||
|
|
|
|||
|
|
@ -8538,7 +8538,11 @@
|
|||
"output_cost_per_token_above_272k_tokens": 1.5e-05,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 3e-05,
|
||||
"output_cost_per_token_priority": 2e-05,
|
||||
"search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01},
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"source": "https://azure.microsoft.com/en-us/blog/gpt-6-astra-sol-and-luna-for-production-agents-in-microsoft-foundry/",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -8591,7 +8595,11 @@
|
|||
"output_cost_per_token_above_272k_tokens": 1.5e-05,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 3e-05,
|
||||
"output_cost_per_token_priority": 2e-05,
|
||||
"search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01},
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"source": "https://azure.microsoft.com/en-us/blog/gpt-6-astra-sol-and-luna-for-production-agents-in-microsoft-foundry/",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -9283,7 +9291,11 @@
|
|||
"output_cost_per_token_above_272k_tokens": 1.65e-05,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 3.3e-05,
|
||||
"output_cost_per_token_priority": 2.2e-05,
|
||||
"search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01},
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"source": "https://azure.microsoft.com/en-us/blog/gpt-6-astra-sol-and-luna-for-production-agents-in-microsoft-foundry/",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -15879,7 +15891,10 @@
|
|||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"provider_specific_entry": {"us":1.1,"fast":2},
|
||||
"provider_specific_entry": {
|
||||
"us": 1.1,
|
||||
"fast": 2
|
||||
},
|
||||
"supports_output_config": true,
|
||||
"supports_speed": true,
|
||||
"supports_fast_mode": true,
|
||||
|
|
@ -16006,6 +16021,14 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
},
|
||||
"cloudflare/decisions/@cf/cloudflare/clef": {
|
||||
"input_cost_per_token": 2.4e-07,
|
||||
"litellm_provider": "cloudflare",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
},
|
||||
"cloudflare/@cf/cloudflare/clef-flash": {
|
||||
"input_cost_per_token": 9e-08,
|
||||
"litellm_provider": "cloudflare",
|
||||
|
|
@ -16014,6 +16037,14 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
},
|
||||
"cloudflare/decisions/@cf/cloudflare/clef-flash": {
|
||||
"input_cost_per_token": 9e-08,
|
||||
"litellm_provider": "cloudflare",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
},
|
||||
"cloudflare/@cf/meta/llama-2-7b-chat-fp16": {
|
||||
"input_cost_per_token": 1.923e-06,
|
||||
"litellm_provider": "cloudflare",
|
||||
|
|
@ -21177,6 +21208,18 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models"
|
||||
},
|
||||
"databricks/decisions/databricks-openjev-qwen35-4b": {
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
"cache_read_input_token_cost": 0.0,
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "databricks",
|
||||
"metadata": {
|
||||
"notes": "Pay-per-token Foundation Model API decision endpoint (OpenJev, Qwen 3.5 4B) queried through the System One API. Databricks published no DBU rate for it as of 2026-10-07, so a deployment's model_info prices set the billed cost."
|
||||
},
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models"
|
||||
},
|
||||
"dataforseo/search": {
|
||||
"input_cost_per_query": 0.003,
|
||||
"litellm_provider": "dataforseo",
|
||||
|
|
@ -29603,7 +29646,11 @@
|
|||
"output_cost_per_token": 3e-06,
|
||||
"output_cost_per_token_batches": 1.5e-06,
|
||||
"rpm": 1000.0,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_low": 0.014,
|
||||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"source": "https://ai.google.dev/gemini-api/docs/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -29680,7 +29727,11 @@
|
|||
"output_cost_per_token": 1.2e-05,
|
||||
"output_cost_per_token_batches": 6e-06,
|
||||
"rpm": 1000.0,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.035,"search_context_size_medium":0.035,"search_context_size_high":0.035},
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_low": 0.035,
|
||||
"search_context_size_medium": 0.035,
|
||||
"search_context_size_high": 0.035
|
||||
},
|
||||
"source": "https://ai.google.dev/gemini-api/docs/models/deep-research-pro-preview-12-2025",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -34963,6 +35014,17 @@
|
|||
"supports_web_search": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
},
|
||||
"openai/decisions/gpt-6-luna": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
"input_cost_per_token_above_272k_tokens": 2e-07,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 922000,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
"regional_processing_uplift_multiplier_us": 1.1,
|
||||
"source": "https://developers.openai.com/api/docs/guides/decisions#pricing-and-availability"
|
||||
},
|
||||
"gpt-5.6": {
|
||||
"cache_creation_input_token_cost": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
|
||||
|
|
@ -38761,7 +38823,7 @@
|
|||
"supports_xhigh_reasoning_effort": true
|
||||
},
|
||||
"meta/muse-voice-transcribe-1.0": {
|
||||
"input_cost_per_second": 0.00005,
|
||||
"input_cost_per_second": 5e-05,
|
||||
"litellm_provider": "meta",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://dev.meta.ai/docs/speech-to-text",
|
||||
|
|
@ -42944,6 +43006,14 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.perplexity.ai/api-reference/decisions-post"
|
||||
},
|
||||
"perplexity/decisions/pplx-decider-v1-27b": {
|
||||
"input_cost_per_token": 4e-08,
|
||||
"litellm_provider": "perplexity",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.perplexity.ai/api-reference/decisions-post"
|
||||
},
|
||||
"perplexity/sonar": {
|
||||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "perplexity",
|
||||
|
|
@ -48929,7 +48999,6 @@
|
|||
"/v1/images/edits",
|
||||
"/v1/batch"
|
||||
]
|
||||
|
||||
},
|
||||
"vertex_ai/gemini-3-pro-image-preview": {
|
||||
"input_cost_per_image": 0.0011,
|
||||
|
|
@ -48963,7 +49032,6 @@
|
|||
"/v1/images/edits",
|
||||
"/v1/batch"
|
||||
]
|
||||
|
||||
},
|
||||
"vertex_ai/gemini-3.1-flash-image": {
|
||||
"cache_read_input_token_cost": 5e-08,
|
||||
|
|
@ -48996,7 +49064,6 @@
|
|||
"/v1/images/edits",
|
||||
"/v1/batch"
|
||||
]
|
||||
|
||||
},
|
||||
"vertex_ai/gemini-3.1-flash-image-preview": {
|
||||
"input_cost_per_image": 0.00056,
|
||||
|
|
@ -49021,7 +49088,6 @@
|
|||
"/v1/images/edits",
|
||||
"/v1/batch"
|
||||
]
|
||||
|
||||
},
|
||||
"vertex_ai/gemini-3.1-flash-lite-image": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
|
|
@ -59585,7 +59651,13 @@
|
|||
{
|
||||
"name": "claude-adaptive-thinking",
|
||||
"pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",
|
||||
"fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"],
|
||||
"fill_missing_for_providers": [
|
||||
"anthropic",
|
||||
"azure_ai",
|
||||
"bedrock",
|
||||
"bedrock_converse",
|
||||
"vertex_ai-anthropic_models"
|
||||
],
|
||||
"description": "Claude at version 4.6 or higher, in any id shape that contains claude-<family>-: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Turns on adaptive thinking for new versions and new families with no code change.",
|
||||
"model_info": {
|
||||
"supports_adaptive_thinking": true
|
||||
|
|
@ -59594,7 +59666,13 @@
|
|||
{
|
||||
"name": "claude-legacy-thinking",
|
||||
"pattern": "claude-[a-z]+-4[-._]6(?!\\d)",
|
||||
"fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"],
|
||||
"fill_missing_for_providers": [
|
||||
"anthropic",
|
||||
"azure_ai",
|
||||
"bedrock",
|
||||
"bedrock_converse",
|
||||
"vertex_ai-anthropic_models"
|
||||
],
|
||||
"description": "Claude at version 4.6 exactly, in any id shape that contains claude-<family>-4-6 (dotted and underscored minors included, dated releases such as claude-sonnet-4-6-20260219 too). The 4.6 family is adaptive-thinking yet still accepts legacy thinking.type=enabled with budget_tokens, so the caller's hard budget cap is forwarded verbatim instead of being rewritten to an uncapped output_config.effort. The lookahead keeps two-digit minors such as 4-60 from matching. 4.7+ and 5+ majors reject the legacy shape and stay on the adaptive translation.",
|
||||
"model_info": {
|
||||
"supports_legacy_thinking": true
|
||||
|
|
@ -59611,7 +59689,13 @@
|
|||
{
|
||||
"name": "claude-mid-conversation-system",
|
||||
"pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",
|
||||
"fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"],
|
||||
"fill_missing_for_providers": [
|
||||
"anthropic",
|
||||
"azure_ai",
|
||||
"bedrock",
|
||||
"bedrock_converse",
|
||||
"vertex_ai-anthropic_models"
|
||||
],
|
||||
"description": "Claude at version 4.8 or higher, in any id shape that contains claude-<family>-: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.",
|
||||
"model_info": {
|
||||
"supports_mid_conversation_system": true
|
||||
|
|
@ -59620,7 +59704,12 @@
|
|||
{
|
||||
"name": "claude-tool-search",
|
||||
"pattern": "claude-[a-z]+-(?:4[-._](?:[5-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",
|
||||
"fill_missing_for_providers": ["anthropic", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"],
|
||||
"fill_missing_for_providers": [
|
||||
"anthropic",
|
||||
"bedrock",
|
||||
"bedrock_converse",
|
||||
"vertex_ai-anthropic_models"
|
||||
],
|
||||
"description": "Claude at version 4.5 or higher, in any id shape that contains claude-<family>-: minors 4.5 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Anthropic's tool search docs list every Claude 4.5 and newer model as supported and Opus 4.1 and earlier as unsupported, so the flag follows the version instead of a per-model list. azure_ai is left out on purpose: Anthropic documents tool search as unavailable on Azure-hosted Foundry deployments, and the azure_ai/ key cannot tell those from Anthropic-hosted ones.",
|
||||
"model_info": {
|
||||
"supports_tool_search": true
|
||||
|
|
@ -59637,7 +59726,11 @@
|
|||
{
|
||||
"name": "openai-reasoning-family-baseline",
|
||||
"pattern": "^(?!.*search-api)(?:[a-z0-9_.-]+/)*(?:ft:)?(?:o[1-9]\\d*(?![a-z0-9])|gpt-[5-9](?:\\.\\d+)?(?![0-9.])|(?:gpt-\\d+(?:\\.\\d+)?(?:-[a-z0-9]+)*-)?(?:codex|deep-research|chat-latest)(?![a-z0-9]))",
|
||||
"fill_missing_for_providers": ["azure", "azure_ai", "openai"],
|
||||
"fill_missing_for_providers": [
|
||||
"azure",
|
||||
"azure_ai",
|
||||
"openai"
|
||||
],
|
||||
"description": "OpenAI reasoning families by id shape, under any provider namespace and with an optional ft: prefix: the o-series (o1, o3-pro, o4-mini), gpt-5 through gpt-9 majors including dotted minors and suffixed variants (gpt-5.5-cyber, gpt-6-astra), and the codex, deep-research and chat-latest lines when standalone or on a gpt base. gpt-5-search-api is excluded because it is a search-only surface. Every model here is a reasoning model, and the Responses API drops the caller's reasoning param for any mapped OpenAI model whose info lacks supports_reasoning, so an id the registry has not named yet keeps its reasoning settings instead of silently losing them. Rules lose to exact entries. Carries no mode and no pricing, so cost stays on the standard unpriced behavior.",
|
||||
"model_info": {
|
||||
"supports_reasoning": true
|
||||
|
|
@ -68033,6 +68126,20 @@
|
|||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"bespoke/decisions/nimble-latest": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "bespoke",
|
||||
"max_input_tokens": 8192,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://github.com/bespokelabsai/nimble",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
],
|
||||
"metadata": {
|
||||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"bespoke/nimble": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "bespoke",
|
||||
|
|
@ -68047,6 +68154,20 @@
|
|||
"notes": "Self-hosted decision model under the name Ollama serves it as; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"bespoke/decisions/nimble": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "bespoke",
|
||||
"max_input_tokens": 8192,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://ollama.com/library/nimble",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
],
|
||||
"metadata": {
|
||||
"notes": "Self-hosted decision model under the name Ollama serves it as; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"bespoke/bespokelabs/Bespoke-Nimble-9B": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "bespoke",
|
||||
|
|
@ -68061,6 +68182,20 @@
|
|||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"bespoke/decisions/bespokelabs/Bespoke-Nimble-9B": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "bespoke",
|
||||
"max_input_tokens": 8192,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://github.com/bespokelabsai/nimble",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
],
|
||||
"metadata": {
|
||||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"laya/english": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "laya",
|
||||
|
|
@ -68074,6 +68209,19 @@
|
|||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"laya/decisions/english": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "laya",
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://github.com/NandhaKishorM/laya",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
],
|
||||
"metadata": {
|
||||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"laya/multilingual": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "laya",
|
||||
|
|
@ -68087,6 +68235,19 @@
|
|||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"laya/decisions/multilingual": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "laya",
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://github.com/NandhaKishorM/laya",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
],
|
||||
"metadata": {
|
||||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"laya/typed-decisions": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "laya",
|
||||
|
|
@ -68100,6 +68261,19 @@
|
|||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"laya/decisions/typed-decisions": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "laya",
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://github.com/NandhaKishorM/laya",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
],
|
||||
"metadata": {
|
||||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"strands_decider/strands-decider-2B-hobson-v19": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "strands_decider",
|
||||
|
|
@ -68110,6 +68284,16 @@
|
|||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"strands_decider/decisions/strands-decider-2B-hobson-v19": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "strands_decider",
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://huggingface.co/StrandsAgents/strands-decider-2B-hobson-v19",
|
||||
"metadata": {
|
||||
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
|
||||
}
|
||||
},
|
||||
"typesafe/jev-1.13.0": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
"litellm_provider": "typesafe",
|
||||
|
|
@ -68117,6 +68301,13 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.typesafe.ai/models"
|
||||
},
|
||||
"typesafe/decisions/jev-1.13.0": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
"litellm_provider": "typesafe",
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.typesafe.ai/models"
|
||||
},
|
||||
"typesafe/jev-latest": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
"litellm_provider": "typesafe",
|
||||
|
|
@ -68124,6 +68315,13 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.typesafe.ai/models"
|
||||
},
|
||||
"typesafe/decisions/jev-latest": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
"litellm_provider": "typesafe",
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.typesafe.ai/models"
|
||||
},
|
||||
"typesafe/jev-preview": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
"litellm_provider": "typesafe",
|
||||
|
|
@ -68131,6 +68329,13 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.typesafe.ai/models"
|
||||
},
|
||||
"typesafe/decisions/jev-preview": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
"litellm_provider": "typesafe",
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.typesafe.ai/models"
|
||||
},
|
||||
"wandb/deepseek-ai/DeepSeek-V4.1-Flash": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"input_cost_per_token": 2e-07,
|
||||
|
|
@ -69110,7 +69315,9 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 4.4e-06,
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supported_regions": ["global"],
|
||||
"supported_regions": [
|
||||
"global"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
|
|
@ -70374,7 +70581,9 @@
|
|||
},
|
||||
"global.xai.grok-4.7": {
|
||||
"supports_regex_lookaround": false,
|
||||
"supported_endpoints": ["/v1/chat/completions"],
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions"
|
||||
],
|
||||
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -70400,7 +70609,9 @@
|
|||
},
|
||||
"us.xai.grok-4.7": {
|
||||
"supports_regex_lookaround": false,
|
||||
"supported_endpoints": ["/v1/chat/completions"],
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions"
|
||||
],
|
||||
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"cache_read_input_token_cost": 5.5e-07,
|
||||
|
|
@ -70848,7 +71059,11 @@
|
|||
"output_cost_per_token": 7.5e-06,
|
||||
"output_cost_per_token_batches": 3.75e-06,
|
||||
"rpm": 1000.0,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_low": 0.014,
|
||||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"source": "https://ai.google.dev/gemini-api/docs/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -70893,7 +71108,11 @@
|
|||
"output_cost_per_token": 7.5e-06,
|
||||
"output_cost_per_token_batches": 3.75e-06,
|
||||
"output_cost_per_token_priority": 1.35e-05,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_low": 0.014,
|
||||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -70939,7 +71158,11 @@
|
|||
"output_cost_per_token": 7.5e-06,
|
||||
"output_cost_per_token_batches": 3.75e-06,
|
||||
"output_cost_per_token_priority": 1.35e-05,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_low": 0.014,
|
||||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -71019,7 +71242,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"claude-haiku-5-5": {
|
||||
"claude-haiku-5-5": {
|
||||
"input_cost_per_token_above_100k_tokens_batches": 2.5e-07,
|
||||
"output_cost_per_token_above_100k_tokens_batches": 1.25e-06,
|
||||
"cache_creation_input_token_cost_above_100k_tokens_batches": 3.125e-07,
|
||||
|
|
@ -71787,6 +72010,13 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'"
|
||||
},
|
||||
"azure_ai/decisions/Microsoft-Decision-1": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'"
|
||||
},
|
||||
"gpt-rosalind-discovery": {
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"input_cost_per_token": 5e-06,
|
||||
|
|
|
|||
|
|
@ -2,9 +2,10 @@
|
|||
|
||||
PARITY pairs the streamed case with its plain twin where both bill the same row."""
|
||||
|
||||
from typing import Final
|
||||
from typing import Final, cast
|
||||
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
COST_MAP,
|
||||
BinaryResponse,
|
||||
CostTrackingTestCase,
|
||||
Deployment,
|
||||
|
|
@ -18,6 +19,7 @@ from integration.cost_calculation.cost_tracking_case import (
|
|||
SseResponse,
|
||||
WavUpload,
|
||||
)
|
||||
from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_key
|
||||
from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames
|
||||
|
||||
|
||||
|
|
@ -7848,6 +7850,74 @@ GPT_6_1_SOL_INPUT_TEXT: Final = CostTrackingTestCase(
|
|||
)
|
||||
|
||||
|
||||
_OPENAI_GPT_6_LUNA_DECISIONS_MODEL: Final = decisions_pricing_key("openai", "gpt-6-luna")
|
||||
_OPENAI_GPT_6_LUNA_DECISIONS_INPUT_TOKENS: Final = 367
|
||||
_OPENAI_GPT_6_LUNA_DECISIONS_OUTPUT_TOKENS: Final = 3
|
||||
_OPENAI_GPT_6_LUNA_DECISIONS_INPUT_RATE: Final = cast(
|
||||
float,
|
||||
COST_MAP[_OPENAI_GPT_6_LUNA_DECISIONS_MODEL].input_cost_per_token,
|
||||
)
|
||||
_OPENAI_GPT_6_LUNA_DECISIONS_SPEND: Final = (
|
||||
_OPENAI_GPT_6_LUNA_DECISIONS_INPUT_TOKENS * _OPENAI_GPT_6_LUNA_DECISIONS_INPUT_RATE
|
||||
)
|
||||
_OPENAI_GPT_6_LUNA_DECISIONS_RESPONSE: Final = JsonResponse(
|
||||
content_type="application/json",
|
||||
body={
|
||||
"model": "gpt-6-luna",
|
||||
"answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}],
|
||||
"usage": {
|
||||
"input_tokens": _OPENAI_GPT_6_LUNA_DECISIONS_INPUT_TOKENS,
|
||||
"input_tokens_details": {"cached_tokens": 256, "cache_write_tokens": 64},
|
||||
"output_tokens": _OPENAI_GPT_6_LUNA_DECISIONS_OUTPUT_TOKENS,
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
"total_tokens": _OPENAI_GPT_6_LUNA_DECISIONS_INPUT_TOKENS + _OPENAI_GPT_6_LUNA_DECISIONS_OUTPUT_TOKENS,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
OPENAI_GPT_6_LUNA_DECISIONS: Final = CostTrackingTestCase(
|
||||
name="openai-gpt-6-luna-decisions-evaluation-pricing",
|
||||
covers="quota_management.spend_tracking.decisions_costs",
|
||||
model="gpt-6-luna",
|
||||
endpoint="/v1/decisions",
|
||||
deployment=Deployment(model="openai/gpt-6-luna"),
|
||||
request={
|
||||
"model": "$MODEL",
|
||||
"input": "The package arrived broken.",
|
||||
"questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}],
|
||||
},
|
||||
response=_OPENAI_GPT_6_LUNA_DECISIONS_RESPONSE,
|
||||
expected=ExactExpected(
|
||||
spend=_OPENAI_GPT_6_LUNA_DECISIONS_SPEND,
|
||||
input_cost=_OPENAI_GPT_6_LUNA_DECISIONS_SPEND,
|
||||
output_cost=0.0,
|
||||
prompt_tokens=_OPENAI_GPT_6_LUNA_DECISIONS_INPUT_TOKENS,
|
||||
completion_tokens=_OPENAI_GPT_6_LUNA_DECISIONS_OUTPUT_TOKENS,
|
||||
),
|
||||
)
|
||||
|
||||
OPENAI_GPT_6_LUNA_SYSTEMONE: Final = CostTrackingTestCase(
|
||||
name="openai-gpt-6-luna-systemone-evaluation-pricing",
|
||||
covers="quota_management.spend_tracking.decisions_costs",
|
||||
model=_OPENAI_GPT_6_LUNA_DECISIONS_MODEL,
|
||||
endpoint="/v1/systemone",
|
||||
deployment=Deployment(model="openai/gpt-6-luna"),
|
||||
request={
|
||||
"model": "$MODEL",
|
||||
"state": "The package arrived broken.",
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
},
|
||||
response=_OPENAI_GPT_6_LUNA_DECISIONS_RESPONSE,
|
||||
expected=ExactExpected(
|
||||
spend=_OPENAI_GPT_6_LUNA_DECISIONS_SPEND,
|
||||
input_cost=_OPENAI_GPT_6_LUNA_DECISIONS_SPEND,
|
||||
output_cost=0.0,
|
||||
prompt_tokens=_OPENAI_GPT_6_LUNA_DECISIONS_INPUT_TOKENS,
|
||||
completion_tokens=_OPENAI_GPT_6_LUNA_DECISIONS_OUTPUT_TOKENS,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
||||
GPT_5_3_CODEX_INPUT_TEXT,
|
||||
GPT_5_3_CODEX_CACHE_READ,
|
||||
|
|
@ -7980,6 +8050,8 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
GPT_5_6_CHAT_REQUEST_TO_EMBEDDING_ENTRY,
|
||||
GPT_5_6_CLIENT_DISCONNECT_MID_STREAM,
|
||||
GPT_6_1_SOL_INPUT_TEXT,
|
||||
OPENAI_GPT_6_LUNA_DECISIONS,
|
||||
OPENAI_GPT_6_LUNA_SYSTEMONE,
|
||||
)
|
||||
|
||||
PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ from __future__ import annotations
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
from litellm.litellm_core_utils.decisions_pricing import is_decisions_pricing_key
|
||||
|
||||
from integration.cost_calculation.cases import (
|
||||
anthropic,
|
||||
azure,
|
||||
|
|
@ -61,14 +63,18 @@ def data_errors() -> tuple[str, ...]:
|
|||
case.session_model for case in REALTIME_CASES if case.session_model is not None
|
||||
)
|
||||
unknown_models: Final = sorted(model for model in case_models if model not in COST_MAP)
|
||||
missing_cases: Final = sorted(model for model in COST_MAP if model not in case_models)
|
||||
missing_cases: Final = sorted(
|
||||
model
|
||||
for model, entry in COST_MAP.items()
|
||||
if model not in case_models and not is_decisions_pricing_key(model, entry.litellm_provider)
|
||||
)
|
||||
duplicate_names: Final = sorted(
|
||||
name for name in {case.name for case in _ALL_CASES} if sum(case.name == name for case in _ALL_CASES) > 1
|
||||
)
|
||||
input_rates: Final = tuple(
|
||||
(entry.litellm_provider, entry.input_cost_per_token, model)
|
||||
for model, entry in COST_MAP.items()
|
||||
if entry.mode != "realtime"
|
||||
if entry.mode != "realtime" and not is_decisions_pricing_key(model, entry.litellm_provider)
|
||||
)
|
||||
shared_input_rate_details: Final = tuple(
|
||||
(
|
||||
|
|
|
|||
|
|
@ -4,8 +4,17 @@ Plain JSON-shaped data: cost_tracking_case.py validates it into CostMapEntry and
|
|||
|
||||
from typing import Final
|
||||
|
||||
from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_key
|
||||
from pydantic import JsonValue
|
||||
|
||||
_PERPLEXITY_DECISION_COST: Final[dict[str, JsonValue]] = {
|
||||
"input_cost_per_token": 4e-08,
|
||||
"litellm_provider": "perplexity",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
}
|
||||
|
||||
COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
|
||||
"dashscope/qwen4-max": {
|
||||
"litellm_provider": "dashscope",
|
||||
|
|
@ -32,6 +41,26 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
|
|||
"input_cost_per_token": 3.2e-06,
|
||||
"output_cost_per_token": 1.6e-05,
|
||||
},
|
||||
"gpt-6-luna": {
|
||||
"cache_creation_input_token_cost": 1.25e-07,
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 922000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-07,
|
||||
},
|
||||
"openai/decisions/gpt-6-luna": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
"input_cost_per_token_above_272k_tokens": 2e-07,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 922000,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
"regional_processing_uplift_multiplier_us": 1.1,
|
||||
"source": "https://developers.openai.com/api/docs/guides/decisions#pricing-and-availability",
|
||||
},
|
||||
"perplexity/sonar-next": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "chat",
|
||||
|
|
@ -43,13 +72,8 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
|
|||
"search_context_size_high": 0.012,
|
||||
},
|
||||
},
|
||||
"perplexity/pplx-decider-v1-27b": {
|
||||
"input_cost_per_token": 4e-08,
|
||||
"litellm_provider": "perplexity",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
},
|
||||
"perplexity/pplx-decider-v1-27b": _PERPLEXITY_DECISION_COST,
|
||||
decisions_pricing_key("perplexity", "pplx-decider-v1-27b"): _PERPLEXITY_DECISION_COST,
|
||||
"deepseek/deepseek-v4-chat": {
|
||||
"litellm_provider": "deepseek",
|
||||
"mode": "chat",
|
||||
|
|
|
|||
|
|
@ -44,8 +44,12 @@ class CostMapEntry(BaseModel):
|
|||
max_output_tokens: int | None = None
|
||||
supports_function_calling: bool | None = None
|
||||
input_cost_per_token: float | None = None
|
||||
input_cost_per_token_above_272k_tokens: float | None = None
|
||||
input_cost_per_query: float | None = None
|
||||
output_cost_per_token: float | None = None
|
||||
regional_processing_uplift_multiplier_eu: float | None = None
|
||||
regional_processing_uplift_multiplier_us: float | None = None
|
||||
source: str | None = None
|
||||
input_cost_per_token_batches: float | None = None
|
||||
output_cost_per_token_batches: float | None = None
|
||||
output_cost_per_image_token_batches: float | None = None
|
||||
|
|
@ -249,6 +253,7 @@ class CostTrackingTestCase(BaseModel):
|
|||
"/v1/audio/speech",
|
||||
"/v1/images/generations",
|
||||
"/v1/images/edits",
|
||||
"/v1/decisions",
|
||||
"/v1/systemone",
|
||||
]
|
||||
| Annotated[str, Field(pattern=r"^/(gemini|anthropic|bedrock)/")]
|
||||
|
|
|
|||
|
|
@ -252,7 +252,7 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase)
|
|||
assert row.spend == 0, f"{case.name}: failure spend was {row.spend}"
|
||||
return
|
||||
assert response.is_success, f"{case.name}: proxy returned {response.status_code}: {response.text[:400]}"
|
||||
if case.endpoint == "/v1/systemone":
|
||||
if case.endpoint in {"/v1/decisions", "/v1/systemone"}:
|
||||
observed: Final = JSON_OBJECT.validate_json(
|
||||
httpx.get(f"{gateway.upstream_url}/__observations", timeout=5, trust_env=False).content
|
||||
)
|
||||
|
|
@ -261,22 +261,37 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase)
|
|||
for value in observed["requests"]
|
||||
if isinstance(value, dict) and value.get("path") == f"/{scenario_id}/v1/decisions"
|
||||
)
|
||||
expected_body: Final = (
|
||||
{
|
||||
"model": "pplx-decider-v1-27b",
|
||||
"state": {"source": "cost-tracking"},
|
||||
"questions": {
|
||||
"is_defect": {
|
||||
"type": "noul",
|
||||
"instructions": "Is this a defect?",
|
||||
}
|
||||
},
|
||||
}
|
||||
if case.rates.litellm_provider == "perplexity"
|
||||
else {
|
||||
"model": "gpt-6-luna",
|
||||
"input": case.request["input"] if case.endpoint == "/v1/decisions" else case.request["state"],
|
||||
"questions": [
|
||||
{
|
||||
"type": "predicate",
|
||||
"name": "is_defect",
|
||||
"instructions": "Is this a defect?",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
assert decision_observations == (
|
||||
{
|
||||
"path": f"/{scenario_id}/v1/decisions",
|
||||
"authorization": "Bearer sk-scripted-provider",
|
||||
"method": "POST",
|
||||
"api_key": "",
|
||||
"body": {
|
||||
"model": "pplx-decider-v1-27b",
|
||||
"state": {"source": "cost-tracking"},
|
||||
"questions": {
|
||||
"is_defect": {
|
||||
"type": "noul",
|
||||
"instructions": "Is this a defect?",
|
||||
}
|
||||
},
|
||||
},
|
||||
"body": expected_body,
|
||||
},
|
||||
)
|
||||
if case.response.content_type == "text/event-stream":
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import litellm
|
|||
from litellm.cost_calculator import get_response_cost_from_hidden_params
|
||||
from litellm.constants import AWS_SIGNING_MAX_THREADS
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_key
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.llms.bedrock.base_aws_llm import AWS_SIGNING_EXECUTOR
|
||||
from litellm.types.decisions import (
|
||||
|
|
@ -307,6 +308,82 @@ def test_decisions_cost_uses_litellm_token_pricing() -> None:
|
|||
assert cost == pytest.approx(expected_cost)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_decisions_uses_evaluation_pricing_and_explicit_base_model(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE)
|
||||
response: Final = await litellm.adecisions(
|
||||
model="openai/gpt-6-luna",
|
||||
input="The package arrived broken.",
|
||||
questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}],
|
||||
api_key="caller-key",
|
||||
api_base="https://api.openai.com/v1",
|
||||
)
|
||||
decisions_key: Final = decisions_pricing_key("openai", "gpt-6-luna")
|
||||
evaluation_cost: Final = litellm.model_cost[decisions_key]
|
||||
chat_cost: Final = litellm.model_cost["gpt-6-luna"]
|
||||
expected_chat_cost: Final = (
|
||||
(_INPUT_TOKENS - _CACHED_TOKENS - _CACHE_WRITE_TOKENS) * float(chat_cost["input_cost_per_token"])
|
||||
+ _CACHED_TOKENS * float(chat_cost["cache_read_input_token_cost"])
|
||||
+ _CACHE_WRITE_TOKENS * float(chat_cost["cache_creation_input_token_cost"])
|
||||
+ _OUTPUT_TOKENS * float(chat_cost["output_cost_per_token"])
|
||||
)
|
||||
|
||||
assert route.called
|
||||
assert response._hidden_params["model"] == "openai/gpt-6-luna"
|
||||
assert response._hidden_params["provider_response_model"] == decisions_key
|
||||
assert litellm.completion_cost(completion_response=response) == pytest.approx(
|
||||
_INPUT_TOKENS * float(evaluation_cost["input_cost_per_token"])
|
||||
)
|
||||
assert litellm.completion_cost(completion_response=response, base_model="gpt-6-luna") == pytest.approx(
|
||||
expected_chat_cost
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_systemone_uses_evaluation_pricing(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE)
|
||||
response: Final = await litellm.asystemone(
|
||||
model="openai/gpt-6-luna",
|
||||
state="The package arrived broken.",
|
||||
questions=_QUESTIONS,
|
||||
api_key="caller-key",
|
||||
api_base="https://api.openai.com/v1",
|
||||
)
|
||||
decisions_key: Final = decisions_pricing_key("openai", "gpt-6-luna")
|
||||
evaluation_cost: Final = litellm.model_cost[decisions_key]
|
||||
|
||||
assert route.called
|
||||
assert response._hidden_params["model"] == "openai/gpt-6-luna"
|
||||
assert response._hidden_params["provider_response_model"] == decisions_key
|
||||
assert litellm.completion_cost(completion_response=response) == pytest.approx(
|
||||
_INPUT_TOKENS * float(evaluation_cost["input_cost_per_token"])
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_systemone_falls_back_to_plain_pricing_without_decisions_row(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
route: Final = respx_mock.post("https://decisions.example/v1/decisions").respond(json=_RESPONSE)
|
||||
response: Final = await litellm.asystemone(
|
||||
model="perplexity/sonar",
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
api_key="caller-key",
|
||||
api_base="https://decisions.example/v1",
|
||||
)
|
||||
plain_cost: Final = litellm.model_cost["perplexity/sonar"]
|
||||
|
||||
assert route.called
|
||||
assert response._hidden_params["provider_response_model"] == "perplexity/sonar"
|
||||
assert litellm.completion_cost(completion_response=response) == pytest.approx(
|
||||
_INPUT_TOKENS * float(plain_cost["input_cost_per_token"])
|
||||
+ _OUTPUT_TOKENS * float(plain_cost["output_cost_per_token"])
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"response",
|
||||
(
|
||||
|
|
@ -1293,13 +1370,8 @@ async def test_openai_decisions_translate_systemone_to_the_openai_wire_contract_
|
|||
}
|
||||
assert response.answers == {"is_defect": NoulAnswer(type="noul", noul=0.9)}
|
||||
assert response.hidden_params["custom_llm_provider"] == "openai"
|
||||
luna_cost: Final = litellm.model_cost["gpt-6-luna"]
|
||||
expected_cost: Final = (
|
||||
(_INPUT_TOKENS - _CACHED_TOKENS - _CACHE_WRITE_TOKENS) * float(luna_cost["input_cost_per_token"])
|
||||
+ _CACHED_TOKENS * float(luna_cost["cache_read_input_token_cost"])
|
||||
+ _CACHE_WRITE_TOKENS * float(luna_cost["cache_creation_input_token_cost"])
|
||||
+ _OUTPUT_TOKENS * float(luna_cost["output_cost_per_token"])
|
||||
)
|
||||
evaluation_cost: Final = litellm.model_cost[decisions_pricing_key("openai", "gpt-6-luna")]
|
||||
expected_cost: Final = _INPUT_TOKENS * float(evaluation_cost["input_cost_per_token"])
|
||||
assert expected_cost > 0
|
||||
assert litellm.completion_cost(completion_response=response) == pytest.approx(expected_cost)
|
||||
|
||||
|
|
@ -1638,6 +1710,28 @@ async def test_azure_ai_decision_posts_deployment_to_foundry_systemone_route(
|
|||
assert response._hidden_params["model"] == "azure_ai/decision-1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_azure_ai_systemone_uses_its_decisions_pricing_row(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post("https://res.services.ai.azure.com/providers/microsoft/v1/systemone").respond(
|
||||
json={**_RESPONSE, "model": "microsoft-decision-1"}
|
||||
)
|
||||
response: Final = await litellm.asystemone(
|
||||
model="azure_ai/Microsoft-Decision-1",
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
api_base="https://res.services.ai.azure.com",
|
||||
api_key="foundry-key",
|
||||
)
|
||||
pricing_key: Final = decisions_pricing_key("azure_ai", "Microsoft-Decision-1")
|
||||
evaluation_cost: Final = litellm.model_cost[pricing_key]
|
||||
|
||||
assert route.called
|
||||
assert response._hidden_params["provider_response_model"] == pricing_key
|
||||
assert litellm.completion_cost(completion_response=response) == pytest.approx(
|
||||
_INPUT_TOKENS * float(evaluation_cost["input_cost_per_token"])
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_azure_ai_decision_reads_foundry_env_and_keeps_gateway_path_prefix(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
|
|
|||
33
tests/unit/litellm_core_utils/test_decisions_pricing.py
Normal file
33
tests/unit/litellm_core_utils/test_decisions_pricing.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from litellm.litellm_core_utils.decisions_pricing import (
|
||||
decisions_pricing_key,
|
||||
decisions_pricing_model,
|
||||
is_decisions_pricing_key,
|
||||
)
|
||||
|
||||
|
||||
def test_decisions_pricing_key_uses_provider_decisions_model_order() -> None:
|
||||
pricing_key: Final = decisions_pricing_key("openai", "gpt-6-luna")
|
||||
|
||||
assert pricing_key == "/".join(("openai", "decisions", "gpt-6-luna"))
|
||||
|
||||
|
||||
def test_is_decisions_pricing_key_matches_only_the_provider_prefix() -> None:
|
||||
decision_key: Final = decisions_pricing_key("openai", "gpt-6-luna")
|
||||
plain_key: Final = "openai/gpt-6-luna"
|
||||
other_provider_key: Final = decisions_pricing_key("perplexity", "gpt-6-luna")
|
||||
|
||||
assert is_decisions_pricing_key(decision_key, "openai")
|
||||
assert not is_decisions_pricing_key(plain_key, "openai")
|
||||
assert not is_decisions_pricing_key(other_provider_key, "openai")
|
||||
|
||||
|
||||
def test_decisions_pricing_model_uses_variant_or_falls_back_to_plain_model() -> None:
|
||||
pricing_key: Final = decisions_pricing_key("openai", "gpt-6-luna")
|
||||
variant_cost_map: Final[Mapping[str, object]] = {pricing_key: {}}
|
||||
plain_cost_map: Final[Mapping[str, object]] = {"openai/gpt-6-luna": {}}
|
||||
|
||||
assert decisions_pricing_model("openai", "gpt-6-luna", variant_cost_map) == pricing_key
|
||||
assert decisions_pricing_model("openai", "gpt-6-luna", plain_cost_map) == "openai/gpt-6-luna"
|
||||
|
|
@ -6,6 +6,7 @@ import httpx
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_key
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.typesafe_passthrough_logging_handler import (
|
||||
TypeSafePassthroughLoggingHandler,
|
||||
|
|
@ -72,6 +73,40 @@ def test_uses_registry_pricing_and_standard_usage():
|
|||
assert response["kwargs"]["combined_usage_object"].total_tokens == 360
|
||||
|
||||
|
||||
def test_uses_decisions_pricing_row_when_available(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
model: Final = "nimble-latest"
|
||||
plain_key: Final = f"bespoke/{model}"
|
||||
decisions_key: Final = decisions_pricing_key("bespoke", model)
|
||||
plain_pricing: Final = {
|
||||
"input_cost_per_token": 1.0,
|
||||
"output_cost_per_token": 1.0,
|
||||
"litellm_provider": "bespoke",
|
||||
"mode": "chat",
|
||||
}
|
||||
decisions_pricing: Final = {
|
||||
"input_cost_per_token": 2.0,
|
||||
"output_cost_per_token": 3.0,
|
||||
"litellm_provider": "bespoke",
|
||||
"mode": "evaluation",
|
||||
}
|
||||
monkeypatch.setitem(litellm.model_cost, plain_key, plain_pricing)
|
||||
monkeypatch.setitem(litellm.model_cost, decisions_key, decisions_pricing)
|
||||
result: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler(
|
||||
httpx_response=_response(),
|
||||
response_body={"model": model, "usage": {"input_tokens": 2, "output_tokens": 1}},
|
||||
logging_obj=_logging_obj(),
|
||||
url_route="https://api.bespoke.example/v1/systemone",
|
||||
result='{"answers": {}}',
|
||||
start_time=datetime(2000, 1, 1),
|
||||
end_time=datetime(2000, 1, 1),
|
||||
cache_hit=False,
|
||||
request_body={"model": model},
|
||||
custom_llm_provider="bespoke",
|
||||
)
|
||||
|
||||
assert result["kwargs"]["response_cost"] == pytest.approx(7.0)
|
||||
|
||||
|
||||
def test_falls_back_to_request_model_when_response_model_is_missing():
|
||||
result = _handler_result(
|
||||
{"usage": {"input_tokens": 10, "output_tokens": 2}},
|
||||
|
|
@ -142,68 +177,110 @@ def test_success_handler_dispatches_to_typesafe_handler():
|
|||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("guardrail_cost", [0.0, 0.25])
|
||||
@pytest.mark.parametrize("metadata_slot", ["metadata", "litellm_metadata"])
|
||||
@pytest.mark.parametrize("provider,requested,routing_model", [
|
||||
("laya", "english", "multilingual"), ("laya", "english", None),
|
||||
("bespoke", "nimble-latest", None),
|
||||
("bespoke", "bespokelabs/Bespoke-Nimble-9B", None),
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"provider,requested,routing_model",
|
||||
[
|
||||
("laya", "english", "multilingual"),
|
||||
("laya", "english", None),
|
||||
("bespoke", "nimble-latest", None),
|
||||
("bespoke", "bespokelabs/Bespoke-Nimble-9B", None),
|
||||
],
|
||||
)
|
||||
async def test_oss_gateway_accounts_for_checkpoint_usage_and_registered_cost(
|
||||
monkeypatch: pytest.MonkeyPatch, routing_model: str | None, metadata_slot: str, guardrail_cost: float,
|
||||
provider: str, requested: str
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
routing_model: str | None,
|
||||
metadata_slot: str,
|
||||
guardrail_cost: float,
|
||||
provider: str,
|
||||
requested: str,
|
||||
) -> None:
|
||||
checkpoint: Final = routing_model or requested
|
||||
model: Final = f"{provider}/{checkpoint}"
|
||||
input_rate: Final = 0.002
|
||||
output_rate: Final = 0.005
|
||||
monkeypatch.setitem(litellm.model_cost, model, {
|
||||
"input_cost_per_token": input_rate, "output_cost_per_token": output_rate,
|
||||
"litellm_provider": provider, "mode": "evaluation",
|
||||
})
|
||||
pricing_row: Final = {
|
||||
"input_cost_per_token": input_rate,
|
||||
"output_cost_per_token": output_rate,
|
||||
"litellm_provider": provider,
|
||||
"mode": "evaluation",
|
||||
}
|
||||
monkeypatch.setitem(litellm.model_cost, model, pricing_row)
|
||||
monkeypatch.setitem(litellm.model_cost, decisions_pricing_key(provider, checkpoint), pricing_row)
|
||||
start: Final = datetime.now()
|
||||
logging_obj: Final = Logging(
|
||||
model=requested, messages=[], stream=False, call_type="pass_through_endpoint",
|
||||
start_time=start, litellm_call_id="oss-accounting", function_id="oss-accounting", kwargs={},
|
||||
model=requested,
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="pass_through_endpoint",
|
||||
start_time=start,
|
||||
litellm_call_id="oss-accounting",
|
||||
function_id="oss-accounting",
|
||||
kwargs={},
|
||||
)
|
||||
from fastapi import Request
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import HttpPassThroughEndpointHelpers
|
||||
|
||||
request: Final = Request({
|
||||
"type": "http", "method": "POST", "path": f"/{provider}/v1/systemone",
|
||||
"headers": [], "query_string": b"",
|
||||
})
|
||||
request: Final = Request(
|
||||
{
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": f"/{provider}/v1/systemone",
|
||||
"headers": [],
|
||||
"query_string": b"",
|
||||
}
|
||||
)
|
||||
auth: Final = UserAPIKeyAuth(
|
||||
api_key="oss-budget-key", token="oss-budget-key",
|
||||
api_key="oss-budget-key",
|
||||
token="oss-budget-key",
|
||||
model_max_budget={f"{provider}/{requested}": {"budget_limit": 0.01, "time_period": "1d"}},
|
||||
)
|
||||
request_body: Final = {"model": requested, metadata_slot: {"model_group": "unbounded-client-choice"}}
|
||||
logging_kwargs: Final = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint(
|
||||
request=request, user_api_key_dict=auth, logging_obj=logging_obj,
|
||||
passthrough_logging_payload={"url": f"https://{provider}.test/v1/systemone"}, _parsed_body=request_body,
|
||||
request=request,
|
||||
user_api_key_dict=auth,
|
||||
logging_obj=logging_obj,
|
||||
passthrough_logging_payload={"url": f"https://{provider}.test/v1/systemone"},
|
||||
_parsed_body=request_body,
|
||||
)
|
||||
logging_kwargs["litellm_params"]["metadata"]["standard_logging_guardrail_information"] = [
|
||||
{"guardrail_name": "trusted-hook", "guardrail_cost": guardrail_cost},
|
||||
]
|
||||
logging_obj.update_environment_variables(
|
||||
model=requested, user="unknown", optional_params={},
|
||||
litellm_params=logging_kwargs["litellm_params"], call_type="pass_through_endpoint",
|
||||
model=requested,
|
||||
user="unknown",
|
||||
optional_params={},
|
||||
litellm_params=logging_kwargs["litellm_params"],
|
||||
call_type="pass_through_endpoint",
|
||||
)
|
||||
body: Final = {
|
||||
"model": "laya-rl-agent" if provider == "laya" else requested, "usage": {"input_tokens": 10, "output_tokens": 3},
|
||||
"model": "laya-rl-agent" if provider == "laya" else requested,
|
||||
"usage": {"input_tokens": 10, "output_tokens": 3},
|
||||
**({"routing": {"model": routing_model}} if routing_model else {}),
|
||||
}
|
||||
normalized: Final = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload(
|
||||
httpx_response=httpx.Response(200, request=httpx.Request("POST", f"https://{provider}.test/v1/systemone"), json=body),
|
||||
response_body=body, request_body={"model": requested}, logging_obj=logging_obj,
|
||||
url_route=f"https://{provider}.test/v1/systemone", result="{}", start_time=start,
|
||||
end_time=datetime.now(), cache_hit=False, custom_llm_provider=provider, **logging_kwargs,
|
||||
httpx_response=httpx.Response(
|
||||
200, request=httpx.Request("POST", f"https://{provider}.test/v1/systemone"), json=body
|
||||
),
|
||||
response_body=body,
|
||||
request_body={"model": requested},
|
||||
logging_obj=logging_obj,
|
||||
url_route=f"https://{provider}.test/v1/systemone",
|
||||
result="{}",
|
||||
start_time=start,
|
||||
end_time=datetime.now(),
|
||||
cache_hit=False,
|
||||
custom_llm_provider=provider,
|
||||
**logging_kwargs,
|
||||
)
|
||||
logged: Final = normalized["kwargs"]
|
||||
expected_cost: Final = 10 * input_rate + 3 * output_rate
|
||||
assert (logged["model"], logged["custom_llm_provider"]) == (model, provider)
|
||||
assert logged["response_cost"] == pytest.approx(expected_cost)
|
||||
assert logged["combined_usage_object"].model_dump(exclude_none=True) == {
|
||||
"prompt_tokens": 10, "completion_tokens": 3, "total_tokens": 13,
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 3,
|
||||
"total_tokens": 13,
|
||||
}
|
||||
assert logging_obj.model_call_details["model"] == model
|
||||
assert logging_obj.model_call_details["response_cost"] == pytest.approx(expected_cost)
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import pytest
|
|||
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_key, is_decisions_pricing_key
|
||||
from litellm.litellm_core_utils.fallback_generalizations import match_capability_generalizations
|
||||
from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name
|
||||
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
||||
|
|
@ -25,6 +26,23 @@ BACKUP_PRICES_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_ba
|
|||
SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json"
|
||||
|
||||
|
||||
def _evaluation_decisions_row_pair(
|
||||
key: str,
|
||||
value: object,
|
||||
prices: Mapping[str, object],
|
||||
) -> tuple[str, Mapping[str, object], Mapping[str, object]] | None:
|
||||
if not isinstance(value, Mapping):
|
||||
return None
|
||||
provider: Final = value.get("litellm_provider")
|
||||
if not isinstance(provider, str) or not is_decisions_pricing_key(key, provider):
|
||||
return None
|
||||
model: Final = key.removeprefix(decisions_pricing_key(provider, ""))
|
||||
plain_row: Final = prices.get(f"{provider}/{model}")
|
||||
if not isinstance(plain_row, Mapping) or plain_row.get("mode") != "evaluation":
|
||||
return None
|
||||
return key, value, plain_row
|
||||
|
||||
|
||||
def build_validator(schema: dict) -> jsonschema.Draft202012Validator:
|
||||
return jsonschema.Draft202012Validator(schema, format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER)
|
||||
|
||||
|
|
@ -430,6 +448,7 @@ def test_openai_reasoning_family_entries_carry_supports_reasoning(prices: dict):
|
|||
if isinstance(entry, dict)
|
||||
and entry.get("litellm_provider") == "openai"
|
||||
and is_openai_reasoning_family(name)
|
||||
and not is_decisions_pricing_key(name, entry["litellm_provider"])
|
||||
and entry.get("supports_reasoning") is not True
|
||||
]
|
||||
assert unflagged == [], (
|
||||
|
|
@ -439,6 +458,18 @@ def test_openai_reasoning_family_entries_carry_supports_reasoning(prices: dict):
|
|||
)
|
||||
|
||||
|
||||
def test_decisions_pricing_rows_match_their_plain_evaluation_rows(prices: dict) -> None:
|
||||
matching_rows: Final = tuple(
|
||||
row_pair
|
||||
for key, value in prices.items()
|
||||
if (row_pair := _evaluation_decisions_row_pair(key, value, prices)) is not None
|
||||
)
|
||||
|
||||
assert matching_rows
|
||||
for key, decisions_row, evaluation_row in matching_rows:
|
||||
assert decisions_row == evaluation_row, f"{key} must copy its plain evaluation row"
|
||||
|
||||
|
||||
_ABSENT: Final = object()
|
||||
|
||||
REASONING_ANNOTATION_KEYS: Final = (
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ from litellm.constants import DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT
|
|||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.humanloop import HumanloopLogger
|
||||
from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_key
|
||||
from litellm.llms.base_llm.audio_transcription.transformation import BaseAudioTranscriptionConfig
|
||||
from litellm.integrations.langfuse.langfuse_prompt_management import LangfusePromptManagement
|
||||
from litellm.litellm_core_utils import litellm_logging
|
||||
|
|
@ -2212,12 +2213,15 @@ def test_completion_registers_test_owned_model_prices() -> None:
|
|||
|
||||
|
||||
@pytest.mark.usefixtures("isolated_openai_model_sets")
|
||||
def test_add_known_models_skips_region_scoped_bedrock_pricing_keys(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def test_add_known_models_skips_non_model_pricing_keys(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
price: Final = {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002, "mode": "chat"}
|
||||
decisions_model: Final = "migration-openai-model"
|
||||
decisions_pricing_model: Final = decisions_pricing_key("openai", decisions_model)
|
||||
model_cost_map: Final = {
|
||||
"migration.bedrock-model-v1:0": {**price, "litellm_provider": "bedrock"},
|
||||
"bedrock/us-west-1/migration.bedrock-model-v1:0": {**price, "litellm_provider": "bedrock"},
|
||||
"migration-openai-model": {**price, "litellm_provider": "openai"},
|
||||
decisions_model: {**price, "litellm_provider": "openai"},
|
||||
decisions_pricing_model: {**price, "litellm_provider": "openai", "mode": "evaluation"},
|
||||
}
|
||||
monkeypatch.setattr(litellm, "bedrock_models", set(litellm.bedrock_models))
|
||||
monkeypatch.setattr(litellm, "models_by_provider", dict(litellm.models_by_provider))
|
||||
|
|
@ -2226,7 +2230,10 @@ def test_add_known_models_skips_region_scoped_bedrock_pricing_keys(monkeypatch:
|
|||
|
||||
assert "migration.bedrock-model-v1:0" in litellm.bedrock_models
|
||||
assert "bedrock/us-west-1/migration.bedrock-model-v1:0" not in litellm.bedrock_models
|
||||
assert "migration-openai-model" in litellm.models_by_provider["openai"]
|
||||
assert decisions_model in litellm.open_ai_chat_completion_models
|
||||
assert decisions_pricing_model not in litellm.open_ai_chat_completion_models
|
||||
assert decisions_model in litellm.models_by_provider["openai"]
|
||||
assert decisions_pricing_model not in litellm.models_by_provider["openai"]
|
||||
|
||||
|
||||
def test_utils_defines_tests_without_invoking_them_at_module_scope() -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue