diff --git a/CLAUDE.md b/CLAUDE.md index a3c24b84ea8..85ba96980b9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,7 +29,7 @@ End-to-end tests belong in `tests/e2e/` and must follow the harness conventions When creating PRs, don't set base to `main`. `litellm_internal_staging` is the default base branch and serves that purpose for both internal and external / OSS contributions -When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule +When writing a PR body, treat the comments and imperative instructions inside .github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule Same applies for filing bug reports and feature requests, with .github/ISSUE_TEMPLATE/bug_report.yml and .github/ISSUE_TEMPLATE/feature_request.yml, respectively diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index 7de42c00ede..a93c45ef840 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -90,7 +90,6 @@ class LangfuseOtelLogger(OpenTelemetry): "generation_name": LangfuseSpanAttributes.GENERATION_NAME, "generation_id": LangfuseSpanAttributes.GENERATION_ID, "parent_observation_id": LangfuseSpanAttributes.PARENT_OBSERVATION_ID, - "version": LangfuseSpanAttributes.GENERATION_VERSION, "mask_input": LangfuseSpanAttributes.MASK_INPUT, "mask_output": LangfuseSpanAttributes.MASK_OUTPUT, "trace_user_id": LangfuseSpanAttributes.TRACE_USER_ID, @@ -99,13 +98,18 @@ class LangfuseOtelLogger(OpenTelemetry): "trace_name": LangfuseSpanAttributes.TRACE_NAME, "trace_id": LangfuseSpanAttributes.TRACE_ID, "trace_metadata": LangfuseSpanAttributes.TRACE_METADATA, - "trace_version": LangfuseSpanAttributes.TRACE_VERSION, - "trace_release": LangfuseSpanAttributes.TRACE_RELEASE, + "trace_release": LangfuseSpanAttributes.RELEASE, "existing_trace_id": LangfuseSpanAttributes.EXISTING_TRACE_ID, "update_trace_keys": LangfuseSpanAttributes.UPDATE_TRACE_KEYS, "debug_langfuse": LangfuseSpanAttributes.DEBUG_LANGFUSE, } + version: Final = ( + metadata.get("trace_version") if metadata.get("trace_version") is not None else metadata.get("version") + ) + if version is not None: + safe_set_attribute(span, LangfuseSpanAttributes.VERSION.value, version) + for key, enum_attr in mapping.items(): if key in metadata and metadata[key] is not None: value = metadata[key] diff --git a/litellm/integrations/otel/presets/langfuse.py b/litellm/integrations/otel/presets/langfuse.py index 5104ee2ff55..c2f64422eff 100644 --- a/litellm/integrations/otel/presets/langfuse.py +++ b/litellm/integrations/otel/presets/langfuse.py @@ -42,9 +42,7 @@ def langfuse_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, public_key: Final = params.get("langfuse_public_key") secret_key: Final = params.get("langfuse_secret_key") if public_key and secret_key: - return { - "Authorization": _V1Langfuse._get_langfuse_authorization_header( - public_key=public_key, secret_key=secret_key - ) - } + return _V1Langfuse._build_langfuse_otel_headers( + _V1Langfuse._get_langfuse_authorization_header(public_key=public_key, secret_key=secret_key) + ) return {} diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 2462c282041..de1092bc02f 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -1,7 +1,7 @@ # What is this? ## Helper utilities import copy -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Any, Final, Literal import httpx @@ -181,7 +181,7 @@ def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metad def get_metadata_variable_name_from_kwargs( - kwargs: dict, + kwargs: Mapping[str, object], ) -> Literal["metadata", "litellm_metadata"]: """ Helper to return what the "metadata" field should be called in the request data diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 00ba060e1e5..3b2cdcf5ff7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -26120,11 +26120,12 @@ "supports_vision": true }, "groq/llama-3.1-8b-instant": { + "deprecation_date": "2026-08-16", "input_cost_per_token": 5e-08, "litellm_provider": "groq", - "max_input_tokens": 128000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-08, "supports_function_calling": true, @@ -26132,9 +26133,10 @@ "supports_tool_choice": true }, "groq/llama-3.3-70b-versatile": { + "deprecation_date": "2026-08-16", "input_cost_per_token": 5.9e-07, "litellm_provider": "groq", - "max_input_tokens": 128000, + "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", @@ -26155,7 +26157,28 @@ "supports_response_schema": false, "supports_tool_choice": true }, + "groq/meta-llama/llama-prompt-guard-2-22m": { + "input_cost_per_token": 3e-08, + "litellm_provider": "groq", + "max_input_tokens": 512, + "max_output_tokens": 512, + "max_tokens": 512, + "mode": "chat", + "output_cost_per_token": 3e-08, + "source": "https://console.groq.com/docs/models" + }, + "groq/meta-llama/llama-prompt-guard-2-86m": { + "input_cost_per_token": 4e-08, + "litellm_provider": "groq", + "max_input_tokens": 512, + "max_output_tokens": 512, + "max_tokens": 512, + "mode": "chat", + "output_cost_per_token": 4e-08, + "source": "https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-86m" + }, "groq/meta-llama/llama-guard-4-12b": { + "deprecation_date": "2026-03-05", "input_cost_per_token": 2e-07, "litellm_provider": "groq", "max_input_tokens": 8192, @@ -26165,6 +26188,7 @@ "output_cost_per_token": 2e-07 }, "groq/meta-llama/llama-4-maverick-17b-128e-instruct": { + "deprecation_date": "2026-03-09", "input_cost_per_token": 2e-07, "litellm_provider": "groq", "max_input_tokens": 131072, @@ -26178,6 +26202,7 @@ "supports_vision": true }, "groq/meta-llama/llama-4-scout-17b-16e-instruct": { + "deprecation_date": "2026-07-17", "input_cost_per_token": 1.1e-07, "litellm_provider": "groq", "max_input_tokens": 131072, @@ -26191,6 +26216,7 @@ "supports_vision": true }, "groq/moonshotai/kimi-k2-instruct-0905": { + "deprecation_date": "2026-04-15", "input_cost_per_token": 1e-06, "output_cost_per_token": 3e-06, "cache_read_input_token_cost": 5e-07, @@ -26208,8 +26234,8 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "groq", "max_input_tokens": 131072, - "max_output_tokens": 32766, - "max_tokens": 32766, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 6e-07, "search_context_cost_per_query": { @@ -26229,8 +26255,8 @@ "input_cost_per_token": 7.5e-08, "litellm_provider": "groq", "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-07, "search_context_cost_per_query": { @@ -26265,7 +26291,26 @@ "supports_tool_choice": true, "supports_web_search": true }, + "groq/canopylabs/orpheus-v1-english": { + "input_cost_per_character": 2.2e-05, + "litellm_provider": "groq", + "max_input_tokens": 4000, + "max_output_tokens": 50000, + "max_tokens": 50000, + "mode": "audio_speech", + "source": "https://console.groq.com/docs/model/canopylabs/orpheus-v1-english" + }, + "groq/canopylabs/orpheus-arabic-saudi": { + "input_cost_per_character": 4e-05, + "litellm_provider": "groq", + "max_input_tokens": 4000, + "max_output_tokens": 50000, + "max_tokens": 50000, + "mode": "audio_speech", + "source": "https://console.groq.com/docs/models" + }, "groq/playai-tts": { + "deprecation_date": "2025-12-31", "input_cost_per_character": 5e-05, "litellm_provider": "groq", "max_input_tokens": 10000, @@ -26273,7 +26318,23 @@ "max_tokens": 10000, "mode": "audio_speech" }, + "groq/qwen/qwen3.6-27b": { + "input_cost_per_token": 6e-07, + "litellm_provider": "groq", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://console.groq.com/docs/model/qwen/qwen3.6-27b", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_tool_choice": true, + "supports_vision": true + }, "groq/qwen/qwen3-32b": { + "deprecation_date": "2026-07-17", "input_cost_per_token": 2.9e-07, "litellm_provider": "groq", "max_input_tokens": 131000, @@ -45826,11 +45887,15 @@ }, "bedrock_mantle/openai.gpt-5.6-sol": { "input_cost_per_token": 5.5e-06, + "input_cost_per_token_above_272k_tokens": 1.1e-05, "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "output_cost_per_token": 3.3e-05, + "output_cost_per_token_above_272k_tokens": 4.95e-05, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 272000, + "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -45854,11 +45919,15 @@ }, "bedrock_mantle/openai.gpt-5.6-terra": { "input_cost_per_token": 2.2e-06, + "input_cost_per_token_above_272k_tokens": 4.4e-06, "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5.5e-06, "cache_read_input_token_cost": 2.2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4.4e-07, "output_cost_per_token": 1.32e-05, + "output_cost_per_token_above_272k_tokens": 1.98e-05, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 272000, + "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -45882,11 +45951,15 @@ }, "bedrock_mantle/openai.gpt-5.6-luna": { "input_cost_per_token": 2.2e-07, + "input_cost_per_token_above_272k_tokens": 4.4e-07, "cache_creation_input_token_cost": 2.75e-07, + "cache_creation_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost": 2.2e-08, + "cache_read_input_token_cost_above_272k_tokens": 4.4e-08, "output_cost_per_token": 1.32e-06, + "output_cost_per_token_above_272k_tokens": 1.98e-06, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 272000, + "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3ac002ce8cc..64a91b880cd 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -845,6 +845,22 @@ def cleanup_router_config_variables(): prisma_client = None +async def _flush_spend_logs_queue_on_shutdown() -> None: + if prisma_client is None: + return + + try: + from litellm.proxy.utils import drain_spend_logs_queue + + await drain_spend_logs_queue( + prisma_client=prisma_client, + db_writer_client=db_writer_client, + proxy_logging_obj=proxy_logging_obj, + ) + except Exception as e: # noqa: BLE001 # shutdown must continue even if the drain fails + verbose_proxy_logger.exception("Error flushing spend logs queue on shutdown: %s", e) + + async def proxy_shutdown_event(): global prisma_client, master_key, user_custom_auth, user_custom_key_generate, user_custom_key_update verbose_proxy_logger.info("Shutting down LiteLLM Proxy Server") @@ -1255,6 +1271,8 @@ async def proxy_startup_event(app: FastAPI): except Exception as e: verbose_proxy_logger.error("Error stopping DB health watchdog task: %s", e) + await _flush_spend_logs_queue_on_shutdown() + await proxy_config.stop_config_sync_subscriber() await proxy_config.stop_auth_cache_invalidation_subscriber() @@ -8731,14 +8749,14 @@ class ProxyStartupEvent: if general_settings.get("disable_spend_logs", False) is False: from litellm.proxy.utils import _monitor_spend_logs_queue - # Start background task to monitor spend logs queue size - asyncio.create_task( + monitor_task: Final = asyncio.create_task( _monitor_spend_logs_queue( prisma_client=prisma_client, db_writer_client=db_writer_client, proxy_logging_obj=proxy_logging_obj, ) ) + prisma_client.spend_logs_queue_monitor_task = monitor_task # rebind-ok: the client owns its monitor handle ### ADD NEW MODELS ### store_model_in_db = get_secret_bool("STORE_MODEL_IN_DB", store_model_in_db) or store_model_in_db diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index cce8379ab25..8a1fae42789 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1,4 +1,5 @@ import asyncio +import contextlib import copy import hashlib import inspect @@ -3006,6 +3007,7 @@ async def prefetch_config_params(prisma_client: "PrismaClient | None", param_nam class PrismaClient: spend_log_transactions: list = [] _spend_log_transactions_lock = asyncio.Lock() + spend_logs_queue_monitor_task: "asyncio.Task[None] | None" = None tool_usage_transactions: list["ToolUsageTransaction"] = [] _tool_usage_transactions_lock = asyncio.Lock() autorouter_turn_transactions: ClassVar[ @@ -5722,13 +5724,22 @@ async def update_spend_logs_job( logs_to_process: Final = prisma_client.spend_log_transactions[:MAX_LOGS_PER_INTERVAL] prisma_client.spend_log_transactions = prisma_client.spend_log_transactions[len(logs_to_process) :] - await ProxyUpdateSpend.update_spend_logs( - n_retry_times=n_retry_times, - prisma_client=prisma_client, - proxy_logging_obj=proxy_logging_obj, - db_writer_client=db_writer_client, - logs_to_process=logs_to_process, - ) + try: + await ProxyUpdateSpend.update_spend_logs( + n_retry_times=n_retry_times, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + db_writer_client=db_writer_client, + logs_to_process=logs_to_process, + ) + except asyncio.CancelledError: + async with prisma_client._spend_log_transactions_lock: + prisma_client.spend_log_transactions[:0] = logs_to_process + verbose_proxy_logger.warning( + "Spend tracking - spend log write cancelled, requeued %d rows for the next flush", + len(logs_to_process), + ) + raise # Guardrail/policy usage tracking (same batch, outside spend-logs update) try: @@ -5787,6 +5798,39 @@ async def update_spend_logs_job( ) +MAX_SPEND_LOG_DRAIN_ITERATIONS: Final = 20 + + +async def drain_spend_logs_queue( + prisma_client: PrismaClient, + db_writer_client: "AsyncHTTPHandler | None", + proxy_logging_obj: ProxyLogging, +) -> None: + monitor_task: Final = prisma_client.spend_logs_queue_monitor_task + if monitor_task is not None: + monitor_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await monitor_task + prisma_client.spend_logs_queue_monitor_task = None # rebind-ok: the client owns its monitor handle + + for _ in range(MAX_SPEND_LOG_DRAIN_ITERATIONS): + if await _total_queued_spend_transactions(prisma_client) == 0: + return + await update_spend_logs_job( + prisma_client=prisma_client, + db_writer_client=db_writer_client, + proxy_logging_obj=proxy_logging_obj, + ) + + remaining: Final = await _total_queued_spend_transactions(prisma_client) + if remaining > 0: + spend_log_error( + "Spend tracking - %d spend log rows still queued after %d drain passes", + remaining, + MAX_SPEND_LOG_DRAIN_ITERATIONS, + ) + + async def _monitor_spend_logs_queue( prisma_client: PrismaClient, db_writer_client: AsyncHTTPHandler | None, diff --git a/litellm/router.py b/litellm/router.py index 1f114f99118..fb2af41dcf2 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -96,6 +96,7 @@ from litellm.router_utils.add_retry_fallback_headers import ( response_in_flight_token_count, ) from litellm.router_utils.auto_router_model_naming import ( + AUTO_ROUTER_MODEL_PREFIX, classify_strategy_router_model, ) from litellm.router_utils.batch_utils import ( @@ -318,6 +319,8 @@ def model_info_is_active_for_environment(model_info: Mapping[str, object] | None _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT") +_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"}) + def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream]) -> bool: for chunk in chunks: @@ -10712,6 +10715,14 @@ class Router: return None + @staticmethod + def _is_strategy_marker_deployment(deployment: Mapping[str, object]) -> bool: + litellm_params: Final = deployment.get("litellm_params") + if not isinstance(litellm_params, Mapping): + return False + deployment_model: Final = litellm_params.get("model") + return isinstance(deployment_model, str) and classify_strategy_router_model(deployment_model) is not None + def _common_checks_available_deployment( self, model: str, @@ -10839,7 +10850,12 @@ class Router: model ] # update the model to the actual value if an alias has been passed in - return model, healthy_deployments + marker_flags: Final = tuple(self._is_strategy_marker_deployment(d) for d in healthy_deployments) + if all(marker_flags) or not any(marker_flags): + return model, healthy_deployments + return model, [ # mutable-ok: matches this function's list contract expected by downstream filters + d for d, is_marker in zip(healthy_deployments, marker_flags, strict=True) if not is_marker + ] def _filter_deployments_by_model_access_groups( self, @@ -11352,6 +11368,10 @@ class Router: return filtered + def _model_name_has_plain_deployments(self, model: str) -> bool: + indices: Final = self.model_name_to_deployment_indices.get(model) or () + return any(not self._is_strategy_marker_deployment(self.model_list[idx]) for idx in indices) + def _select_pre_routing_strategy( self, model: str, request_kwargs: dict ) -> "TaggedPreRoutingStrategy[PreRoutingStrategy] | None": @@ -11360,7 +11380,14 @@ class Router: that share a `model_name` by matching the request's tags against each registered strategy's tags before falling back to the first registered. Returns the tagged registry entry so the caller can tell whether the - request's tags were what selected it. + request's tags were what selected it, and can locate the marker + deployment the strategy was registered from via its (model_name, tags) + pair. + + With tag filtering enabled, strategies that all carry real tags matching + none of the request's do not capture it when the name also has plain + deployments: returning None hands the request to ordinary tag-aware + deployment selection. """ candidates: Final[list[TaggedPreRoutingStrategy[PreRoutingStrategy]]] = [ *self.auto_routers.get(model, []), @@ -11370,8 +11397,6 @@ class Router: ] if not candidates: return None - if len(candidates) == 1: - return candidates[0] request_tags: Final = _get_tags_from_request_kwargs(request_kwargs) if request_tags: @@ -11383,6 +11408,12 @@ class Router: for tagged in candidates: if "default" in tagged.tags: return tagged + if ( + self.enable_tag_filtering + and all(tagged.tags for tagged in candidates) + and self._model_name_has_plain_deployments(model) + ): + return None return candidates[0] async def async_pre_routing_hook( @@ -11445,25 +11476,47 @@ class Router: ) # `model` (the alias, e.g. "smart-router") is never the deployment actually - # called - apply the alias's own litellm_params (besides `model` itself, - # which is just the alias marker) to the request, since the tier/route - # deployment the hook selected won't have them. Router-only fields - # (tpm, rpm, weight, complexity_router_config, ...) are excluded from the - # actual outbound LLM call downstream by litellm.types.utils.all_litellm_params, + # called - apply the router marker's own litellm_params to the request, + # since the tier/route deployment the hook selected won't have them. The + # marker entry is looked up by its `auto_router/` model prefix and the + # selected strategy's tags, never by list position: plain deployments may + # share the alias `model_name` and must not leak their params (`api_base`, + # `api_key`, ...) onto the routed call. Router-only fields (tpm, rpm, + # weight, complexity_router_config, ...) are excluded from the actual + # outbound LLM call downstream by litellm.types.utils.all_litellm_params, # not here. Custom pricing fields ARE call params, so they must be # excluded here: they price the alias, not the deployment the hook # selected, and forwarding them re-registers the routed deployment at # the alias's price (an explicit 0 makes every alias request bill $0). if pre_routing_hook_response is not None: - alias_index: Final = self.model_name_to_deployment_indices.get(model, []) - if alias_index: - alias_litellm_params: Final = self.model_list[alias_index[0]].get("litellm_params", {}) - for key, value in alias_litellm_params.items(): - if key != "model" and key not in CustomPricingLiteLLMParams.model_fields and value is not None: - request_kwargs.setdefault(key, value) + for key, value in self._forwardable_alias_marker_params(model=model, strategy_tags=selected_strategy.tags): + request_kwargs.setdefault(key, value) return pre_routing_hook_response + def _forwardable_alias_marker_params( + self, model: str, strategy_tags: tuple[str, ...] + ) -> tuple[tuple[str, object], ...]: + marker_params: Final = tuple( + litellm_params + for idx in self.model_name_to_deployment_indices.get(model, ()) + if isinstance(litellm_params := self.model_list[idx].get("litellm_params", {}), dict) + and str(litellm_params.get("model", "")).startswith(AUTO_ROUTER_MODEL_PREFIX) + ) + tag_matched: Final = tuple( + params for params in marker_params if tuple(params.get("tags") or ()) == strategy_tags + ) + selected: Final = tag_matched[0] if tag_matched else (marker_params[0] if marker_params else None) + if selected is None: + return () + return tuple( + (key, value) + for key, value in selected.items() + if key not in _ALIAS_PARAMS_NEVER_FORWARDED + and key not in CustomPricingLiteLLMParams.model_fields + and value is not None + ) + def _consumed_request_tags_stamp( self, selected_strategy: "TaggedPreRoutingStrategy[PreRoutingStrategy]", diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index 40ed89ebfd8..1120323b4f9 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -584,8 +584,26 @@ async def get_deployments_for_tag( return healthy_deployments +def _tags_in_metadata(metadata: object) -> list[str]: + """ + Tags out of a metadata bucket the caller controls the shape of. + + A request can send its metadata (and its ``tags``) as anything the JSON body + allowed, an unparsed string or null included, so any shape that is not a list + of string tags carries no tags rather than raising. + """ + if not isinstance(metadata, Mapping): + return [] + typed_metadata: Final[Mapping[str, object]] = metadata + tags: Final = typed_metadata.get("tags") + if isinstance(tags, str) or not isinstance(tags, Sequence): + return [] + typed_tags: Final[Sequence[object]] = tags + return [tag for tag in typed_tags if isinstance(tag, str)] + + def _get_tags_from_request_kwargs( - request_kwargs: dict[Any, Any] | None = None, + request_kwargs: Mapping[Any, Any] | None = None, metadata_variable_name: Literal["metadata", "litellm_metadata"] | None = None, ) -> list[str]: """ @@ -604,12 +622,11 @@ def _get_tags_from_request_kwargs( return [] resolved_variable_name: Final = metadata_variable_name or get_metadata_variable_name_from_kwargs(request_kwargs) if resolved_variable_name in request_kwargs: - metadata: Final = request_kwargs[resolved_variable_name] or {} - tags = metadata.get("tags", []) - return tags if tags is not None else [] - elif "litellm_params" in request_kwargs: - litellm_params: Final = request_kwargs["litellm_params"] or {} - _metadata: Final = litellm_params.get(resolved_variable_name, {}) or {} - tags = _metadata.get("tags", []) - return tags if tags is not None else [] + return _tags_in_metadata(request_kwargs[resolved_variable_name]) + if "litellm_params" in request_kwargs: + litellm_params: Final = request_kwargs["litellm_params"] + if not isinstance(litellm_params, Mapping): + return [] + typed_litellm_params: Final[Mapping[str, object]] = litellm_params + return _tags_in_metadata(typed_litellm_params.get(resolved_variable_name)) return [] diff --git a/litellm/types/integrations/langfuse_otel.py b/litellm/types/integrations/langfuse_otel.py index 9ef48bdcdd0..c58dc567cda 100644 --- a/litellm/types/integrations/langfuse_otel.py +++ b/litellm/types/integrations/langfuse_otel.py @@ -16,12 +16,13 @@ class LangfuseOtelConfig(BaseModel): class LangfuseSpanAttributes(str, Enum): LANGFUSE_ENVIRONMENT = "langfuse.environment" + VERSION = "langfuse.version" + RELEASE = "langfuse.release" # ---- Generation-level metadata ---- GENERATION_NAME = "langfuse.generation.name" GENERATION_ID = "langfuse.generation.id" PARENT_OBSERVATION_ID = "langfuse.generation.parent_observation_id" - GENERATION_VERSION = "langfuse.generation.version" MASK_INPUT = "langfuse.generation.mask_input" MASK_OUTPUT = "langfuse.generation.mask_output" @@ -36,8 +37,6 @@ class LangfuseSpanAttributes(str, Enum): TRACE_NAME = "langfuse.trace.name" TRACE_ID = "langfuse.trace.id" TRACE_METADATA = "langfuse.trace.metadata" - TRACE_VERSION = "langfuse.trace.version" - TRACE_RELEASE = "langfuse.trace.release" EXISTING_TRACE_ID = "langfuse.trace.existing_id" UPDATE_TRACE_KEYS = "langfuse.trace.update_keys" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 00ba060e1e5..3b2cdcf5ff7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -26120,11 +26120,12 @@ "supports_vision": true }, "groq/llama-3.1-8b-instant": { + "deprecation_date": "2026-08-16", "input_cost_per_token": 5e-08, "litellm_provider": "groq", - "max_input_tokens": 128000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-08, "supports_function_calling": true, @@ -26132,9 +26133,10 @@ "supports_tool_choice": true }, "groq/llama-3.3-70b-versatile": { + "deprecation_date": "2026-08-16", "input_cost_per_token": 5.9e-07, "litellm_provider": "groq", - "max_input_tokens": 128000, + "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", @@ -26155,7 +26157,28 @@ "supports_response_schema": false, "supports_tool_choice": true }, + "groq/meta-llama/llama-prompt-guard-2-22m": { + "input_cost_per_token": 3e-08, + "litellm_provider": "groq", + "max_input_tokens": 512, + "max_output_tokens": 512, + "max_tokens": 512, + "mode": "chat", + "output_cost_per_token": 3e-08, + "source": "https://console.groq.com/docs/models" + }, + "groq/meta-llama/llama-prompt-guard-2-86m": { + "input_cost_per_token": 4e-08, + "litellm_provider": "groq", + "max_input_tokens": 512, + "max_output_tokens": 512, + "max_tokens": 512, + "mode": "chat", + "output_cost_per_token": 4e-08, + "source": "https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-86m" + }, "groq/meta-llama/llama-guard-4-12b": { + "deprecation_date": "2026-03-05", "input_cost_per_token": 2e-07, "litellm_provider": "groq", "max_input_tokens": 8192, @@ -26165,6 +26188,7 @@ "output_cost_per_token": 2e-07 }, "groq/meta-llama/llama-4-maverick-17b-128e-instruct": { + "deprecation_date": "2026-03-09", "input_cost_per_token": 2e-07, "litellm_provider": "groq", "max_input_tokens": 131072, @@ -26178,6 +26202,7 @@ "supports_vision": true }, "groq/meta-llama/llama-4-scout-17b-16e-instruct": { + "deprecation_date": "2026-07-17", "input_cost_per_token": 1.1e-07, "litellm_provider": "groq", "max_input_tokens": 131072, @@ -26191,6 +26216,7 @@ "supports_vision": true }, "groq/moonshotai/kimi-k2-instruct-0905": { + "deprecation_date": "2026-04-15", "input_cost_per_token": 1e-06, "output_cost_per_token": 3e-06, "cache_read_input_token_cost": 5e-07, @@ -26208,8 +26234,8 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "groq", "max_input_tokens": 131072, - "max_output_tokens": 32766, - "max_tokens": 32766, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 6e-07, "search_context_cost_per_query": { @@ -26229,8 +26255,8 @@ "input_cost_per_token": 7.5e-08, "litellm_provider": "groq", "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-07, "search_context_cost_per_query": { @@ -26265,7 +26291,26 @@ "supports_tool_choice": true, "supports_web_search": true }, + "groq/canopylabs/orpheus-v1-english": { + "input_cost_per_character": 2.2e-05, + "litellm_provider": "groq", + "max_input_tokens": 4000, + "max_output_tokens": 50000, + "max_tokens": 50000, + "mode": "audio_speech", + "source": "https://console.groq.com/docs/model/canopylabs/orpheus-v1-english" + }, + "groq/canopylabs/orpheus-arabic-saudi": { + "input_cost_per_character": 4e-05, + "litellm_provider": "groq", + "max_input_tokens": 4000, + "max_output_tokens": 50000, + "max_tokens": 50000, + "mode": "audio_speech", + "source": "https://console.groq.com/docs/models" + }, "groq/playai-tts": { + "deprecation_date": "2025-12-31", "input_cost_per_character": 5e-05, "litellm_provider": "groq", "max_input_tokens": 10000, @@ -26273,7 +26318,23 @@ "max_tokens": 10000, "mode": "audio_speech" }, + "groq/qwen/qwen3.6-27b": { + "input_cost_per_token": 6e-07, + "litellm_provider": "groq", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://console.groq.com/docs/model/qwen/qwen3.6-27b", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_tool_choice": true, + "supports_vision": true + }, "groq/qwen/qwen3-32b": { + "deprecation_date": "2026-07-17", "input_cost_per_token": 2.9e-07, "litellm_provider": "groq", "max_input_tokens": 131000, @@ -45826,11 +45887,15 @@ }, "bedrock_mantle/openai.gpt-5.6-sol": { "input_cost_per_token": 5.5e-06, + "input_cost_per_token_above_272k_tokens": 1.1e-05, "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "output_cost_per_token": 3.3e-05, + "output_cost_per_token_above_272k_tokens": 4.95e-05, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 272000, + "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -45854,11 +45919,15 @@ }, "bedrock_mantle/openai.gpt-5.6-terra": { "input_cost_per_token": 2.2e-06, + "input_cost_per_token_above_272k_tokens": 4.4e-06, "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5.5e-06, "cache_read_input_token_cost": 2.2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4.4e-07, "output_cost_per_token": 1.32e-05, + "output_cost_per_token_above_272k_tokens": 1.98e-05, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 272000, + "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -45882,11 +45951,15 @@ }, "bedrock_mantle/openai.gpt-5.6-luna": { "input_cost_per_token": 2.2e-07, + "input_cost_per_token_above_272k_tokens": 4.4e-07, "cache_creation_input_token_cost": 2.75e-07, + "cache_creation_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost": 2.2e-08, + "cache_read_input_token_cost_above_272k_tokens": 4.4e-08, "output_cost_per_token": 1.32e-06, + "output_cost_per_token_above_272k_tokens": 1.98e-06, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 272000, + "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py b/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py index f7c0b5452fe..e44c56e1fdf 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py @@ -1,5 +1,6 @@ """Per-request multi-tenant credential routing (V1 parity).""" +import base64 import os import sys @@ -42,6 +43,17 @@ def test_langfuse_dynamic_headers_need_both_keys(): assert headers is not None and "Authorization" in headers +def test_langfuse_dynamic_headers_carry_v4_ingestion_version(): + headers = dynamic_otlp_headers( + "langfuse_otel", {"langfuse_public_key": "pk", "langfuse_secret_key": "sk"} + ) + expected_auth = "Basic " + base64.b64encode(b"pk:sk").decode() + assert headers == { + "Authorization": expected_auth, + "x-langfuse-ingestion-version": "4", + } + + def test_weave_dynamic_headers(): headers = dynamic_otlp_headers( "weave_otel", {"wandb_api_key": "w", "weave_project_id": "p"} diff --git a/tests/test_litellm/integrations/test_langfuse_otel.py b/tests/test_litellm/integrations/test_langfuse_otel.py index 28f138c7acd..9392f974570 100644 --- a/tests/test_litellm/integrations/test_langfuse_otel.py +++ b/tests/test_litellm/integrations/test_langfuse_otel.py @@ -211,7 +211,7 @@ class TestLangfuseOtelIntegration: LangfuseSpanAttributes.GENERATION_NAME.value: "gen-name", LangfuseSpanAttributes.GENERATION_ID.value: "gen-id", LangfuseSpanAttributes.PARENT_OBSERVATION_ID.value: "parent-id", - LangfuseSpanAttributes.GENERATION_VERSION.value: "v1", + LangfuseSpanAttributes.VERSION.value: "t-ver", LangfuseSpanAttributes.MASK_INPUT.value: True, LangfuseSpanAttributes.MASK_OUTPUT.value: False, LangfuseSpanAttributes.TRACE_USER_ID.value: "user-123", @@ -221,8 +221,7 @@ class TestLangfuseOtelIntegration: LangfuseSpanAttributes.TRACE_NAME.value: "trace-name", LangfuseSpanAttributes.TRACE_ID.value: "traceid", # stripped dashes LangfuseSpanAttributes.TRACE_METADATA.value: json.dumps({"k": "v"}), - LangfuseSpanAttributes.TRACE_VERSION.value: "t-ver", - LangfuseSpanAttributes.TRACE_RELEASE.value: "rel-1", + LangfuseSpanAttributes.RELEASE.value: "rel-1", LangfuseSpanAttributes.EXISTING_TRACE_ID.value: "existing-id", LangfuseSpanAttributes.UPDATE_TRACE_KEYS.value: json.dumps( ["key1", "key2"] @@ -240,6 +239,52 @@ class TestLangfuseOtelIntegration: actual == expected ), "Mismatch between expected and actual OTEL attribute mapping." + @pytest.mark.parametrize( + "metadata, expected_version", + [ + ( + {"version": "v-observation", "trace_version": "v-trace"}, + "v-trace", + ), + ({"trace_version": "v-trace"}, "v-trace"), + ({"version": "v-observation"}, "v-observation"), + ({"version": "v-observation", "trace_version": ""}, ""), + ({}, None), + ], + ids=[ + "trace-version-wins-as-documented", + "trace-only", + "observation-version-is-the-fallback", + "empty-trace-version-is-not-absent", + "neither-key-emits-nothing", + ], + ) + def test_version_emitted_on_langfuse_v4_key(self, metadata, expected_version): + kwargs = {"litellm_params": {"metadata": {"trace_release": "rel-9", **metadata}}} + + with patch( + "litellm.integrations.arize._utils.safe_set_attribute" + ) as mock_safe_set_attribute: + LangfuseOtelLogger._set_langfuse_specific_attributes( + MagicMock(), kwargs, None + ) + + emitted = { + call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list + } + + if expected_version is None: + assert "langfuse.version" not in emitted + else: + assert emitted["langfuse.version"] == expected_version + assert emitted["langfuse.release"] == "rel-9" + for retired_key in ( + "langfuse.generation.version", + "langfuse.trace.version", + "langfuse.trace.release", + ): + assert retired_key not in emitted + def test_set_langfuse_specific_attributes_with_content(self): """Test that _set_langfuse_specific_attributes correctly sets observation.output with regular content response.""" from litellm.types.integrations.langfuse_otel import LangfuseSpanAttributes diff --git a/tests/test_litellm/interactions/test_google_interactions_integration.py b/tests/test_litellm/interactions/test_google_interactions_integration.py index 9c651cc94f5..41f0fa0d7fb 100644 --- a/tests/test_litellm/interactions/test_google_interactions_integration.py +++ b/tests/test_litellm/interactions/test_google_interactions_integration.py @@ -55,17 +55,10 @@ class TestGoogleInteractionsCreate: print(f"Usage: {response.usage}") def test_create_with_content_list(self, api_key): - """Test creating an interaction with a structured content list (Turn format).""" + """Test creating an interaction with a structured content list (Content[] input).""" response = interactions.create( model="gemini/gemini-2.5-flash", - input=[ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is the capital of France?"} - ], - } - ], + input=[{"type": "text", "text": "What is the capital of France?"}], api_key=api_key, ) @@ -169,25 +162,25 @@ class TestGoogleInteractionsStreaming: class TestGoogleInteractionsMultiTurn: - """Tests for multi-turn conversations using Turn[] input.""" + """Tests for multi-turn conversations using Step[] input.""" def test_multi_turn_conversation(self, api_key): - """Test a multi-turn conversation per OpenAPI spec (Turn[] format).""" + """Test a multi-turn conversation per OpenAPI spec (Step[] format).""" response = interactions.create( model="gemini/gemini-2.5-flash", input=[ { - "role": "user", + "type": "user_input", "content": [{"type": "text", "text": "My name is Alice."}], }, { - "role": "model", + "type": "model_output", "content": [ {"type": "text", "text": "Hello Alice! Nice to meet you."} ], }, { - "role": "user", + "type": "user_input", "content": [{"type": "text", "text": "What is my name?"}], }, ], diff --git a/tests/test_litellm/interactions/test_openapi_compliance.py b/tests/test_litellm/interactions/test_openapi_compliance.py index 1fe343ca6ee..2665f8703a6 100644 --- a/tests/test_litellm/interactions/test_openapi_compliance.py +++ b/tests/test_litellm/interactions/test_openapi_compliance.py @@ -167,17 +167,39 @@ class TestRequestCompliance: assert text_schema["properties"]["type"].get("const") == "text" print("✓ TextContent schema is correct") - def test_turn_schema(self, spec_dict): - """Verify Turn schema for multi-turn conversations.""" - turn_schema = spec_dict["components"]["schemas"]["Turn"] + def test_step_schema(self, spec_dict): + """Verify step-based multi-turn input. - assert "role" in turn_schema["properties"] - assert "content" in turn_schema["properties"] + Google replaced the role-carrying `Turn` schema with typed steps + (spec update of Aug 13, 2026): conversation history is now a `Step[]` + where `UserInputStep`/`ModelOutputStep` pin `type` values that our + transformations read to recover the role. Assert exactly what our code + depends on: `InteractionsInput` accepts a Step array, both step kinds + are part of the `Step` union, each pins its `type` const, and each + carries a `Content[]` content field. + """ + input_schema = spec_dict["components"]["schemas"]["InteractionsInput"] + step_array_items = [ + option["items"]["$ref"].split("/")[-1] + for option in input_schema["oneOf"] + if option.get("type") == "array" and "$ref" in option.get("items", {}) + ] + assert "Step" in step_array_items, f"InteractionsInput should accept Step[], got arrays of {step_array_items}" - # Content can be string or Content[] - content_prop = turn_schema["properties"]["content"] - assert "oneOf" in content_prop - print("✓ Turn schema supports role + content") + step_variants = { + option["$ref"].split("/")[-1] + for option in spec_dict["components"]["schemas"]["Step"]["oneOf"] + if "$ref" in option + } + assert {"UserInputStep", "ModelOutputStep"} <= step_variants, f"Step union is missing role steps: {step_variants}" + + for step_name, type_value in [("UserInputStep", "user_input"), ("ModelOutputStep", "model_output")]: + step_schema = spec_dict["components"]["schemas"][step_name] + assert step_schema["properties"]["type"].get("const") == type_value + assert "type" in step_schema["required"] + content_items = step_schema["properties"]["content"]["items"] + assert content_items["$ref"].split("/")[-1] == "Content" + print(f"✓ {step_name} pins type '{type_value}' with Content[] content") class TestResponseCompliance: diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 158cdb45f6b..5983607d708 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -485,6 +485,70 @@ def test_generic_cost_per_token_minimax_m3_above_512k_tokens(): assert round(completion_cost, 10) == round(expected_completion, 10) +@pytest.mark.parametrize( + "model", + [ + "bedrock_mantle/openai.gpt-5.6-sol", + "bedrock_mantle/openai.gpt-5.6-terra", + "bedrock_mantle/openai.gpt-5.6-luna", + ], +) +def test_generic_cost_per_token_bedrock_mantle_gpt56_long_context(model): + """Bedrock GPT-5.6 supports a 1M context window, billed at the long-context rates above 272K.""" + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model_cost_map = litellm.model_cost[model] + assert model_cost_map["max_input_tokens"] == 1000000 + + cached_tokens = 100000 + completion_tokens = 1000 + + short_prompt_tokens = 272000 + short_usage = Usage( + prompt_tokens=short_prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=short_prompt_tokens + completion_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), + ) + short_prompt_cost, short_completion_cost = generic_cost_per_token( + model=model, + usage=short_usage, + custom_llm_provider="bedrock_mantle", + ) + assert round(short_prompt_cost, 10) == round( + model_cost_map["input_cost_per_token"] * (short_prompt_tokens - cached_tokens) + + model_cost_map["cache_read_input_token_cost"] * cached_tokens, + 10, + ) + assert round(short_completion_cost, 10) == round( + model_cost_map["output_cost_per_token"] * completion_tokens, 10 + ) + + long_prompt_tokens = 900000 + long_usage = Usage( + prompt_tokens=long_prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=long_prompt_tokens + completion_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), + ) + long_prompt_cost, long_completion_cost = generic_cost_per_token( + model=model, + usage=long_usage, + custom_llm_provider="bedrock_mantle", + ) + assert round(long_prompt_cost, 10) == round( + model_cost_map["input_cost_per_token_above_272k_tokens"] + * (long_prompt_tokens - cached_tokens) + + model_cost_map["cache_read_input_token_cost_above_272k_tokens"] + * cached_tokens, + 10, + ) + assert round(long_completion_cost, 10) == round( + model_cost_map["output_cost_per_token_above_272k_tokens"] * completion_tokens, 10 + ) + + def test_generic_cost_per_token_honors_non_standard_above_threshold(): """Regression for #30344: get_model_info must keep arbitrary input/output_cost_per_token_above__tokens thresholds, not only the hard-coded diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index a47a56376a4..8281f3387d9 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -1526,7 +1526,11 @@ class TestBedrockMantleResponsesPricing: assert info["cache_creation_input_token_cost"] == pytest.approx(cache_creation_cost) assert info["cache_read_input_token_cost"] == pytest.approx(cache_read_cost) assert info["output_cost_per_token"] == pytest.approx(output_cost) - assert info["max_input_tokens"] == 272000 + assert info["max_input_tokens"] == 1000000 + assert info["input_cost_per_token_above_272k_tokens"] == pytest.approx(input_cost * 2) + assert info["cache_creation_input_token_cost_above_272k_tokens"] == pytest.approx(cache_creation_cost * 2) + assert info["cache_read_input_token_cost_above_272k_tokens"] == pytest.approx(cache_read_cost * 2) + assert info["output_cost_per_token_above_272k_tokens"] == pytest.approx(output_cost * 1.5) @pytest.mark.parametrize( "model, input_cost, output_cost", diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py index 40ca7e3a64e..ad41592db1e 100644 --- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py +++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py @@ -205,6 +205,50 @@ async def test_proxy_shutdown_event_prisma_disconnect_raises_error(monkeypatch): await proxy_shutdown_event() +# --------------------------------------------------------------------------- +# _flush_spend_logs_queue_on_shutdown +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_flush_spend_logs_queue_on_shutdown_drains_before_disconnect(monkeypatch): + fake_prisma = MagicMock() + monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False) + monkeypatch.setattr(ps, "db_writer_client", None, raising=False) + + drain = AsyncMock() + import litellm.proxy.utils as utils_mod + + monkeypatch.setattr(utils_mod, "drain_spend_logs_queue", drain) + + await ps._flush_spend_logs_queue_on_shutdown() + + observed = { + "drain_calls": drain.await_count, + "drain_prisma": drain.await_args.kwargs["prisma_client"] is fake_prisma, + } + assert observed == { + "drain_calls": 1, + "drain_prisma": True, + } + + +@pytest.mark.asyncio +async def test_flush_spend_logs_queue_on_shutdown_swallows_drain_errors(monkeypatch): + monkeypatch.setattr(ps, "prisma_client", MagicMock(), raising=False) + monkeypatch.setattr(ps, "db_writer_client", None, raising=False) + + import litellm.proxy.utils as utils_mod + + monkeypatch.setattr( + utils_mod, + "drain_spend_logs_queue", + AsyncMock(side_effect=RuntimeError("db gone")), + ) + + await ps._flush_spend_logs_queue_on_shutdown() + + # --------------------------------------------------------------------------- # _initialize_shared_aiohttp_session # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/conftest.py b/tests/test_litellm/proxy/utils/prisma_and_spend/conftest.py index 74c9abd9978..19abcb5d66d 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/conftest.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/conftest.py @@ -128,6 +128,7 @@ def mock_prisma_client() -> MagicMock: client.proxy_logging_obj.failure_handler = AsyncMock() client.spend_log_transactions = [] client._spend_log_transactions_lock = asyncio.Lock() + client.spend_logs_queue_monitor_task = None client.tool_usage_transactions = [] client._tool_usage_transactions_lock = asyncio.Lock() client.jsonify_object = lambda data: dict(data) diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py index d9eeb168611..54d59e690f9 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py @@ -17,8 +17,10 @@ from unittest.mock import AsyncMock, MagicMock import pytest from litellm.proxy.utils import ( + MAX_SPEND_LOG_DRAIN_ITERATIONS, _monitor_spend_logs_queue, _raise_failed_update_spend_exception, + drain_spend_logs_queue, update_daily_tag_spend, update_spend, update_spend_logs_job, @@ -263,6 +265,198 @@ async def test_update_spend_logs_job_processes_and_clears_queue( } +@pytest.mark.asyncio +async def test_update_spend_logs_job_requeues_popped_rows_when_write_cancelled( + mock_prisma_client: Any, make_spend_log_row: Any +) -> None: + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + mock_prisma_client.spend_log_transactions = [ + make_spend_log_row(request_id="r1"), + make_spend_log_row(request_id="r2"), + ] + + row_arriving_mid_flush = make_spend_log_row(request_id="r3") + + async def _cancel_mid_write(*args: Any, **kwargs: Any) -> None: + mock_prisma_client.spend_log_transactions.append(row_arriving_mid_flush) + raise asyncio.CancelledError() + + mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock( + side_effect=_cancel_mid_write + ) + + with pytest.raises(asyncio.CancelledError): + await update_spend_logs_job( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=proxy_logging, + ) + + assert [ + row["request_id"] for row in mock_prisma_client.spend_log_transactions + ] == ["r1", "r2", "r3"] + + +@pytest.mark.asyncio +async def test_update_spend_logs_job_does_not_requeue_when_cancelled_after_write( + mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + """Rows are already committed once guardrail tracking runs, so replaying + them would double-count the non-idempotent daily guardrail increments. + """ + import litellm.proxy.guardrails.usage_tracking as guard_mod + + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")] + mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock() + + monkeypatch.setattr( + guard_mod, + "process_spend_logs_guardrail_usage", + AsyncMock(side_effect=asyncio.CancelledError()), + raising=False, + ) + + with pytest.raises(asyncio.CancelledError): + await update_spend_logs_job( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=proxy_logging, + ) + + assert mock_prisma_client.spend_log_transactions == [] + + +@pytest.mark.asyncio +async def test_drain_spend_logs_queue_flushes_rows_queued_while_draining( + mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + import litellm.proxy.db.spend_log_tool_index as tool_mod + import litellm.proxy.guardrails.usage_tracking as guard_mod + + monkeypatch.setattr( + guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False + ) + monkeypatch.setattr( + tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False + ) + + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")] + + written: list[str] = [] + + async def _write(*args: Any, **kwargs: Any) -> None: + written.extend(row["request_id"] for row in kwargs["data"]) + if len(written) == 1: + mock_prisma_client.spend_log_transactions.append( + make_spend_log_row(request_id="r2") + ) + + mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_write) + + await drain_spend_logs_queue( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=proxy_logging, + ) + + assert written == ["r1", "r2"] + assert mock_prisma_client.spend_log_transactions == [] + + +@pytest.mark.asyncio +async def test_drain_spend_logs_queue_stops_monitor_and_keeps_its_popped_rows( + mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + import litellm.proxy.db.spend_log_tool_index as tool_mod + import litellm.proxy.guardrails.usage_tracking as guard_mod + + monkeypatch.setattr( + guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False + ) + monkeypatch.setattr( + tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False + ) + + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")] + + write_started = asyncio.Event() + written: list[str] = [] + write_calls = {"n": 0} + + async def _write(*args: Any, **kwargs: Any) -> None: + write_calls["n"] += 1 + if write_calls["n"] == 1: + write_started.set() + await asyncio.Event().wait() + written.extend(row["request_id"] for row in kwargs["data"]) + + mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_write) + + async def _monitor() -> None: + await update_spend_logs_job( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=proxy_logging, + ) + + mock_prisma_client.spend_logs_queue_monitor_task = asyncio.create_task(_monitor()) + await write_started.wait() + + await drain_spend_logs_queue( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=proxy_logging, + ) + + assert written == ["r1"] + assert mock_prisma_client.spend_log_transactions == [] + assert mock_prisma_client.spend_logs_queue_monitor_task is None + + +@pytest.mark.asyncio +async def test_drain_spend_logs_queue_gives_up_after_max_passes( + mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + import litellm.proxy.db.spend_log_tool_index as tool_mod + import litellm.proxy.guardrails.usage_tracking as guard_mod + + monkeypatch.setattr( + guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False + ) + monkeypatch.setattr( + tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False + ) + + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")] + + async def _write_and_refill(*args: Any, **kwargs: Any) -> None: + mock_prisma_client.spend_log_transactions.append(make_spend_log_row()) + + mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock( + side_effect=_write_and_refill + ) + + await drain_spend_logs_queue( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=proxy_logging, + ) + + assert ( + mock_prisma_client.db.litellm_spendlogs.create_many.await_count + == MAX_SPEND_LOG_DRAIN_ITERATIONS + ) + + @pytest.mark.asyncio async def test_monitor_spend_logs_queue_invokes_job_when_queue_nonempty( mock_prisma_client: Any, diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 7fc78aee619..3a69587fdd4 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -1176,6 +1176,41 @@ class TestPreRoutingStrategyRegistry: } assert router._select_pre_routing_strategy("smart", {}).strategy is cn + @staticmethod + def _router_with_plain_smart_deployment(enable_tag_filtering: bool) -> Router: + return Router( + model_list=[{"model_name": "smart", "litellm_params": {"model": "openai/gpt-4o-mini"}}], + enable_tag_filtering=enable_tag_filtering, + ) + + def test_select_falls_through_to_plain_deployments_when_no_tag_matches_under_tag_filtering(self): + router = self._router_with_plain_smart_deployment(enable_tag_filtering=True) + cn, us = object(), object() + + router.complexity_routers = {"smart": [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)]} + assert router._select_pre_routing_strategy("smart", {}) is None + assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["cn"]}}).strategy is cn + + router.complexity_routers = { + "smart": [ + TaggedPreRoutingStrategy(tags=("cn",), strategy=cn), + TaggedPreRoutingStrategy(tags=("us",), strategy=us), + ] + } + assert router._select_pre_routing_strategy("smart", {}) is None + assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["row"]}}) is None + assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["us"]}}).strategy is us + + router.complexity_routers["router-only"] = [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)] + assert router._select_pre_routing_strategy("router-only", {}).strategy is cn + + def test_select_keeps_capturing_when_tag_filtering_is_disabled(self): + router = self._router_with_plain_smart_deployment(enable_tag_filtering=False) + cn = object() + + router.complexity_routers = {"smart": [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)]} + assert router._select_pre_routing_strategy("smart", {}).strategy is cn + class TestAsyncPreRoutingHookMultiFormat: """Test async_pre_routing_hook with multiple input formats.""" @@ -2079,14 +2114,16 @@ class TestRouterPreRoutingAliasOverrides: assert field not in request_kwargs @pytest.mark.asyncio - async def test_alias_overrides_exclude_only_model(self): - """`model` (the alias marker, e.g. auto_router/complexity_router) is - excluded since it's never a real provider model. Router-only fields - like complexity_router_config DO flow through into request_kwargs at - this layer - they're filtered from the actual outbound LLM call - downstream by litellm.types.utils.all_litellm_params instead, not by - the router's pre-routing hook. See test_router_init_only_params_are_ - never_sent_to_a_provider for the guard on that downstream filter.""" + async def test_alias_overrides_exclude_only_marker_and_connection_params(self): + """`model` (the alias marker, e.g. auto_router/complexity_router) and + provider-connection params (api_base/api_key/api_version) are excluded + since they never describe the tier deployment actually called. + Router-only fields like complexity_router_config DO flow through into + request_kwargs at this layer - they're filtered from the actual + outbound LLM call downstream by litellm.types.utils.all_litellm_params + instead, not by the router's pre-routing hook. See + test_router_init_only_params_are_never_sent_to_a_provider for the + guard on that downstream filter.""" router = self._make_router() request_kwargs: Dict = {} @@ -2106,9 +2143,10 @@ class TestRouterPreRoutingAliasOverrides: assert request_kwargs["complexity_router_default_model"] == "gpt-4o" def test_router_init_only_params_are_never_sent_to_a_provider(self): - """The router's pre-routing hook only excludes `model` (see - test_alias_overrides_exclude_only_model above) - every other alias - litellm_param, including router-init-only fields like + """The router's pre-routing hook only excludes `model` and + provider-connection params (see test_alias_overrides_exclude_only_ + marker_and_connection_params above) - every other alias litellm_param, + including router-init-only fields like complexity_router_config, flows into request_kwargs unfiltered. That's only safe because litellm.completion()/acompletion() itself strips anything listed in all_litellm_params before building the provider @@ -2201,6 +2239,154 @@ class TestRouterPreRoutingAliasOverrides: assert request_kwargs["drop_params"] is True +class TestRouterPreRoutingSharedAliasName: + """ + Regression tests for https://github.com/BerriAI/litellm/issues/36619. + + A plain deployment and an `auto_router/` marker can share a `model_name`. + The alias-param forwarding after a pre-routing rewrite must read the + marker entry, never whichever same-name entry happens to sit first in + `model_list` - otherwise the plain entry's api_base/api_key get grafted + onto the routed tier's call (a Gemini path under api.openai.com, 404). + """ + + @staticmethod + def _plain_entry() -> dict: + return { + "model_name": "gpt4o", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "sk-plain-entry", + "api_base": "https://plain-entry.example/v1", + }, + } + + @staticmethod + def _marker_entry() -> dict: + return { + "model_name": "gpt4o", + "litellm_params": { + "model": "auto_router/complexity_router", + "drop_params": True, + "complexity_router_config": {"tiers": {"SIMPLE": "gemini-flash", "MEDIUM": "gemini-flash"}}, + "complexity_router_default_model": "gemini-flash", + }, + } + + @staticmethod + def _tier_entry() -> dict: + return { + "model_name": "gemini-flash", + "litellm_params": {"model": "gemini/gemini-3.6-flash", "api_key": "sk-tier"}, + } + + @pytest.mark.asyncio + @pytest.mark.parametrize("plain_entry_first", [True, False], ids=["plain_entry_first", "marker_entry_first"]) + async def test_marker_params_forwarded_regardless_of_model_list_order(self, plain_entry_first): + """In either config order the routed call gets the marker's own params + (drop_params) and never the plain sibling's api_base/api_key.""" + shared_name_entries = ( + [self._plain_entry(), self._marker_entry()] + if plain_entry_first + else [self._marker_entry(), self._plain_entry()] + ) + router = Router(model_list=[*shared_name_entries, self._tier_entry()]) + request_kwargs: Dict = {} + + result = await router.async_pre_routing_hook( + model="gpt4o", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": "What is the capital of France?"}], + ) + + assert result is not None + assert result.model == "gemini-flash" + assert "api_base" not in request_kwargs + assert "api_key" not in request_kwargs + assert request_kwargs["drop_params"] is True + + @pytest.mark.asyncio + async def test_connection_params_on_the_marker_itself_are_not_forwarded(self): + """Even when the marker entry carries api_base/api_key/api_version, + they describe no real deployment and must not reach the routed call, + while the marker's other params still do.""" + marker_with_connection_params = { + "model_name": "smart", + "litellm_params": { + **self._marker_entry()["litellm_params"], + "api_key": "sk-marker", + "api_base": "https://marker.example/v1", + "api_version": "2024-01-01", + }, + } + router = Router(model_list=[marker_with_connection_params, self._tier_entry()]) + request_kwargs: Dict = {} + + result = await router.async_pre_routing_hook( + model="smart", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": "hi"}], + ) + + assert result is not None + assert "api_base" not in request_kwargs + assert "api_key" not in request_kwargs + assert "api_version" not in request_kwargs + assert request_kwargs["drop_params"] is True + + @pytest.mark.asyncio + async def test_tag_scoped_markers_forward_the_selected_markers_params(self): + """With two tag-scoped markers under one name, the forwarded params + come from the marker whose tags matched the request, not from the + first marker in the list.""" + + def tagged_marker(routed_model: str, tags: list, drop_params: bool | None) -> dict: + return { + "model_name": "smart", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_default_model": routed_model, + "complexity_router_config": {"tiers": {"SIMPLE": [routed_model], "MEDIUM": [routed_model]}}, + "tags": tags, + **({"drop_params": drop_params} if drop_params is not None else {}), + }, + } + + router = Router( + model_list=[ + tagged_marker("gpt-cn", ["cn"], None), + tagged_marker("gpt-us", ["us"], True), + ] + ) + + us_kwargs: Dict = {"metadata": {"tags": ["us"]}} + us_result = await router.async_pre_routing_hook( + model="smart", + request_kwargs=us_kwargs, + messages=[{"role": "user", "content": "hi"}], + ) + assert us_result is not None and us_result.model == "gpt-us" + assert us_kwargs["drop_params"] is True + + cn_kwargs: Dict = {"metadata": {"tags": ["cn"]}} + cn_result = await router.async_pre_routing_hook( + model="smart", + request_kwargs=cn_kwargs, + messages=[{"role": "user", "content": "hi"}], + ) + assert cn_result is not None and cn_result.model == "gpt-cn" + assert "drop_params" not in cn_kwargs + + def test_forwardable_alias_marker_params_reads_the_marker_entry_only(self): + router = Router(model_list=[self._plain_entry(), self._marker_entry(), self._tier_entry()]) + + forwarded = dict(router._forwardable_alias_marker_params(model="gpt4o", strategy_tags=())) + + assert forwarded["drop_params"] is True + assert "api_key" not in forwarded and "api_base" not in forwarded + assert router._forwardable_alias_marker_params(model="gemini-flash", strategy_tags=()) == () + + class TestAdaptiveSoftFloors: def test_adaptive_defaults_use_cost_weighted_cold_policy(self): config = ComplexityRouterConfig( diff --git a/tests/test_litellm/router_strategy/test_router_tag_routing.py b/tests/test_litellm/router_strategy/test_router_tag_routing.py index 93011bb29cc..73491490b14 100644 --- a/tests/test_litellm/router_strategy/test_router_tag_routing.py +++ b/tests/test_litellm/router_strategy/test_router_tag_routing.py @@ -423,6 +423,33 @@ def test_get_tags_from_request_kwargs_various_inputs(): assert _get_tags_from_request_kwargs({"foo": "bar"}) == [] +@pytest.mark.parametrize( + "request_kwargs", + [ + {"metadata": "not-a-dict"}, + {"litellm_metadata": "not-a-dict"}, + {"litellm_metadata": ["not", "a", "dict"]}, + {"litellm_params": "not-a-dict"}, + {"litellm_params": {"metadata": "not-a-dict"}}, + {"metadata": {"tags": "free"}}, + {"metadata": {"tags": {"free": "paid"}}}, + ], +) +def test_get_tags_from_request_kwargs_reads_no_tags_from_a_non_dict_shape(request_kwargs): + """Metadata and `tags` are request-controlled, so a client can send either as a + string, a list or null. Every shape that cannot hold string tags reads as untagged + instead of raising, because callers run on the hot request path.""" + from litellm.router_strategy.tag_based_routing import _get_tags_from_request_kwargs + + assert _get_tags_from_request_kwargs(request_kwargs) == [] + + +def test_get_tags_from_request_kwargs_keeps_only_string_tags(): + from litellm.router_strategy.tag_based_routing import _get_tags_from_request_kwargs + + assert _get_tags_from_request_kwargs({"metadata": {"tags": ["free", 7, None, "paid"]}}) == ["free", "paid"] + + # --- _split_tags unit tests --- diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 0b47409eae1..bdbf33fb0e1 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7617,6 +7617,104 @@ class TestAutoRouterMaxInputCharsWiring: assert self._registered_auto_router(router).max_input_chars == DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS +class TestTaggedAutoRouterOnSharedModelName: + """A tagged auto-router marker sharing its model_name with a plain deployment must not + capture requests whose tags don't match it when tag filtering is enabled (#36620).""" + + class _FixedRouteLayer: + def __call__(self, text: str): + from semantic_router.schema import RouteChoice + + return RouteChoice(name="gemini-flash") + + @classmethod + def _router(cls, marker_tags, include_plain_sibling: bool, enable_tag_filtering: bool) -> "litellm.Router": + pytest.importorskip("semantic_router", reason="auto-router needs the semantic-router extra") + marker = { + "model_name": "gpt4o", + "litellm_params": { + "model": "auto_router/gpt4o-router", + "auto_router_config": json.dumps( + {"routes": [{"name": "gemini-flash", "utterances": ["capital city questions"]}]} + ), + "auto_router_default_model": "gemini-flash", + "auto_router_embedding_model": "text-embedding-3-small", + **({"tags": marker_tags} if marker_tags else {}), + }, + } + plain = {"model_name": "gpt4o", "litellm_params": {"model": "openai/gpt-4o"}} + tier = {"model_name": "gemini-flash", "litellm_params": {"model": "gemini/gemini-3.6-flash"}} + router = litellm.Router( + model_list=[plain, marker, tier] if include_plain_sibling else [marker, tier], + enable_tag_filtering=enable_tag_filtering, + ) + router.auto_routers["gpt4o"][0].strategy.routelayer = cls._FixedRouteLayer() + return router + + @staticmethod + async def _hook_response(router: "litellm.Router", request_kwargs: dict): + return await router.async_pre_routing_hook( + model="gpt4o", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": "What is the capital of France?"}], + ) + + @pytest.mark.asyncio + async def test_untagged_request_bypasses_the_tagged_marker_when_a_plain_deployment_shares_the_name(self): + router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True) + + assert await self._hook_response(router, {}) is None + + @pytest.mark.asyncio + async def test_request_tagged_for_the_marker_is_still_semantically_routed(self): + router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True) + + response = await self._hook_response(router, {"metadata": {"tags": ["route"]}}) + + assert response is not None + assert response.model == "gemini-flash" + + @pytest.mark.asyncio + async def test_marker_only_alias_still_captures_untagged_requests(self): + router = self._router(marker_tags=["route"], include_plain_sibling=False, enable_tag_filtering=True) + + response = await self._hook_response(router, {}) + + assert response is not None + assert response.model == "gemini-flash" + + @pytest.mark.asyncio + async def test_untagged_marker_sharing_the_name_still_captures_untagged_requests(self): + router = self._router(marker_tags=None, include_plain_sibling=True, enable_tag_filtering=True) + + response = await self._hook_response(router, {}) + + assert response is not None + assert response.model == "gemini-flash" + + @pytest.mark.asyncio + async def test_untagged_selection_never_lands_on_the_marker_deployment(self): + router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True) + + for _ in range(20): + deployment = await router.async_get_available_deployment( + model="gpt4o", + request_kwargs={}, + messages=[{"role": "user", "content": "What is the capital of France?"}], + ) + assert deployment["litellm_params"]["model"] == "openai/gpt-4o" + + def test_deployment_without_litellm_params_mapping_is_not_a_marker(self): + assert litellm.Router._is_strategy_marker_deployment({"model_name": "gpt4o"}) is False + + def test_model_name_has_plain_deployments_reflects_the_pool(self): + mixed = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True) + marker_only = self._router(marker_tags=["route"], include_plain_sibling=False, enable_tag_filtering=True) + + assert mixed._model_name_has_plain_deployments("gpt4o") is True + assert marker_only._model_name_has_plain_deployments("gpt4o") is False + + class TestGetAllowedFailsFromPolicy: def _make_router(self, **policy_kwargs) -> litellm.Router: from litellm.types.router import AllowedFailsPolicy diff --git a/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.test.tsx b/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.test.tsx new file mode 100644 index 00000000000..51eecc421a7 --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.test.tsx @@ -0,0 +1,28 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import RouterSettingsSummary from "./RouterSettingsSummary"; + +describe("RouterSettingsSummary", () => { + it("should list each configured fallback mapping", () => { + render( + , + ); + + expect(screen.getByText("gpt-4")).toBeInTheDocument(); + expect(screen.getByText("gpt-4o, claude-sonnet")).toBeInTheDocument(); + expect(screen.getByText("gpt-4o")).toBeInTheDocument(); + expect(screen.getByText("gpt-4o-mini")).toBeInTheDocument(); + expect(screen.getByText("Number of Retries: 3")).toBeInTheDocument(); + }); + + it("should show the empty state when every setting is null", () => { + render(); + + expect(screen.getByText("No router settings configured")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx b/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx new file mode 100644 index 00000000000..daaddc0cdfb --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx @@ -0,0 +1,56 @@ +import { Badge } from "@/components/ui/badge"; +import { hasRouterSettings } from "./routerSettingsPayload"; + +interface RouterSettingsSummaryProps { + routerSettings: Record | null | undefined; + emptyText?: string; +} + +const fallbackEntries = (fallbacks: unknown): Array<[string, string[]]> => { + if (!Array.isArray(fallbacks)) return []; + return fallbacks.flatMap((entry) => + entry && typeof entry === "object" ? (Object.entries(entry) as Array<[string, string[]]>) : [], + ); +}; + +export default function RouterSettingsSummary({ + routerSettings, + emptyText = "No router settings configured", +}: RouterSettingsSummaryProps) { + if (!hasRouterSettings(routerSettings)) { + return
{emptyText}
; + } + + const settings = routerSettings as Record; + const fallbacks = fallbackEntries(settings.fallbacks); + + return ( +
+ {settings.routing_strategy != null && ( +
+ Routing Strategy: {String(settings.routing_strategy)} +
+ )} + {settings.num_retries != null &&
Number of Retries: {String(settings.num_retries)}
} + {settings.allowed_fails != null &&
Allowed Failures: {String(settings.allowed_fails)}
} + {settings.cooldown_time != null &&
Cooldown Time: {String(settings.cooldown_time)}s
} + {settings.timeout != null &&
Timeout: {String(settings.timeout)}s
} + {settings.retry_after != null &&
Retry After: {String(settings.retry_after)}s
} + {Boolean(settings.enable_tag_filtering) &&
Tag Filtering: Enabled
} + {fallbacks.length > 0 && ( +
+
Fallbacks:
+
+ {fallbacks.map(([model, targets]) => ( +
+ {model} + -> + {Array.isArray(targets) ? targets.join(", ") : String(targets)} +
+ ))} +
+
+ )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts b/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts new file mode 100644 index 00000000000..542ed17d7a2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, it } from "vitest"; +import { hasRouterSettings, routerSettingsEditorValue, routerSettingsUpdate } from "./routerSettingsPayload"; + +describe("hasRouterSettings", () => { + it("should treat unset, empty and all-null settings as absent", () => { + expect(hasRouterSettings(undefined)).toBe(false); + expect(hasRouterSettings(null)).toBe(false); + expect(hasRouterSettings({})).toBe(false); + expect( + hasRouterSettings({ num_retries: null, fallbacks: [], model_group_alias: {}, enable_tag_filtering: false }), + ).toBe(false); + }); + + it("should detect configured settings", () => { + expect(hasRouterSettings({ num_retries: 3 })).toBe(true); + expect(hasRouterSettings({ fallbacks: [{ "gpt-4": ["gpt-4o"] }] })).toBe(true); + expect(hasRouterSettings({ enable_tag_filtering: true })).toBe(true); + expect(hasRouterSettings({ num_retries: 0 })).toBe(true); + }); +}); + +describe("routerSettingsEditorValue", () => { + it("should hand the editor only the fields it renders", () => { + expect( + routerSettingsEditorValue({ + num_retries: 2, + tag_routing_prefix: "team-", + fallbacks: [{ "gpt-4": ["gpt-4o"] }], + }), + ).toStrictEqual({ router_settings: { num_retries: 2, fallbacks: [{ "gpt-4": ["gpt-4o"] }] } }); + }); + + it("should keep an explicitly stored null so the editor renders it as empty", () => { + expect(routerSettingsEditorValue({ num_retries: null })).toStrictEqual({ router_settings: { num_retries: null } }); + }); + + it("should leave the editor uninitialised when the key has no stored settings", () => { + expect(routerSettingsEditorValue(null)).toBeUndefined(); + expect(routerSettingsEditorValue(undefined)).toBeUndefined(); + }); +}); + +describe("routerSettingsUpdate", () => { + const fallbacks = [{ "gpt-4": ["gpt-4o"] }]; + // Accepted by UpdateRouterConfig on /key/update but not rendered by the accordion. + const unsupported = { + tag_routing_prefix: "team-", + model_group_retry_policy: { "gpt-4": { TimeoutErrorRetries: 2 } }, + }; + + // The editor is a fixed-field form, so the merge has to hold two opposing guarantees at once: + // routing fields it cannot render survive, and a field it does render that the admin emptied + // is still sent as null. Orderings vary because an object spread resolves collisions by position. + const storedOrderings: Array<[string, Record]> = [ + ["unsupported fields first", { ...unsupported, num_retries: 2, fallbacks }], + ["unsupported fields last", { num_retries: 2, fallbacks, ...unsupported }], + [ + "unsupported fields interleaved", + { + tag_routing_prefix: unsupported.tag_routing_prefix, + num_retries: 2, + model_group_retry_policy: unsupported.model_group_retry_policy, + fallbacks, + }, + ], + ]; + + it.each(storedOrderings)( + "should keep unsupported stored fields and still clear an emptied editor field (%s)", + (_ordering, stored) => { + const result = routerSettingsUpdate({ num_retries: 4, fallbacks: null }, stored); + + expect(result).toMatchObject({ ...unsupported, num_retries: 4, fallbacks: null }); + }, + ); + + it("should send an empty object, not a null blob, when the last stored setting is cleared", () => { + expect(routerSettingsUpdate({ fallbacks: null, num_retries: null }, { fallbacks, num_retries: 2 })).toEqual({}); + }); + + it("should keep clearing owned fields explicitly while an unsupported setting still stands", () => { + expect(routerSettingsUpdate({ fallbacks: null, num_retries: null }, { fallbacks, ...unsupported })).toMatchObject({ + ...unsupported, + fallbacks: null, + num_retries: null, + }); + }); + + it("should null every editor-owned field the editor left out", () => { + expect(routerSettingsUpdate({ fallbacks: null }, { timeout: 30, ...unsupported })).toEqual({ + ...unsupported, + routing_strategy: null, + allowed_fails: null, + cooldown_time: null, + num_retries: null, + timeout: null, + retry_after: null, + fallbacks: null, + context_window_fallbacks: null, + retry_policy: null, + model_group_alias: null, + enable_tag_filtering: null, + routing_strategy_args: null, + }); + }); + + it("should send the edited settings when the user configured something", () => { + expect(routerSettingsUpdate({ fallbacks }, null)).toMatchObject({ fallbacks }); + }); + + it("should leave the field off when nothing is stored and nothing was configured", () => { + expect(routerSettingsUpdate({ fallbacks: null, num_retries: null }, {})).toBeUndefined(); + expect(routerSettingsUpdate(undefined, { fallbacks })).toBeUndefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts b/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts new file mode 100644 index 00000000000..1d048425a6d --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts @@ -0,0 +1,63 @@ +import { RouterSettingsAccordionValue } from "./RouterSettingsAccordion"; + +export type RouterSettings = RouterSettingsAccordionValue["router_settings"]; + +const EDITOR_OWNED_FIELDS: Record = { + routing_strategy: true, + allowed_fails: true, + cooldown_time: true, + num_retries: true, + timeout: true, + retry_after: true, + fallbacks: true, + context_window_fallbacks: true, + retry_policy: true, + model_group_alias: true, + enable_tag_filtering: true, + routing_strategy_args: true, +}; + +const EDITOR_OWNED_KEYS = Object.keys(EDITOR_OWNED_FIELDS) as Array; + +const isMeaningfulRouterSetting = (value: unknown): boolean => { + if (value === null || value === undefined || value === "" || value === false) return false; + if (Array.isArray(value)) return value.length > 0; + if (typeof value === "object") return Object.keys(value).length > 0; + return true; +}; + +export const hasRouterSettings = (settings: Record | null | undefined): boolean => + settings != null && Object.values(settings).some(isMeaningfulRouterSetting); + +/** + * The stored settings narrowed to what the editor renders. The stored blob is untyped JSON + * from /key/info, so this projection is the one place it is read as RouterSettings. + */ +export const routerSettingsEditorValue = ( + stored: Record | null | undefined, +): RouterSettingsAccordionValue | undefined => + stored + ? { + router_settings: Object.fromEntries( + EDITOR_OWNED_KEYS.filter((key) => key in stored).map((key) => [key, stored[key]]), + ) as RouterSettings, + } + : undefined; + +/** + * Router settings to put on a /key/update payload, or undefined to leave the field off. + * The editor only renders EDITOR_OWNED_FIELDS, so its value is merged over the stored object + * rather than replacing it, and routing fields the editor cannot show survive an unrelated edit. + * Emptying every field sends {}, which the proxy reads as "no key-level override" so the key + * falls back to its team and global settings, where an all-null blob would pin it to nulls. + */ +export const routerSettingsUpdate = ( + edited: RouterSettings | null | undefined, + stored: Record | null | undefined, +): Record | undefined => { + if (!edited) return undefined; + const editorOwned = Object.fromEntries(EDITOR_OWNED_KEYS.map((key) => [key, edited[key] ?? null])); + const merged: Record = { ...stored, ...editorOwned }; + if (hasRouterSettings(merged)) return merged; + return hasRouterSettings(stored) ? {} : undefined; +}; diff --git a/ui/litellm-dashboard/src/components/common_components/routerSettingsWiring.test.tsx b/ui/litellm-dashboard/src/components/common_components/routerSettingsWiring.test.tsx new file mode 100644 index 00000000000..d6ed0ae07db --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/routerSettingsWiring.test.tsx @@ -0,0 +1,79 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render, screen, waitFor } from "@testing-library/react"; +import type { ReactElement, ReactNode } from "react"; +import { describe, expect, it, vi } from "vitest"; +import type { FallbackGroup } from "../Settings/RouterSettings/Fallbacks/FallbackGroupConfig"; +import type { RouterSettingsFormValue } from "../router_settings/RouterSettingsForm"; +import RouterSettingsAccordion from "./RouterSettingsAccordion"; +import { routerSettingsEditorValue } from "./routerSettingsPayload"; + +vi.mock("../networking", () => ({ + getRouterSettingsCall: vi.fn().mockResolvedValue({}), +})); + +vi.mock("@/components/llm_calls/fetch_models", () => ({ + fetchAvailableModels: vi.fn().mockResolvedValue([{ model_group: "gpt-5.5" }, { model_group: "gpt-4o-mini" }]), + fetchAvailableModelsForTeam: vi.fn().mockResolvedValue([]), +})); + +vi.mock("@tremor/react", () => ({ + TabGroup: ({ children }: { children: ReactNode }) =>
{children}
, + TabList: ({ children }: { children: ReactNode }) =>
{children}
, + Tab: ({ children }: { children: ReactNode }) =>
{children}
, + TabPanels: ({ children }: { children: ReactNode }) =>
{children}
, + TabPanel: ({ children }: { children: ReactNode }) =>
{children}
, +})); + +vi.mock("../router_settings/RouterSettingsForm", () => ({ + default: ({ value }: { value: RouterSettingsFormValue }) => ( +
{JSON.stringify(value.routerSettings)}
+ ), +})); + +vi.mock("../Settings/RouterSettings/Fallbacks/FallbackSelectionForm", () => ({ + FallbackSelectionForm: ({ groups }: { groups: FallbackGroup[] }) => ( +
+ {groups.map((g) => `${g.primaryModel ?? "none"}->${g.fallbackModels.join("|") || "none"}`).join(" ")} +
+ ), +})); + +// Captured verbatim from GET /key/info on a live proxy for a key created through the UI. +// tag_routing_prefix is stored on the key and accepted by /key/update, but the accordion +// has no control for it, so the projection must drop it without losing the rest. +const KEY_INFO_ROUTER_SETTINGS: Record = { + fallbacks: [{ "gpt-5.5": ["gpt-4o-mini"] }], + num_retries: 3, + tag_routing_prefix: "team-", +}; + +const renderAccordion = (stored: Record | null): ReactElement => ( + + + +); + +describe("key router settings wiring, /key/info payload through to rendered output", () => { + it("should prefill both tabs from the stored settings", async () => { + render(renderAccordion(KEY_INFO_ROUTER_SETTINGS)); + + await waitFor(() => { + expect(screen.getByTestId("fallbacks")).toHaveTextContent("gpt-5.5->gpt-4o-mini"); + }); + expect(JSON.parse(screen.getByTestId("loadbalancing").textContent ?? "{}")).toMatchObject({ num_retries: 3 }); + }); + + it("should not leak a field the accordion has no control for into the editor", async () => { + render(renderAccordion(KEY_INFO_ROUTER_SETTINGS)); + + await waitFor(() => expect(screen.getByTestId("loadbalancing")).toBeInTheDocument()); + expect(screen.getByTestId("loadbalancing").textContent).not.toContain("tag_routing_prefix"); + }); + + it("should render an empty editor for a key holding only fields it cannot show", async () => { + render(renderAccordion({ tag_routing_prefix: "team-" })); + + await waitFor(() => expect(screen.getByTestId("fallbacks")).toHaveTextContent("none->none")); + expect(JSON.parse(screen.getByTestId("loadbalancing").textContent ?? "null")).toEqual({}); + }); +}); diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx index 920d1f4af5a..caa5e0a38f0 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx @@ -95,6 +95,7 @@ export interface KeyResponse { object_permission?: ObjectPermission | null; access_group_ids?: string[]; budget_fallbacks?: Record; + router_settings?: Record | null; budget_limits?: Array<{ budget_duration: string; max_budget: number; reset_at?: string }>; auto_rotate?: boolean; rotation_interval?: string; diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx index fb8ab87e45f..caad6fc30fc 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx @@ -59,6 +59,24 @@ vi.mock("../organisms/create_key_button", () => ({ fetchTeamModels: vi.fn().mockResolvedValue(["team-model-1", "team-model-2"]), })); +const routerSettingsMocks = vi.hoisted(() => ({ + receivedValue: undefined as { router_settings: Record } | undefined, + editedValue: null as Record | null, +})); + +vi.mock("../common_components/RouterSettingsAccordion", async () => { + const { forwardRef, useImperativeHandle } = await import("react"); + return { + default: forwardRef(({ value }: { value?: { router_settings: Record } }, ref) => { + routerSettingsMocks.receivedValue = value; + useImperativeHandle(ref, () => ({ + getValue: () => ({ router_settings: routerSettingsMocks.editedValue ?? value?.router_settings ?? {} }), + })); + return
; + }), + }; +}); + vi.mock("@/app/(dashboard)/hooks/organizations/useOrganizations", () => ({ useOrganizations: vi.fn().mockReturnValue({ data: [ @@ -160,6 +178,83 @@ describe("KeyEditView", () => { last_rotation_at: undefined, key_rotation_at: undefined, }; + describe("router settings", () => { + const UNSUPPORTED_STORED_FIELD = { tag_routing_prefix: "team-" }; + const STORED_ROUTER_SETTINGS = { + num_retries: 2, + fallbacks: [{ "gpt-4": ["gpt-4o"] }], + ...UNSUPPORTED_STORED_FIELD, + }; + + const renderWithRouterSettings = (onSubmit: (values: Record) => Promise) => + renderWithProviders( + {}} + onSubmit={onSubmit} + accessToken="test-token" + userID="test-user" + userRole="proxy_admin" + premiumUser={true} + />, + ); + + beforeEach(() => { + routerSettingsMocks.receivedValue = undefined; + routerSettingsMocks.editedValue = null; + }); + + it("should load the fields it renders into the editor and withhold the ones it does not", async () => { + renderWithRouterSettings(async () => {}); + + await waitFor(() => { + expect(routerSettingsMocks.receivedValue).toStrictEqual({ + router_settings: { num_retries: 2, fallbacks: [{ "gpt-4": ["gpt-4o"] }] }, + }); + }); + }); + + it("should submit edited fallbacks alongside routing fields the editor cannot show", async () => { + const onSubmit = vi.fn().mockResolvedValue(undefined); + renderWithRouterSettings(onSubmit); + routerSettingsMocks.editedValue = { num_retries: 2, fallbacks: [{ "gpt-4": ["gpt-4o", "gpt-4o-mini"] }] }; + + fireEvent.click(screen.getByText("Save Changes")); + + await waitFor(() => { + expect(onSubmit).toHaveBeenCalledWith( + expect.objectContaining({ + router_settings: expect.objectContaining({ + ...UNSUPPORTED_STORED_FIELD, + num_retries: 2, + fallbacks: [{ "gpt-4": ["gpt-4o", "gpt-4o-mini"] }], + }), + }), + ); + }); + }); + + it("should submit cleared router settings so removing every fallback is persisted", async () => { + const onSubmit = vi.fn().mockResolvedValue(undefined); + renderWithRouterSettings(onSubmit); + routerSettingsMocks.editedValue = { num_retries: null, fallbacks: null }; + + fireEvent.click(screen.getByText("Save Changes")); + + await waitFor(() => { + expect(onSubmit).toHaveBeenCalledWith( + expect.objectContaining({ + router_settings: expect.objectContaining({ + ...UNSUPPORTED_STORED_FIELD, + num_retries: null, + fallbacks: null, + }), + }), + ); + }); + }); + }); + it("should render", async () => { const { getByText } = renderWithProviders( >( keyData.budget_fallbacks && typeof keyData.budget_fallbacks === "object" ? keyData.budget_fallbacks : {}, ); + const routerSettingsRef = useRef(null); const { data: organizations, isLoading: isOrganizationsLoading } = useOrganizations(); const { data: projects } = useProjects(); const { data: uiSettingsData } = useUISettings(); @@ -286,6 +289,14 @@ export function KeyEditView({ values.budget_fallbacks = {}; } + const routerSettings = routerSettingsUpdate( + routerSettingsRef.current?.getValue()?.router_settings, + keyData.router_settings, + ); + if (routerSettings) { + values.router_settings = routerSettings; + } + await onSubmit(withNormalizedEstimates(values)); } finally { setIsKeySaving(false); @@ -799,6 +810,15 @@ export function KeyEditView({ )} + + + + { }); }); + it("should render the key's saved router fallbacks", async () => { + vi.mocked(useAuthorized).mockReturnValue(baseUseAuthorizedMock); + + renderWithProviders( + {}} + keyId={"test-key-id"} + onKeyDataUpdate={() => {}} + teams={[]} + />, + ); + + expect(await screen.findByText("Router Settings")).toBeInTheDocument(); + expect(screen.getByText("gpt-4")).toBeInTheDocument(); + expect(screen.getByText("gpt-4o")).toBeInTheDocument(); + expect(screen.getByText("Number of Retries: 2")).toBeInTheDocument(); + }); + it("should render tags", async () => { vi.mocked(useAuthorized).mockReturnValue(baseUseAuthorizedMock); diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index 6fd6547a995..f7720874968 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -13,6 +13,8 @@ import { isProxyAdminRole, isUserTeamAdminForSingleTeam, rolesWithWriteAccess } import { mapDisplayToInternalNames, mapInternalToDisplayNames } from "../callback_info_helpers"; import AutoRotationView from "../common_components/AutoRotationView"; import DeleteResourceModal from "../common_components/DeleteResourceModal"; +import RouterSettingsSummary from "../common_components/RouterSettingsSummary"; +import { hasRouterSettings } from "../common_components/routerSettingsPayload"; import { extractLoggingSettings, formatMetadataForDisplay, stripTagsFromMetadata } from "../key_info_utils"; import { KeyResponse } from "../key_team_helpers/key_list"; import LoggingSettingsView from "../logging_settings_view"; @@ -837,6 +839,15 @@ export default function KeyInfoView({
)} + {hasRouterSettings(currentKeyData.router_settings) && ( +
+ Router Settings +
+ +
+
+ )} +
Tags