mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-06 08:16:43 +00:00
Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_bump_tornado_658
This commit is contained in:
mateo-berri
commit
f7b37e4e6e
35 changed files with 3062 additions and 236 deletions
|
|
@ -18,7 +18,7 @@ type: application
|
|||
# This is the chart version. This version number should be incremented each time you make changes
|
||||
# to the chart and its templates, including the app version.
|
||||
# Versions are expected to follow Semantic Versioning (https://semver.org/)
|
||||
version: 1.1.2
|
||||
version: 1.1.3
|
||||
|
||||
# This is the version number of the application being deployed. This version number should be
|
||||
# incremented each time you make changes to the application. Versions are not expected to
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
suite: "hpa with behavior"
|
||||
suite: "hpa"
|
||||
templates:
|
||||
- hpa.yaml
|
||||
tests:
|
||||
|
|
@ -23,14 +23,44 @@ tests:
|
|||
- equal: { path: spec.behavior.scaleUp.stabilizationWindowSeconds, value: 60 }
|
||||
- equal: { path: spec.behavior.scaleDown.stabilizationWindowSeconds, value: 90 }
|
||||
|
||||
---
|
||||
suite: "hpa without behavior"
|
||||
templates:
|
||||
- hpa.yaml
|
||||
tests:
|
||||
- it: "does not render behavior when not set"
|
||||
set:
|
||||
autoscaling.enabled: true
|
||||
asserts:
|
||||
- isKind: { of: HorizontalPodAutoscaler }
|
||||
- isNull: { path: spec.behavior }
|
||||
|
||||
- it: "scales on cpu at the documented 60 percent by default"
|
||||
set:
|
||||
autoscaling.enabled: true
|
||||
asserts:
|
||||
- isKind: { of: HorizontalPodAutoscaler }
|
||||
- equal: { path: "spec.metrics[0].resource.name", value: cpu }
|
||||
- equal: { path: "spec.metrics[0].resource.target.type", value: Utilization }
|
||||
- equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 60 }
|
||||
|
||||
- it: "does not scale on memory by default"
|
||||
set:
|
||||
autoscaling.enabled: true
|
||||
asserts:
|
||||
- lengthEqual: { path: spec.metrics, count: 1 }
|
||||
|
||||
- it: "honours an explicit cpu target override"
|
||||
set:
|
||||
autoscaling.enabled: true
|
||||
autoscaling.targetCPUUtilizationPercentage: 75
|
||||
asserts:
|
||||
- equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 75 }
|
||||
|
||||
- it: "renders a memory metric only when a memory target is set"
|
||||
set:
|
||||
autoscaling.enabled: true
|
||||
autoscaling.targetMemoryUtilizationPercentage: 80
|
||||
asserts:
|
||||
- lengthEqual: { path: spec.metrics, count: 2 }
|
||||
- equal: { path: "spec.metrics[1].resource.name", value: memory }
|
||||
- equal: { path: "spec.metrics[1].resource.target.averageUtilization", value: 80 }
|
||||
|
||||
- it: "renders no hpa when autoscaling is disabled"
|
||||
asserts:
|
||||
- hasDocuments: { count: 0 }
|
||||
|
|
|
|||
|
|
@ -200,7 +200,16 @@ autoscaling:
|
|||
enabled: false
|
||||
minReplicas: 1
|
||||
maxReplicas: 100
|
||||
targetCPUUtilizationPercentage: 80
|
||||
# 60 is the documented recommendation. See "Recommended Machine Specifications"
|
||||
# in https://docs.litellm.ai/docs/proxy/prod. A new replica clears the startupProbe
|
||||
# above only after up to failureThreshold x periodSeconds = 300 seconds, so a target
|
||||
# high enough to trip near saturation adds capacity minutes after it was needed.
|
||||
targetCPUUtilizationPercentage: 60
|
||||
# Deliberately left unset rather than given a value. The prisma query engine's
|
||||
# resident memory is a high-water mark that ratchets to the pod's worst-ever write
|
||||
# and is never returned, so a memory target reads the largest write a pod ever did
|
||||
# rather than what it is doing now, and replicas ratchet up without scaling back in.
|
||||
# Memory is a floor to provision under 'resources', not a signal to scale on.
|
||||
# targetMemoryUtilizationPercentage: 80
|
||||
# behavior: {}
|
||||
|
||||
|
|
|
|||
|
|
@ -36,6 +36,8 @@ from litellm.types.utils import (
|
|||
from litellm.utils import print_verbose, token_counter
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from openai.types.completion_usage import CompletionUsage
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.types.litellm_core_utils.streaming_chunk_builder_utils import (
|
||||
UsagePerChunk,
|
||||
|
|
@ -794,7 +796,7 @@ class ChunkProcessor:
|
|||
|
||||
@staticmethod
|
||||
def _extract_usage_chunk(chunk: "_UsageBearingChunk | ModelResponse | ModelResponseStream") -> Usage | None:
|
||||
usage_chunk: Usage | None = None
|
||||
usage_chunk: Usage | CompletionUsage | None = None
|
||||
if hasattr(chunk, "usage") and chunk.usage is not None:
|
||||
usage_chunk = chunk.usage
|
||||
elif "usage" in chunk:
|
||||
|
|
@ -806,7 +808,9 @@ class ChunkProcessor:
|
|||
|
||||
if isinstance(usage_chunk, dict):
|
||||
return Usage(**usage_chunk)
|
||||
return usage_chunk
|
||||
if usage_chunk is None or isinstance(usage_chunk, Usage):
|
||||
return usage_chunk
|
||||
return Usage(**usage_chunk.model_dump())
|
||||
|
||||
def _calculate_usage_per_chunk(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -1378,31 +1378,38 @@ def process_anthropic_headers(headers: httpx.Headers | dict) -> dict:
|
|||
return additional_headers
|
||||
|
||||
|
||||
def _anthropic_model_entry(model: ModelInfoResponse, created_at: str) -> Mapping[str, object]:
|
||||
def _anthropic_model_entry(
|
||||
model: ModelInfoResponse, created_at: str, display_names: Mapping[str, str]
|
||||
) -> Mapping[str, object]:
|
||||
return { # mutable-ok: JSON response body, serialized by the route and never mutated
|
||||
"type": "model",
|
||||
"id": model["id"],
|
||||
"display_name": model["id"],
|
||||
"display_name": display_names.get(model["id"], model["id"]),
|
||||
"created_at": created_at,
|
||||
"max_input_tokens": model.get("max_input_tokens"),
|
||||
"max_tokens": model.get("max_output_tokens"),
|
||||
}
|
||||
|
||||
|
||||
def create_anthropic_model_list_response(models: Sequence[ModelInfoResponse]) -> Mapping[str, object]:
|
||||
def create_anthropic_model_list_response(
|
||||
models: Sequence[ModelInfoResponse],
|
||||
display_names: Mapping[str, str] = MappingProxyType({}),
|
||||
) -> Mapping[str, object]:
|
||||
"""Build the Anthropic-native /v1/models envelope.
|
||||
|
||||
Clients that send an anthropic-version header parse the Anthropic Models API
|
||||
shape (type/display_name/created_at plus has_more/first_id/last_id) and filter
|
||||
the list themselves, so every model is returned here. The token limits carry
|
||||
over from the OpenAI-shaped listing, named as the Messages API names them, and
|
||||
are always present because the vendor shape declares them nullable, not optional
|
||||
are always present because the vendor shape declares them nullable, not optional.
|
||||
display_names maps a listed model id to a configured human-readable name; ids
|
||||
without an entry fall back to the id itself, matching the vendor behavior
|
||||
"""
|
||||
created_at: Final = (
|
||||
datetime.fromtimestamp(DEFAULT_MODEL_CREATED_AT_TIME, tz=timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
)
|
||||
data: Final = [ # mutable-ok: JSON response body, serialized by the route and never mutated
|
||||
_anthropic_model_entry(model, created_at) for model in models
|
||||
_anthropic_model_entry(model, created_at, display_names) for model in models
|
||||
]
|
||||
return { # mutable-ok: JSON response body, serialized by the route and never mutated
|
||||
"data": data,
|
||||
|
|
|
|||
|
|
@ -949,7 +949,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
# For Gemini 3+ models, use thinkingLevel instead of thinkingBudget
|
||||
if model and VertexGeminiConfig._is_gemini_3_or_newer(model):
|
||||
if thinking_enabled:
|
||||
if thinking_budget is None or thinking_budget == 0:
|
||||
if thinking_budget == 0:
|
||||
params["includeThoughts"] = False
|
||||
else:
|
||||
params["includeThoughts"] = True
|
||||
|
|
|
|||
|
|
@ -177,8 +177,9 @@ class VertexAIDeepSeekOCRConfig(BaseOCRConfig):
|
|||
content_item = {"type": "image_url", "image_url": document_url}
|
||||
|
||||
# Build DeepSeek OCR request
|
||||
provider_model: Final = model if model.startswith("deepseek-ai/") else f"deepseek-ai/{model}"
|
||||
data: Final = {
|
||||
"model": "deepseek-ai/" + model,
|
||||
"model": provider_model,
|
||||
"messages": [{"role": "user", "content": [content_item]}],
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8637,6 +8637,16 @@ def _set_stream_builder_response_cost(response: ModelResponse, logging_obj: Opti
|
|||
hidden_params["response_cost"] = response_cost
|
||||
|
||||
|
||||
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
|
||||
if logging_obj is None:
|
||||
return
|
||||
if isinstance(getattr(usage, "cost", None), (int, float)):
|
||||
return
|
||||
computed_cost: Final = logging_obj._response_cost_calculator(result=response)
|
||||
if isinstance(computed_cost, (int, float)) and computed_cost > 0:
|
||||
setattr(usage, "cost", computed_cost)
|
||||
|
||||
|
||||
def stream_chunk_builder(
|
||||
chunks: list,
|
||||
messages: list | None = None,
|
||||
|
|
@ -8731,12 +8741,7 @@ def stream_chunk_builder(
|
|||
)
|
||||
break
|
||||
|
||||
if litellm.include_cost_in_streaming_usage and logging_obj is not None:
|
||||
setattr(
|
||||
usage,
|
||||
"cost",
|
||||
logging_obj._response_cost_calculator(result=response),
|
||||
)
|
||||
_stamp_streaming_usage_cost(usage, response, logging_obj)
|
||||
_set_stream_builder_response_cost(response, logging_obj)
|
||||
|
||||
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
|
||||
|
|
@ -8915,10 +8920,7 @@ def stream_chunk_builder(
|
|||
)
|
||||
break
|
||||
|
||||
# Add cost to usage object if include_cost_in_streaming_usage is True
|
||||
if litellm.include_cost_in_streaming_usage and logging_obj is not None:
|
||||
setattr(usage, "cost", logging_obj._response_cost_calculator(result=response))
|
||||
|
||||
_stamp_streaming_usage_cost(usage, response, logging_obj)
|
||||
_set_stream_builder_response_cost(response, logging_obj)
|
||||
|
||||
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -10,13 +10,36 @@ legacy internal names with `general_settings.use_team_public_model_name: false`.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, cast
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
||||
|
||||
def configured_display_names(
|
||||
entries: Sequence[tuple[str, str]],
|
||||
llm_router: Router | None,
|
||||
) -> Mapping[str, str]:
|
||||
"""response_id -> configured `model_info.display_name` for the listing entries
|
||||
that have one.
|
||||
|
||||
Metadata is looked up by each entry's internal lookup id (so team-scoped rows
|
||||
resolve), while the returned map is keyed by the public response id the
|
||||
Anthropic-shaped listing is built from. Entries without a configured name are
|
||||
omitted so the listing falls back to the id itself.
|
||||
"""
|
||||
if llm_router is None:
|
||||
return MappingProxyType({})
|
||||
resolved: Final = (
|
||||
(response_id, llm_router.get_configured_display_name(lookup_id)) for response_id, lookup_id in entries
|
||||
)
|
||||
return MappingProxyType(
|
||||
{response_id: display_name for response_id, display_name in resolved if display_name is not None}
|
||||
)
|
||||
|
||||
|
||||
class TeamModelNameTranslator:
|
||||
"""Translates internal team routing keys to their public names for the model
|
||||
listing/retrieve responses. Stateless; the live router and general_settings
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ from litellm.proxy._types import (
|
|||
from litellm.proxy.auth.auth_checks import (
|
||||
_delete_cache_access_object, # pyright: ignore[reportPrivateUsage] # the access-group endpoints reach for this same cache primitive
|
||||
)
|
||||
from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
|
||||
from litellm.repositories.table_repositories import AccessGroupRepository
|
||||
|
||||
|
||||
|
|
@ -72,8 +73,9 @@ _REPOINT_KEY_SQL: Final = (
|
|||
|
||||
|
||||
def _raw_executor(prisma_client: object) -> _RawExecutor:
|
||||
"""Narrow the untyped Prisma client down to the raw-query call this module makes."""
|
||||
return AccessGroupRepository(prisma_client).prisma_client.db # pyright: ignore[reportAny] # untyped Prisma client
|
||||
"""Narrow the untyped Prisma client down to the raw-query call this module makes, pinned to the writer."""
|
||||
db: Final = AccessGroupRepository(prisma_client).prisma_client.db # pyright: ignore[reportAny] # untyped Prisma client
|
||||
return WriterPinnedClient(db).db # pyright: ignore[reportAny, reportReturnType] # untyped Prisma client behind the pin
|
||||
|
||||
|
||||
async def _invalidate_access_group_cache(access_group_id: str) -> None:
|
||||
|
|
|
|||
|
|
@ -352,7 +352,10 @@ from litellm.proxy.common_utils.load_config_utils import (
|
|||
get_file_contents_from_s3,
|
||||
)
|
||||
from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations
|
||||
from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator
|
||||
from litellm.proxy.common_utils.model_listing_utils import (
|
||||
TeamModelNameTranslator,
|
||||
configured_display_names,
|
||||
)
|
||||
from litellm.proxy.common_utils.openai_endpoint_utils import (
|
||||
remove_sensitive_info_from_deployment,
|
||||
)
|
||||
|
|
@ -10223,7 +10226,8 @@ async def model_list(
|
|||
# The internal routing key drives the metadata/fallback lookup, while the
|
||||
# public name is what the client sees as the model id.
|
||||
model_data = []
|
||||
for response_id, lookup_id in TeamModelNameTranslator.listing_entries(all_models, llm_router, settings):
|
||||
admin_entries: Final = TeamModelNameTranslator.listing_entries(all_models, llm_router, settings)
|
||||
for response_id, lookup_id in admin_entries:
|
||||
model_info = create_model_info_response(
|
||||
model_id=lookup_id,
|
||||
provider="openai",
|
||||
|
|
@ -10236,7 +10240,10 @@ async def model_list(
|
|||
|
||||
if wants_anthropic_format:
|
||||
admin_listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above
|
||||
return create_anthropic_model_list_response(admin_listing)
|
||||
return create_anthropic_model_list_response(
|
||||
admin_listing,
|
||||
display_names=configured_display_names(admin_entries, llm_router),
|
||||
)
|
||||
|
||||
return dict(
|
||||
data=model_data,
|
||||
|
|
@ -10267,7 +10274,8 @@ async def model_list(
|
|||
# The internal routing key drives the metadata/fallback lookup, while the
|
||||
# public name is what the client sees as the model id.
|
||||
model_data = []
|
||||
for response_id, lookup_id in TeamModelNameTranslator.listing_entries(all_models, llm_router, settings):
|
||||
entries: Final = TeamModelNameTranslator.listing_entries(all_models, llm_router, settings)
|
||||
for response_id, lookup_id in entries:
|
||||
model_info = create_model_info_response(
|
||||
model_id=lookup_id,
|
||||
provider="openai",
|
||||
|
|
@ -10280,7 +10288,10 @@ async def model_list(
|
|||
|
||||
if wants_anthropic_format:
|
||||
listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above
|
||||
return create_anthropic_model_list_response(listing)
|
||||
return create_anthropic_model_list_response(
|
||||
listing,
|
||||
display_names=configured_display_names(entries, llm_router),
|
||||
)
|
||||
|
||||
return dict(
|
||||
data=model_data,
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from typing import Any, Final, Literal
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
|
||||
from litellm.llms.bedrock.rerank.handler import BedrockRerankHandler
|
||||
|
|
@ -43,10 +44,23 @@ async def arerank(
|
|||
"""
|
||||
Async: Reranks a list of documents based on their relevance to the query
|
||||
"""
|
||||
_custom_llm_provider: str | None = (
|
||||
None # rebind-ok: set by the declared-provider guard or the get_llm_provider unpack; read in the except
|
||||
)
|
||||
try:
|
||||
loop: Final = asyncio.get_event_loop()
|
||||
kwargs["arerank"] = True
|
||||
|
||||
declared_provider: Final = declared_authenticating_provider(model, custom_llm_provider)
|
||||
if declared_provider is not None:
|
||||
_custom_llm_provider = declared_provider # rebind-ok: see pre-declaration above
|
||||
else:
|
||||
_, _custom_llm_provider, _, _ = litellm.get_llm_provider( # rebind-ok: see pre-declaration above
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=kwargs.get("api_base", None),
|
||||
)
|
||||
|
||||
func: Final = partial(
|
||||
rerank,
|
||||
model,
|
||||
|
|
@ -70,7 +84,11 @@ async def arerank(
|
|||
response = init_response
|
||||
return response
|
||||
except Exception as e:
|
||||
raise e
|
||||
raise exception_type(
|
||||
model=model,
|
||||
custom_llm_provider=_custom_llm_provider or custom_llm_provider,
|
||||
original_exception=e,
|
||||
)
|
||||
|
||||
|
||||
@client
|
||||
|
|
@ -115,6 +133,7 @@ def rerank(
|
|||
model_info: Final = kwargs.get("model_info", None)
|
||||
user: Final = kwargs.get("user", None)
|
||||
client: Final = kwargs.get("client", None)
|
||||
_custom_llm_provider: str | None = None # rebind-ok: set by the get_llm_provider unpack; read in the except
|
||||
try:
|
||||
_is_async: Final = kwargs.pop("arerank", False) is True
|
||||
optional_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
|
|
@ -127,7 +146,7 @@ def rerank(
|
|||
|
||||
(
|
||||
model,
|
||||
_custom_llm_provider,
|
||||
_custom_llm_provider, # rebind-ok: see pre-declaration above
|
||||
dynamic_api_key,
|
||||
dynamic_api_base,
|
||||
) = litellm.get_llm_provider(
|
||||
|
|
@ -538,4 +557,8 @@ def rerank(
|
|||
return response
|
||||
except Exception as e:
|
||||
verbose_logger.error("Error in rerank: %s", e)
|
||||
raise exception_type(model=model, custom_llm_provider=custom_llm_provider, original_exception=e)
|
||||
raise exception_type(
|
||||
model=model,
|
||||
custom_llm_provider=_custom_llm_provider or custom_llm_provider,
|
||||
original_exception=e,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1169,16 +1169,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
|
||||
def _emit_response_completed_event(self, litellm_model_response: ModelResponse) -> ResponseCompletedEvent | None:
|
||||
if litellm_model_response:
|
||||
# Add cost to usage object if include_cost_in_streaming_usage is True
|
||||
if litellm.include_cost_in_streaming_usage and self.litellm_logging_obj is not None:
|
||||
usage: Final[object] = getattr(litellm_model_response, "usage", None)
|
||||
if usage is not None:
|
||||
setattr(
|
||||
usage,
|
||||
"cost",
|
||||
self.litellm_logging_obj._response_cost_calculator(result=litellm_model_response),
|
||||
)
|
||||
|
||||
# Transform the response
|
||||
responses_api_response: Final = (
|
||||
LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response(
|
||||
|
|
|
|||
|
|
@ -407,23 +407,7 @@ class BaseResponsesAPIStreamingIterator:
|
|||
openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED,
|
||||
):
|
||||
self.completed_response = openai_responses_api_chunk
|
||||
# Add cost to usage object if include_cost_in_streaming_usage is True
|
||||
if litellm.include_cost_in_streaming_usage and self.logging_obj is not None:
|
||||
response_obj: Final[ResponsesAPIResponse | None] = getattr(
|
||||
openai_responses_api_chunk, "response", None
|
||||
)
|
||||
if response_obj:
|
||||
usage_obj: Final[ResponseAPIUsage | None] = getattr(response_obj, "usage", None)
|
||||
if usage_obj is not None:
|
||||
try:
|
||||
cost: Final[float | None] = self.logging_obj._response_cost_calculator(
|
||||
result=response_obj
|
||||
)
|
||||
if cost is not None:
|
||||
setattr(usage_obj, "cost", cost)
|
||||
except Exception:
|
||||
# Best-effort usage cost annotation should not break stream replay.
|
||||
pass
|
||||
_stamp_responses_usage_cost(getattr(openai_responses_api_chunk, "response", None), self.logging_obj)
|
||||
|
||||
if _chunk_type == openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED:
|
||||
self._handle_logging_failed_response()
|
||||
|
|
@ -1274,6 +1258,24 @@ def _add_text_like_part_events(
|
|||
)
|
||||
|
||||
|
||||
def _stamp_responses_usage_cost(
|
||||
response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None
|
||||
) -> None:
|
||||
if response_obj is None or logging_obj is None:
|
||||
return
|
||||
usage_obj: Final[ResponseAPIUsage | None] = getattr(response_obj, "usage", None)
|
||||
if usage_obj is None:
|
||||
return
|
||||
if isinstance(getattr(usage_obj, "cost", None), (int, float)):
|
||||
return
|
||||
try:
|
||||
cost: Final[float | None] = logging_obj._response_cost_calculator(result=response_obj)
|
||||
except Exception:
|
||||
return
|
||||
if isinstance(cost, (int, float)) and cost > 0:
|
||||
setattr(usage_obj, "cost", cost)
|
||||
|
||||
|
||||
def build_synthetic_response_events(
|
||||
*,
|
||||
transformed: ResponsesAPIResponse,
|
||||
|
|
@ -1281,15 +1283,7 @@ def build_synthetic_response_events(
|
|||
chunk_size: int,
|
||||
) -> list[ResponsesAPIStreamingResponse]:
|
||||
openai_types: Final = _get_openai_response_types()
|
||||
if litellm.include_cost_in_streaming_usage and logging_obj is not None:
|
||||
usage_obj: Final = transformed.usage if hasattr(transformed, "usage") else None
|
||||
if usage_obj is not None:
|
||||
try:
|
||||
cost: Final[float | None] = logging_obj._response_cost_calculator(result=transformed)
|
||||
if cost is not None:
|
||||
setattr(usage_obj, "cost", cost)
|
||||
except Exception:
|
||||
pass
|
||||
_stamp_responses_usage_cost(transformed, logging_obj)
|
||||
|
||||
events: Final[list[ResponsesAPIStreamingResponse]] = [
|
||||
_build_response_status_event(openai_types.ResponsesAPIStreamEvents.RESPONSE_CREATED, transformed),
|
||||
|
|
|
|||
|
|
@ -9746,6 +9746,26 @@ class Router:
|
|||
coerce_token_limit(model_info.get("max_output_tokens")),
|
||||
)
|
||||
|
||||
def get_configured_display_name(self, model_name: str) -> "str | None":
|
||||
"""
|
||||
Return the display_name explicitly configured in a concrete deployment's
|
||||
model_info for model_name, via O(1) index lookup.
|
||||
|
||||
Returns None for wildcard-expanded or unknown names, and treats a
|
||||
non-string or empty configured value as absent rather than failing the
|
||||
listing. Like get_configured_token_limits, this never triggers pattern
|
||||
matching or deep copies, so it is safe to call per listed model on the
|
||||
/v1/models hot path.
|
||||
"""
|
||||
deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name)
|
||||
if deployment is None:
|
||||
return None
|
||||
|
||||
display_name: Final = deployment.model_info.get("display_name")
|
||||
if isinstance(display_name, str) and display_name.strip():
|
||||
return display_name
|
||||
return None
|
||||
|
||||
def get_deployment_credentials_with_provider(
|
||||
self, model_id: str, team_id: str | None = None
|
||||
) -> dict[str, Any] | None:
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -24,25 +24,10 @@ from junit_properties import (
|
|||
)
|
||||
|
||||
|
||||
class FakeMarker:
|
||||
def __init__(self, name: str, *args: object) -> None:
|
||||
self.name = name
|
||||
self.args = args
|
||||
|
||||
|
||||
class FakeItem:
|
||||
"""The three attributes junit_properties reads off a pytest Item."""
|
||||
|
||||
def __init__(
|
||||
self, nodeid: str, location: tuple[str, int | None, str], markers: tuple[FakeMarker, ...] = ()
|
||||
) -> None:
|
||||
self.nodeid = nodeid
|
||||
self.location = location
|
||||
self.user_properties: list[tuple[str, str]] = []
|
||||
self._markers = markers
|
||||
|
||||
def iter_markers(self, name: str):
|
||||
return (marker for marker in self._markers if marker.name == name)
|
||||
def collected_item(request: pytest.FixtureRequest, name: str) -> pytest.Item:
|
||||
"""The Item pytest collected for test ``name`` in this file: the real nodeid,
|
||||
location and marker machinery the collection hook reads, as pytest built it."""
|
||||
return next(item for item in request.session.items if item.path == request.path and item.name == name)
|
||||
|
||||
|
||||
def repo_root() -> Path | None:
|
||||
|
|
@ -109,22 +94,22 @@ class TestSourceFromLocation:
|
|||
|
||||
|
||||
class TestResultProperties:
|
||||
def test_every_test_carries_package_covers_and_source(self) -> None:
|
||||
item = FakeItem(
|
||||
"logging/test_x.py::TestFoo::test_bar",
|
||||
("logging/test_x.py", 40, "TestFoo.test_bar"),
|
||||
(FakeMarker("covers", "LOG-1", "LOG-2"),),
|
||||
)
|
||||
assert result_properties(item) == (
|
||||
("package", "logging"),
|
||||
def test_every_test_carries_package_covers_and_source(self, request: pytest.FixtureRequest) -> None:
|
||||
"""Read off this test's own collected Item, so the nodeid and location are
|
||||
whatever pytest reports for the launch shape in use, and the marker is added
|
||||
at run time so the coverage registry's collect-only pass never sees it."""
|
||||
test = type(self).test_every_test_carries_package_covers_and_source
|
||||
request.applymarker(pytest.mark.covers("LOG-1", "LOG-2"))
|
||||
assert result_properties(collected_item(request, test.__name__)) == (
|
||||
("package", "root"),
|
||||
("covers", "LOG-1,LOG-2"),
|
||||
("source", "tests/e2e/logging/test_x.py:41"),
|
||||
("source", f"tests/e2e/test_junit_properties.py:{test.__code__.co_firstlineno}"),
|
||||
)
|
||||
|
||||
def test_attach_is_idempotent(self) -> None:
|
||||
def test_attach_is_idempotent(self, request: pytest.FixtureRequest) -> None:
|
||||
"""Collection can run the hook more than once; a second pass must not
|
||||
double the <property> entries in the report."""
|
||||
item = FakeItem("logging/test_x.py::test_bar", ("logging/test_x.py", 40, "test_bar"))
|
||||
item = collected_item(request, type(self).test_attach_is_idempotent.__name__)
|
||||
attach_result_properties(item)
|
||||
attach_result_properties(item)
|
||||
assert [name for name, _ in item.user_properties] == ["package", "covers", "source"]
|
||||
|
|
|
|||
|
|
@ -5,9 +5,11 @@ Note: Vertex AI OCR automatically converts URLs to base64 data URIs since
|
|||
the Vertex AI endpoint doesn't have internet access.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from base_ocr_unit_tests import BaseOCRTest
|
||||
|
||||
|
|
@ -139,3 +141,19 @@ def test_vertex_ai_ocr_routing():
|
|||
assert isinstance(
|
||||
deepseek_variant, VertexAIDeepSeekOCRConfig
|
||||
), "DeepSeek variant should route to VertexAIDeepSeekOCRConfig"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ("deepseek-ocr-maas", "deepseek-ai/deepseek-ocr-maas"))
|
||||
def test_deepseek_request_uses_single_provider_namespace(model: str) -> None:
|
||||
from litellm.llms.vertex_ai.ocr.deepseek_transformation import (
|
||||
VertexAIDeepSeekOCRConfig,
|
||||
)
|
||||
|
||||
request: Final = VertexAIDeepSeekOCRConfig().transform_ocr_request(
|
||||
model=model,
|
||||
document={"type": "image_url", "image_url": "data:image/png;base64,AA=="},
|
||||
optional_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request.data["model"] == "deepseek-ai/deepseek-ocr-maas"
|
||||
|
|
|
|||
|
|
@ -1522,7 +1522,7 @@ def test_gpt_5_6_alias_prices_match_sol(local_model_cost_map):
|
|||
sol = litellm.model_cost["gpt-5.6-sol"]
|
||||
|
||||
cost_fields = sorted(field for field in sol if "cost" in field)
|
||||
assert len(cost_fields) == 23
|
||||
assert len(cost_fields) == 27
|
||||
|
||||
for field in cost_fields:
|
||||
assert alias.get(field) == sol.get(field), field
|
||||
|
|
@ -4039,8 +4039,8 @@ def test_fast_service_tier_matches_priority_above_the_context_threshold(_local_m
|
|||
)
|
||||
|
||||
assert fast == priority
|
||||
assert fast[0] == pytest.approx(300_000 * 8e-06, rel=1e-9)
|
||||
assert fast[1] == pytest.approx(1_000 * 3e-05, rel=1e-9)
|
||||
assert fast[0] == pytest.approx(300_000 * 1.6e-05, rel=1e-9)
|
||||
assert fast[1] == pytest.approx(1_000 * 6e-05, rel=1e-9)
|
||||
|
||||
|
||||
def test_priority_reasoning_tokens_bill_at_the_priority_output_rate(_local_model_cost_map):
|
||||
|
|
@ -4200,6 +4200,86 @@ def test_generic_cost_per_token_gemini_37_flash(_local_model_cost_map):
|
|||
assert completion_cost == pytest.approx(0.001875)
|
||||
|
||||
|
||||
GEMINI_38_FLASH_LAUNCH_PRICING = [
|
||||
("gemini-3.8-flash", 7.5e-07, 3.75e-06, 7.5e-08),
|
||||
("gemini/gemini-3.8-flash", 7.5e-07, 3.75e-06, 7.5e-08),
|
||||
("vertex_ai/gemini-3.8-flash", 7.5e-07, 3.75e-06, 7.5e-08),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model,input_cost,output_cost,cache_read_cost", GEMINI_38_FLASH_LAUNCH_PRICING)
|
||||
def test_gemini_38_flash_launch_pricing(model, input_cost, output_cost, cache_read_cost, _local_model_cost_map):
|
||||
model_cost_map = litellm.model_cost[model]
|
||||
assert model_cost_map["input_cost_per_token"] == input_cost
|
||||
assert model_cost_map["output_cost_per_token"] == output_cost
|
||||
assert model_cost_map["output_cost_per_reasoning_token"] == output_cost
|
||||
assert model_cost_map["cache_read_input_token_cost"] == cache_read_cost
|
||||
assert model_cost_map["mode"] == "chat"
|
||||
assert model_cost_map["supports_reasoning"] is True
|
||||
assert model_cost_map["supports_function_calling"] is True
|
||||
assert model_cost_map["max_input_tokens"] == 1048576
|
||||
|
||||
|
||||
GEMINI_38_FLASH_FIELDS_SHARED_WITH_37_FLASH = (
|
||||
"input_cost_per_token",
|
||||
"output_cost_per_token",
|
||||
"output_cost_per_reasoning_token",
|
||||
"cache_read_input_token_cost",
|
||||
"input_cost_per_token_batches",
|
||||
"output_cost_per_token_batches",
|
||||
"input_cost_per_token_flex",
|
||||
"output_cost_per_token_flex",
|
||||
"cache_read_input_token_cost_flex",
|
||||
"input_cost_per_token_priority",
|
||||
"output_cost_per_token_priority",
|
||||
"cache_read_input_token_cost_priority",
|
||||
"search_context_cost_per_query",
|
||||
"google_maps_grounding_cost_per_query",
|
||||
"prompt_cache_min_tokens",
|
||||
"max_input_tokens",
|
||||
"max_output_tokens",
|
||||
"supports_reasoning",
|
||||
"supports_function_calling",
|
||||
"supports_prompt_caching",
|
||||
"supports_vision",
|
||||
"supports_pdf_input",
|
||||
"supports_audio_input",
|
||||
"supports_video_input",
|
||||
"supports_response_schema",
|
||||
"supports_tool_choice",
|
||||
"supports_web_search",
|
||||
"supports_url_context",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prefix", ["", "gemini/", "vertex_ai/"])
|
||||
def test_gemini_38_flash_matches_37_flash_promotional_pricing(prefix, _local_model_cost_map):
|
||||
new_model = litellm.model_cost[f"{prefix}gemini-3.8-flash"]
|
||||
old_model = litellm.model_cost[f"{prefix}gemini-3.7-flash"]
|
||||
for field in GEMINI_38_FLASH_FIELDS_SHARED_WITH_37_FLASH:
|
||||
assert new_model[field] == old_model[field], field
|
||||
|
||||
|
||||
def test_generic_cost_per_token_gemini_38_flash(_local_model_cost_map):
|
||||
usage = Usage(
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=500,
|
||||
total_tokens=1500,
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(
|
||||
reasoning_tokens=200,
|
||||
text_tokens=300,
|
||||
),
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=1000),
|
||||
)
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
model="gemini-3.8-flash",
|
||||
usage=usage,
|
||||
custom_llm_provider="gemini",
|
||||
)
|
||||
assert prompt_cost == pytest.approx(0.00075)
|
||||
assert completion_cost == pytest.approx(0.001875)
|
||||
|
||||
|
||||
def test_grok_46_launch_pricing(_local_model_cost_map):
|
||||
model_cost_map = litellm.model_cost["xai/grok-4.6"]
|
||||
assert model_cost_map["input_cost_per_token"] == 2e-06
|
||||
|
|
|
|||
|
|
@ -592,6 +592,59 @@ def test_stream_chunk_builder_litellm_usage_chunks():
|
|||
assert usage.total_tokens == 77
|
||||
|
||||
|
||||
def test_calculate_usage_honors_openai_sdk_completion_usage_chunks():
|
||||
from openai.types.completion_usage import CompletionUsage
|
||||
|
||||
content_chunk = ModelResponseStream(
|
||||
id="chatcmpl-sdk-usage-1",
|
||||
created=1745513206,
|
||||
model="mantle-claude",
|
||||
object="chat.completion.chunk",
|
||||
system_fingerprint=None,
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
finish_reason="stop",
|
||||
index=0,
|
||||
delta=Delta(
|
||||
provider_specific_fields=None,
|
||||
content="ok",
|
||||
role=None,
|
||||
function_call=None,
|
||||
tool_calls=None,
|
||||
audio=None,
|
||||
),
|
||||
logprobs=None,
|
||||
)
|
||||
],
|
||||
provider_specific_fields=None,
|
||||
stream_options={"include_usage": True},
|
||||
)
|
||||
usage_chunk = ModelResponseStream(
|
||||
id="chatcmpl-sdk-usage-1",
|
||||
created=1745513207,
|
||||
model="mantle-claude",
|
||||
object="chat.completion.chunk",
|
||||
system_fingerprint=None,
|
||||
choices=[],
|
||||
provider_specific_fields=None,
|
||||
stream_options={"include_usage": True},
|
||||
)
|
||||
usage_chunk.usage = CompletionUsage(
|
||||
prompt_tokens=20, completion_tokens=60, total_tokens=80, cost=0.000704
|
||||
)
|
||||
assert type(usage_chunk.usage) is CompletionUsage
|
||||
|
||||
chunks = [content_chunk, usage_chunk]
|
||||
usage = ChunkProcessor(chunks=chunks).calculate_usage(
|
||||
chunks=chunks, model="mantle-claude", completion_output=""
|
||||
)
|
||||
|
||||
assert usage.prompt_tokens == 20
|
||||
assert usage.completion_tokens == 60
|
||||
assert usage.total_tokens == 80
|
||||
assert getattr(usage, "cost", None) == pytest.approx(0.000704)
|
||||
|
||||
|
||||
def test_get_model_from_chunks_azure_model_router():
|
||||
"""
|
||||
Test that _get_model_from_chunks finds the actual model from Azure Model Router chunks.
|
||||
|
|
|
|||
|
|
@ -1096,10 +1096,13 @@ def test_natively_signed_parallel_turn_never_carries_a_placeholder(model):
|
|||
"gemini-3.5-flash",
|
||||
"gemini-3.6-flash",
|
||||
"gemini-3.7-flash",
|
||||
"gemini-3.8-flash",
|
||||
"vertex_ai/gemini-3.5-flash",
|
||||
"vertex_ai/gemini-3.7-flash",
|
||||
"vertex_ai/gemini-3.8-flash",
|
||||
"gemini/gemini-3.5-flash",
|
||||
"gemini/gemini-3.7-flash",
|
||||
"gemini/gemini-3.8-flash",
|
||||
],
|
||||
)
|
||||
def test_placeholder_scoped_to_first_call_across_gemini_3_variants(model):
|
||||
|
|
|
|||
|
|
@ -1185,6 +1185,18 @@ def test_vertex_ai_map_thinking_param_with_budget_tokens_0():
|
|||
}
|
||||
|
||||
|
||||
def test_vertex_ai_map_thinking_param_without_budget_tokens_for_gemini_3():
|
||||
v = VertexGeminiConfig()
|
||||
result = v.map_openai_params(
|
||||
non_default_params={"thinking": {"type": "enabled"}},
|
||||
optional_params={},
|
||||
model="gemini-3.5-flash",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert result["thinkingConfig"] == {"includeThoughts": True}
|
||||
|
||||
|
||||
def test_vertex_ai_map_tools():
|
||||
v = VertexGeminiConfig()
|
||||
optional_params = {}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,57 @@
|
|||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.db.prisma_client import PrismaWrapper
|
||||
from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper
|
||||
from litellm.proxy.management_helpers.access_group_key_sync import (
|
||||
sync_key_access_group_membership,
|
||||
sync_key_regeneration_access_group_membership,
|
||||
)
|
||||
|
||||
|
||||
def _routed_prisma_client():
|
||||
writer_inner = MagicMock(name="writer_prisma")
|
||||
reader_inner = MagicMock(name="reader_prisma")
|
||||
writer_inner.query_raw = AsyncMock(return_value=[])
|
||||
reader_inner.query_raw = AsyncMock(return_value=[])
|
||||
writer = PrismaWrapper(original_prisma=writer_inner, iam_token_db_auth=False)
|
||||
reader = PrismaWrapper(original_prisma=reader_inner, iam_token_db_auth=False)
|
||||
routing = RoutingPrismaWrapper(writer=writer, reader=reader)
|
||||
return SimpleNamespace(db=routing), writer_inner, reader_inner
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_regeneration_repoint_update_runs_on_the_writer():
|
||||
prisma_client, writer_inner, reader_inner = _routed_prisma_client()
|
||||
|
||||
await sync_key_regeneration_access_group_membership(
|
||||
prisma_client=prisma_client,
|
||||
previous_key_token="old-token",
|
||||
new_key_token="new-token",
|
||||
data=None,
|
||||
existing_key_row=MagicMock(),
|
||||
)
|
||||
|
||||
writer_inner.query_raw.assert_awaited_once()
|
||||
assert writer_inner.query_raw.await_args.args[0].startswith('UPDATE "LiteLLM_AccessGroupTable"')
|
||||
reader_inner.query_raw.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_membership_attach_and_detach_updates_run_on_the_writer():
|
||||
prisma_client, writer_inner, reader_inner = _routed_prisma_client()
|
||||
|
||||
await sync_key_access_group_membership(
|
||||
prisma_client=prisma_client,
|
||||
key_token="token",
|
||||
previous_access_group_ids=["ag-old"],
|
||||
updated_access_group_ids=["ag-new"],
|
||||
)
|
||||
|
||||
assert writer_inner.query_raw.await_count == 2
|
||||
assert all(
|
||||
call.args[0].startswith('UPDATE "LiteLLM_AccessGroupTable"') for call in writer_inner.query_raw.await_args_list
|
||||
)
|
||||
reader_inner.query_raw.assert_not_awaited()
|
||||
|
|
@ -45,6 +45,7 @@ def patched_models(monkeypatch):
|
|||
deployment = MagicMock()
|
||||
deployment.litellm_params.model = "gpt-4"
|
||||
router.get_deployment_by_model_group_name = MagicMock(return_value=deployment)
|
||||
router.get_configured_display_name = MagicMock(return_value=None)
|
||||
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", MagicMock())
|
||||
|
|
@ -187,6 +188,83 @@ def test_anthropic_format_carries_router_configured_token_limits(client, auth_as
|
|||
assert (claude["max_input_tokens"], claude["max_tokens"]) == (500000, 4096)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/models", "/models"])
|
||||
def test_anthropic_format_uses_configured_display_name(client, auth_as, patched_models, path):
|
||||
"""A deployment's ``model_info.display_name`` becomes the Anthropic-native
|
||||
``display_name`` so Claude Code's picker shows a clean name while the id keeps
|
||||
routing; models without one keep the id fallback, and the OpenAI-shaped
|
||||
listing carries no display_name either way."""
|
||||
|
||||
def _configured(model_name):
|
||||
return "Kimi K3" if model_name == "gpt-4" else None
|
||||
|
||||
patched_models.get_configured_display_name = MagicMock(side_effect=_configured)
|
||||
|
||||
with auth_as():
|
||||
anthropic_response = client.get(path, headers={"anthropic-version": "2023-06-01"})
|
||||
openai_response = client.get(path)
|
||||
|
||||
assert anthropic_response.status_code == 200
|
||||
gpt_4, claude = anthropic_response.json()["data"]
|
||||
assert (gpt_4["id"], gpt_4["display_name"]) == ("gpt-4", "Kimi K3")
|
||||
assert (claude["id"], claude["display_name"]) == ("claude-sonnet", "claude-sonnet")
|
||||
|
||||
assert openai_response.status_code == 200
|
||||
openai_models = openai_response.json()["data"]
|
||||
assert [m["id"] for m in openai_models] == ["gpt-4", "claude-sonnet"]
|
||||
assert all("display_name" not in m for m in openai_models)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("params", [{}, {"scope": "expand"}])
|
||||
def test_anthropic_display_name_resolved_via_internal_team_key(
|
||||
client, auth_as, patched_models, monkeypatch, params
|
||||
):
|
||||
"""For a team-scoped row the configured display name must be looked up by the
|
||||
internal routing key while the entry itself is keyed by the public name, so
|
||||
the clean name lands on the id the client actually sees."""
|
||||
from litellm.proxy import utils as proxy_utils
|
||||
from litellm.proxy.auth import model_checks
|
||||
|
||||
internal_name = "model_name_team-1_c0ffee"
|
||||
|
||||
patched_models.get_model_list = MagicMock(
|
||||
return_value=[
|
||||
{
|
||||
"model_name": internal_name,
|
||||
"model_info": {
|
||||
"team_id": "team-1",
|
||||
"team_public_model_name": "gpt-4-team",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
patched_models.get_model_names = MagicMock(return_value=[internal_name])
|
||||
patched_models.get_configured_display_name = MagicMock(
|
||||
side_effect=lambda model_name: "Team GPT" if model_name == internal_name else None
|
||||
)
|
||||
|
||||
async def _fake_get_available_models_for_user(**kwargs):
|
||||
return [internal_name]
|
||||
|
||||
monkeypatch.setattr(
|
||||
proxy_utils,
|
||||
"get_available_models_for_user",
|
||||
_fake_get_available_models_for_user,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
model_checks, "get_complete_model_list", lambda **kwargs: [internal_name]
|
||||
)
|
||||
|
||||
with auth_as():
|
||||
response = client.get(
|
||||
"/v1/models", params=params, headers={"anthropic-version": "2023-06-01"}
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
(entry,) = response.json()["data"]
|
||||
assert (entry["id"], entry["display_name"]) == ("gpt-4-team", "Team GPT")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/models", "/models"])
|
||||
def test_get_models_invalid_scope_returns_400(client, auth_as, patched_models, path):
|
||||
"""Pins: ``GET /v1/models``, ``GET /models`` (error path: invalid scope)."""
|
||||
|
|
|
|||
|
|
@ -19,7 +19,10 @@ from litellm.proxy._types import (
|
|||
LitellmUserRoles,
|
||||
UserAPIKeyAuth,
|
||||
)
|
||||
from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator
|
||||
from litellm.proxy.common_utils.model_listing_utils import (
|
||||
TeamModelNameTranslator,
|
||||
configured_display_names,
|
||||
)
|
||||
from litellm.proxy.proxy_server import (
|
||||
_get_proxy_model_info,
|
||||
_translate_model_name_for_response,
|
||||
|
|
@ -1391,6 +1394,27 @@ def test_resolve_public_name_respects_legacy_flag():
|
|||
)
|
||||
|
||||
|
||||
def test_configured_display_names_keyed_by_response_id():
|
||||
"""The map is keyed by the public response id while the router lookup uses
|
||||
the internal routing key, and entries without a configured name are omitted."""
|
||||
router = MagicMock()
|
||||
router.get_configured_display_name = MagicMock(
|
||||
side_effect=lambda model_name: "Team Sonnet" if model_name == "model_name_team-abc-123_4a6b8" else None
|
||||
)
|
||||
|
||||
assert configured_display_names(
|
||||
entries=[
|
||||
("team-claude-sonnet", "model_name_team-abc-123_4a6b8"),
|
||||
("gpt-4o", "gpt-4o"),
|
||||
],
|
||||
llm_router=router,
|
||||
) == {"team-claude-sonnet": "Team Sonnet"}
|
||||
|
||||
|
||||
def test_configured_display_names_empty_without_router():
|
||||
assert configured_display_names(entries=[("gpt-4o", "gpt-4o")], llm_router=None) == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retrieve_model_by_public_name_returns_200(monkeypatch):
|
||||
"""Regression: `GET /v1/models/{public_name}` must NOT 404. The listing
|
||||
|
|
|
|||
|
|
@ -111,6 +111,99 @@ def test_together_rerank_honors_api_base(respx_mock: respx.MockRouter):
|
|||
assert mock_route.calls[0].request.headers["authorization"] == "Bearer fake-together-key"
|
||||
|
||||
|
||||
DASHSCOPE_404_BODY = {
|
||||
"error": {
|
||||
"message": "The model `does-not-exist` does not exist or you do not have access to it.",
|
||||
"type": "invalid_request_error",
|
||||
"param": None,
|
||||
"code": "model_not_found",
|
||||
},
|
||||
"request_id": "mock-request-id",
|
||||
}
|
||||
|
||||
|
||||
def test_rerank_error_names_provider_and_keeps_body(respx_mock: respx.MockRouter, monkeypatch):
|
||||
"""Regression for the rerank error path mapping with the unresolved provider param:
|
||||
a provider 404 surfaced as 'None - ' instead of naming the provider and its error body."""
|
||||
monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False)
|
||||
monkeypatch.delenv("DASHSCOPE_API_BASE_RERANK", raising=False)
|
||||
|
||||
mock_route = respx_mock.post("https://dashscope.example/v1/reranks")
|
||||
mock_route.return_value = httpx.Response(404, json=DASHSCOPE_404_BODY)
|
||||
|
||||
with pytest.raises(litellm.NotFoundError) as exc_info:
|
||||
litellm.rerank(
|
||||
model="dashscope/does-not-exist",
|
||||
query=MARKER_QUERY,
|
||||
documents=[MARKER_DOC],
|
||||
api_key="fake-dashscope-key",
|
||||
api_base="https://dashscope.example/v1",
|
||||
)
|
||||
|
||||
assert mock_route.called
|
||||
assert "DashscopeException" in str(exc_info.value)
|
||||
assert "does not exist or you do not have access to it" in str(exc_info.value)
|
||||
assert "None - " not in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_arerank_error_is_mapped_to_litellm_exception(respx_mock: respx.MockRouter, monkeypatch):
|
||||
"""Regression for arerank's bare re-raise: provider errors escaped as raw
|
||||
provider exception classes instead of the mapped litellm exception contract."""
|
||||
monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False)
|
||||
monkeypatch.delenv("DASHSCOPE_API_BASE_RERANK", raising=False)
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
|
||||
mock_route = respx_mock.post("https://dashscope.example/v1/reranks")
|
||||
mock_route.return_value = httpx.Response(404, json=DASHSCOPE_404_BODY)
|
||||
|
||||
with pytest.raises(litellm.NotFoundError) as exc_info:
|
||||
await litellm.arerank(
|
||||
model="dashscope/does-not-exist",
|
||||
query=MARKER_QUERY,
|
||||
documents=[MARKER_DOC],
|
||||
api_key="fake-dashscope-key",
|
||||
api_base="https://dashscope.example/v1",
|
||||
)
|
||||
|
||||
assert mock_route.called
|
||||
assert "DashscopeException" in str(exc_info.value)
|
||||
assert "does not exist or you do not have access to it" in str(exc_info.value)
|
||||
assert "None - " not in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_arerank_declared_authenticating_provider_skips_resolution(monkeypatch):
|
||||
"""Regression for the event-loop hazard in arerank's provider pre-resolution:
|
||||
get_llm_provider runs the blocking OAuth device flow for github_copilot/chatgpt,
|
||||
so arerank must adopt the declared provider instead of resolving it, while the
|
||||
except path still maps with that declared provider."""
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
||||
resolution_calls = []
|
||||
|
||||
def record_resolution(*args, **kwargs):
|
||||
resolution_calls.append((args, kwargs))
|
||||
return "gpt-4o", "github_copilot", None, None
|
||||
|
||||
def rerank_raises_provider_error(*args, **kwargs):
|
||||
raise BaseLLMException(status_code=401, message='{"error":"bad key"}')
|
||||
|
||||
monkeypatch.setattr(litellm, "get_llm_provider", record_resolution)
|
||||
monkeypatch.setattr("litellm.rerank_api.main.rerank", rerank_raises_provider_error)
|
||||
|
||||
with pytest.raises(litellm.AuthenticationError) as exc_info:
|
||||
await litellm.arerank(
|
||||
model="github_copilot/gpt-4o",
|
||||
query=MARKER_QUERY,
|
||||
documents=[MARKER_DOC],
|
||||
)
|
||||
|
||||
assert resolution_calls == []
|
||||
assert "Github_copilotException" in str(exc_info.value)
|
||||
assert "None - " not in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_together_rerank_async_honors_env_api_base(respx_mock: respx.MockRouter, monkeypatch):
|
||||
"""Regression: TOGETHER_AI_API_BASE was honored by chat but ignored by rerank."""
|
||||
|
|
|
|||
|
|
@ -326,3 +326,55 @@ def test_run_post_success_hooks_does_not_report_generation_time_as_overhead():
|
|||
|
||||
assert iterator.completed_response._hidden_params["_response_ms"] == 10000.0
|
||||
assert "litellm_overhead_time_ms" not in iterator.completed_response._hidden_params
|
||||
|
||||
|
||||
def _responses_api_response_with_usage() -> ResponsesAPIResponse:
|
||||
from litellm.types.llms.openai import ResponseAPIUsage
|
||||
|
||||
return ResponsesAPIResponse(
|
||||
id="resp_lit6427",
|
||||
created_at=int(datetime(2025, 1, 1).timestamp()),
|
||||
status="completed",
|
||||
model="mantle-claude",
|
||||
object="response",
|
||||
output=[],
|
||||
usage=ResponseAPIUsage(input_tokens=20, output_tokens=60, total_tokens=80),
|
||||
)
|
||||
|
||||
|
||||
def test_stamp_responses_usage_cost_stamps_computed_cost():
|
||||
from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
|
||||
|
||||
response = _responses_api_response_with_usage()
|
||||
logging_obj = Mock(spec=LiteLLMLoggingObj)
|
||||
logging_obj._response_cost_calculator.return_value = 0.000704
|
||||
|
||||
_stamp_responses_usage_cost(response, logging_obj)
|
||||
|
||||
assert getattr(response.usage, "cost", None) == pytest.approx(0.000704)
|
||||
logging_obj._response_cost_calculator.assert_called_once_with(result=response)
|
||||
|
||||
|
||||
def test_stamp_responses_usage_cost_keeps_provider_reported_cost():
|
||||
from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
|
||||
|
||||
response = _responses_api_response_with_usage()
|
||||
setattr(response.usage, "cost", 0.5)
|
||||
logging_obj = Mock(spec=LiteLLMLoggingObj)
|
||||
|
||||
_stamp_responses_usage_cost(response, logging_obj)
|
||||
|
||||
assert getattr(response.usage, "cost", None) == pytest.approx(0.5)
|
||||
logging_obj._response_cost_calculator.assert_not_called()
|
||||
|
||||
|
||||
def test_stamp_responses_usage_cost_survives_calculator_failure():
|
||||
from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
|
||||
|
||||
response = _responses_api_response_with_usage()
|
||||
logging_obj = Mock(spec=LiteLLMLoggingObj)
|
||||
logging_obj._response_cost_calculator.side_effect = RuntimeError("cost map unavailable")
|
||||
|
||||
_stamp_responses_usage_cost(response, logging_obj)
|
||||
|
||||
assert getattr(response.usage, "cost", None) is None
|
||||
|
|
|
|||
|
|
@ -26,9 +26,7 @@ import pytest
|
|||
|
||||
@pytest.fixture(scope="module")
|
||||
def model_data():
|
||||
json_path = os.path.join(
|
||||
os.path.dirname(__file__), "../../model_prices_and_context_window.json"
|
||||
)
|
||||
json_path = os.path.join(os.path.dirname(__file__), "../../model_prices_and_context_window.json")
|
||||
with open(json_path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
|
@ -51,21 +49,14 @@ def test_usgov_sonnet_4_5_pricing(model_data, model_key):
|
|||
info = model_data[model_key]
|
||||
|
||||
assert info["input_cost_per_token"] == 3.6e-06, (
|
||||
f"{model_key}: input_cost_per_token should be $3.60/MTok "
|
||||
f"(got {info['input_cost_per_token']})"
|
||||
f"{model_key}: input_cost_per_token should be $3.60/MTok (got {info['input_cost_per_token']})"
|
||||
)
|
||||
assert (
|
||||
info["output_cost_per_token"] == 1.8e-05
|
||||
), f"{model_key}: output_cost_per_token should be $18.00/MTok"
|
||||
assert (
|
||||
info["cache_creation_input_token_cost"] == 4.5e-06
|
||||
), f"{model_key}: 5m cache write should be $4.50/MTok"
|
||||
assert (
|
||||
info["cache_creation_input_token_cost_above_1hr"] == 7.2e-06
|
||||
), f"{model_key}: 1h cache write should be $7.20/MTok"
|
||||
assert (
|
||||
info["cache_read_input_token_cost"] == 3.6e-07
|
||||
), f"{model_key}: cache read should be $0.36/MTok"
|
||||
assert info["output_cost_per_token"] == 1.8e-05, f"{model_key}: output_cost_per_token should be $18.00/MTok"
|
||||
assert info["cache_creation_input_token_cost"] == 4.5e-06, f"{model_key}: 5m cache write should be $4.50/MTok"
|
||||
assert info["cache_creation_input_token_cost_above_1hr"] == 7.2e-06, (
|
||||
f"{model_key}: 1h cache write should be $7.20/MTok"
|
||||
)
|
||||
assert info["cache_read_input_token_cost"] == 3.6e-07, f"{model_key}: cache read should be $0.36/MTok"
|
||||
|
||||
|
||||
def test_usgov_carries_20_percent_premium_over_global(model_data):
|
||||
|
|
@ -84,9 +75,7 @@ def test_usgov_carries_20_percent_premium_over_global(model_data):
|
|||
"cache_read_input_token_cost",
|
||||
):
|
||||
ratio = usgov_info[field] / global_info[field]
|
||||
assert (
|
||||
abs(ratio - 1.2) < 1e-9
|
||||
), f"{field}: us-gov / global ratio is {ratio}, expected 1.2"
|
||||
assert abs(ratio - 1.2) < 1e-9, f"{field}: us-gov / global ratio is {ratio}, expected 1.2"
|
||||
|
||||
|
||||
# The us-gov.anthropic.* cross-region inference profile is the only us-gov
|
||||
|
|
@ -112,9 +101,7 @@ def test_usgov_cross_region_above_200k_carries_gov_premium(model_data, field, ex
|
|||
"""
|
||||
info = model_data[USGOV_CROSS_REGION_KEY]
|
||||
assert field in info, f"{USGOV_CROSS_REGION_KEY}: missing field {field}"
|
||||
assert (
|
||||
info[field] == expected
|
||||
), f"{USGOV_CROSS_REGION_KEY}: {field} should be {expected} (got {info[field]})"
|
||||
assert info[field] == expected, f"{USGOV_CROSS_REGION_KEY}: {field} should be {expected} (got {info[field]})"
|
||||
|
||||
|
||||
def test_usgov_cross_region_above_200k_ratio_to_global(model_data):
|
||||
|
|
@ -127,6 +114,176 @@ def test_usgov_cross_region_above_200k_ratio_to_global(model_data):
|
|||
usgov_info = model_data[USGOV_CROSS_REGION_KEY]
|
||||
for field in EXPECTED_USGOV_ABOVE_200K:
|
||||
ratio = usgov_info[field] / global_info[field]
|
||||
assert (
|
||||
abs(ratio - 1.2) < 1e-9
|
||||
), f"{field}: us-gov / global ratio is {ratio}, expected 1.2"
|
||||
assert abs(ratio - 1.2) < 1e-9, f"{field}: us-gov / global ratio is {ratio}, expected 1.2"
|
||||
|
||||
|
||||
CLAUDE_GOV_EXPECTED = {
|
||||
"anthropic.claude-sonnet-5": {
|
||||
"input_cost_per_token": 2.4e-06,
|
||||
"output_cost_per_token": 1.2e-05,
|
||||
"cache_creation_input_token_cost": 3e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 4.8e-06,
|
||||
"cache_read_input_token_cost": 2.4e-07,
|
||||
},
|
||||
"anthropic.claude-opus-4-8": {
|
||||
"input_cost_per_token": 6e-06,
|
||||
"output_cost_per_token": 3e-05,
|
||||
"cache_creation_input_token_cost": 7.5e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1.2e-05,
|
||||
"cache_read_input_token_cost": 6e-07,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
USGOV_CLAUDE_KEY_TEMPLATES = {
|
||||
"bedrock/us-gov-east-1/{base_key}": "bedrock",
|
||||
"bedrock/us-gov-west-1/{base_key}": "bedrock",
|
||||
"us-gov.{base_key}": "bedrock_converse",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("base_key", CLAUDE_GOV_EXPECTED)
|
||||
@pytest.mark.parametrize("key_template,expected_provider", USGOV_CLAUDE_KEY_TEMPLATES.items())
|
||||
def test_usgov_claude_sonnet5_opus48_pricing(model_data, key_template, expected_provider, base_key):
|
||||
"""Sonnet 5 and Opus 4.8 gov entries, both in-region keys and the us-gov.
|
||||
geo inference profile the model cards list for GovCloud, must match the
|
||||
rates AWS publishes on the Bedrock pricing page (1.2x global).
|
||||
"""
|
||||
gov_key = key_template.format(base_key=base_key)
|
||||
assert gov_key in model_data, f"Missing model entry: {gov_key}"
|
||||
info = model_data[gov_key]
|
||||
assert info["litellm_provider"] == expected_provider
|
||||
for field, expected in CLAUDE_GOV_EXPECTED[base_key].items():
|
||||
assert info[field] == expected, f"{gov_key}: {field} should be {expected} (got {info[field]})"
|
||||
ratio = info[field] / model_data[base_key][field]
|
||||
assert abs(ratio - 1.2) < 1e-9, f"{gov_key}: {field} gov/global ratio is {ratio}, expected 1.2"
|
||||
|
||||
|
||||
CONVERSE_GOV_EXPECTED = {
|
||||
"nvidia.nemotron-nano-3-30b": (7.2e-08, 2.88e-07),
|
||||
"nvidia.nemotron-nano-12b-v2": (2.4e-07, 7.2e-07),
|
||||
"nvidia.nemotron-super-3-120b": (1.8e-07, 7.8e-07),
|
||||
"openai.gpt-oss-20b-1:0": (8.4e-08, 3.6e-07),
|
||||
"openai.gpt-oss-120b-1:0": (1.8e-07, 7.2e-07),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("base_key", CONVERSE_GOV_EXPECTED)
|
||||
@pytest.mark.parametrize("region", ["us-gov-east-1", "us-gov-west-1"])
|
||||
def test_usgov_converse_model_pricing(model_data, region, base_key):
|
||||
"""Nemotron and gpt-oss gov entries must match the AWS Bedrock offer file,
|
||||
which prices both GovCloud regions identically at 1.2x commercial.
|
||||
"""
|
||||
gov_key = f"bedrock/{region}/{base_key}"
|
||||
assert gov_key in model_data, f"Missing model entry: {gov_key}"
|
||||
info = model_data[gov_key]
|
||||
expected_input, expected_output = CONVERSE_GOV_EXPECTED[base_key]
|
||||
assert info["input_cost_per_token"] == expected_input
|
||||
assert info["output_cost_per_token"] == expected_output
|
||||
assert info["litellm_provider"] == "bedrock"
|
||||
base = model_data[base_key]
|
||||
assert abs(info["input_cost_per_token"] / base["input_cost_per_token"] - 1.2) < 1e-9
|
||||
assert abs(info["output_cost_per_token"] / base["output_cost_per_token"] - 1.2) < 1e-9
|
||||
|
||||
|
||||
def test_usgov_west_llama3_8b_output_price_fixed(model_data):
|
||||
"""The us-gov-west-1 llama3-8b entry carried the 70B output rate ($2.65/MTok);
|
||||
the AWS Bedrock offer file prices output at $0.60/MTok. AWS lists the model
|
||||
in us-gov-west-1 only, so there is no east entry to check.
|
||||
"""
|
||||
info = model_data["bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0"]
|
||||
assert info["input_cost_per_token"] == 3e-07
|
||||
assert info["output_cost_per_token"] == 6e-07
|
||||
|
||||
|
||||
MANTLE_GOV_TIERED_EXPECTED = {
|
||||
"openai.gpt-5.6-luna": {
|
||||
"input_cost_per_token": 2.64e-07,
|
||||
"input_cost_per_token_above_272k_tokens": 5.28e-07,
|
||||
"cache_creation_input_token_cost": 3.3e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 6.6e-07,
|
||||
"cache_read_input_token_cost": 2.64e-08,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 5.28e-08,
|
||||
"output_cost_per_token": 1.584e-06,
|
||||
"output_cost_per_token_above_272k_tokens": 2.376e-06,
|
||||
},
|
||||
"openai.gpt-5.6-terra": {
|
||||
"input_cost_per_token": 2.64e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 5.28e-06,
|
||||
"cache_creation_input_token_cost": 3.3e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 6.6e-06,
|
||||
"cache_read_input_token_cost": 2.64e-07,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 5.28e-07,
|
||||
"output_cost_per_token": 1.584e-05,
|
||||
"output_cost_per_token_above_272k_tokens": 2.376e-05,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MANTLE_GOV_TIERED_EXPECTED)
|
||||
def test_usgov_west_mantle_terra_luna_pricing(model_data, model):
|
||||
"""Terra and Luna carry 1.2x commercial across every tier in the
|
||||
us-gov-west-1 offer file; the us-gov-east-1 offer file has no SKUs for them.
|
||||
"""
|
||||
gov_key = f"bedrock_mantle/us-gov-west-1/{model}"
|
||||
assert gov_key in model_data, f"Missing model entry: {gov_key}"
|
||||
info = model_data[gov_key]
|
||||
for field, expected in MANTLE_GOV_TIERED_EXPECTED[model].items():
|
||||
assert info[field] == expected, f"{gov_key}: {field} should be {expected} (got {info[field]})"
|
||||
assert info["litellm_provider"] == "bedrock_mantle"
|
||||
assert f"bedrock_mantle/us-gov-east-1/{model}" not in model_data
|
||||
|
||||
|
||||
@pytest.mark.parametrize("region", ["us-gov-east-1", "us-gov-west-1"])
|
||||
def test_usgov_mantle_gpt_5_4_pricing_has_no_long_context_tier(model_data, region):
|
||||
"""gpt-5.4 gov rates come from the offer file, which publishes only the
|
||||
standard tier in GovCloud: no long-context SKUs exist there, unlike commercial.
|
||||
"""
|
||||
gov_key = f"bedrock_mantle/{region}/openai.gpt-5.4"
|
||||
assert gov_key in model_data, f"Missing model entry: {gov_key}"
|
||||
info = model_data[gov_key]
|
||||
assert info["input_cost_per_token"] == 3.3e-06
|
||||
assert info["cache_read_input_token_cost"] == 3.3e-07
|
||||
assert info["output_cost_per_token"] == 1.98e-05
|
||||
assert not any(field.endswith("_above_272k_tokens") for field in info)
|
||||
|
||||
|
||||
def test_usgov_mantle_grok_4_3_west_only(model_data):
|
||||
"""grok-4.3 is priced in the us-gov-west-1 offer file only; the east offer
|
||||
file carries grok-4.6 instead.
|
||||
"""
|
||||
info = model_data["bedrock_mantle/us-gov-west-1/xai.grok-4.3"]
|
||||
assert info["input_cost_per_token"] == 1.5e-06
|
||||
assert info["output_cost_per_token"] == 3e-06
|
||||
assert info["cache_read_input_token_cost"] == 2.4e-07
|
||||
assert "bedrock_mantle/us-gov-east-1/xai.grok-4.3" not in model_data
|
||||
|
||||
|
||||
AZURE_GOV_EXPECTED = {
|
||||
"azure/us-gov/gpt-5.1": {
|
||||
"input_cost_per_token": 1.71875e-06,
|
||||
"cache_read_input_token_cost": 1.71875e-07,
|
||||
"output_cost_per_token": 1.375e-05,
|
||||
},
|
||||
"azure/us-gov/o3-mini": {
|
||||
"input_cost_per_token": 1.513e-06,
|
||||
"cache_read_input_token_cost": 7.57e-07,
|
||||
"output_cost_per_token": 6.05e-06,
|
||||
},
|
||||
"azure/us-gov/text-embedding-3-large": {"input_cost_per_token": 1.63e-07},
|
||||
"azure/us-gov/text-embedding-3-small": {"input_cost_per_token": 2.5e-08},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gov_key", AZURE_GOV_EXPECTED)
|
||||
def test_azure_usgov_pricing(model_data, gov_key):
|
||||
"""Azure Government meters from the Azure retail prices API
|
||||
(usgovvirginia/usgovarizona, serviceName 'Foundry Models'). No Government
|
||||
retirement schedule is published, so these entries carry no deprecation_date.
|
||||
"""
|
||||
assert gov_key in model_data, f"Missing model entry: {gov_key}"
|
||||
info = model_data[gov_key]
|
||||
for field, expected in AZURE_GOV_EXPECTED[gov_key].items():
|
||||
assert info[field] == expected, f"{gov_key}: {field} should be {expected} (got {info[field]})"
|
||||
assert info["litellm_provider"] == "azure"
|
||||
assert "deprecation_date" not in info
|
||||
|
|
|
|||
|
|
@ -75,6 +75,22 @@ def test_additional_current_models_are_present():
|
|||
assert entry["output_cost_per_token"] > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"key, published_price_per_audio_minute",
|
||||
[
|
||||
("cloudflare/@cf/openai/whisper", 0.00045),
|
||||
("cloudflare/@cf/openai/whisper-large-v3-turbo", 0.00051),
|
||||
],
|
||||
)
|
||||
def test_whisper_transcription_pricing_is_stored_per_second(key, published_price_per_audio_minute):
|
||||
entry = litellm.model_cost[key]
|
||||
assert entry["litellm_provider"] == "cloudflare"
|
||||
assert entry["mode"] == "audio_transcription"
|
||||
assert entry["supported_endpoints"] == ["/v1/audio/transcriptions"]
|
||||
assert entry["output_cost_per_second"] == 0.0
|
||||
assert entry["input_cost_per_second"] == pytest.approx(published_price_per_audio_minute / 60)
|
||||
|
||||
|
||||
def test_root_and_backup_have_identical_cloudflare_keys():
|
||||
if not os.path.exists(ROOT_MAP):
|
||||
pytest.skip("root cost map only ships in source checkouts")
|
||||
|
|
|
|||
|
|
@ -3150,8 +3150,8 @@ def _stream_builder_logging_obj() -> LiteLLMLogging:
|
|||
return logging_obj
|
||||
|
||||
|
||||
def test_stream_chunk_builder_reports_streaming_usage_cost_when_enabled(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True)
|
||||
def test_stream_chunk_builder_stamps_streaming_usage_cost_by_default(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False)
|
||||
chunks: Final = [
|
||||
_stream_builder_text_chunk("gpt-4o", "Hello "),
|
||||
_stream_builder_text_chunk("gpt-4o", "world.", finish_reason="stop"),
|
||||
|
|
@ -3168,11 +3168,45 @@ def test_stream_chunk_builder_reports_streaming_usage_cost_when_enabled(monkeypa
|
|||
assert response._hidden_params["response_cost"] == pytest.approx(usage_cost)
|
||||
|
||||
|
||||
def test_stream_chunk_builder_defers_cost_to_logging_obj_when_usage_cost_absent(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False)
|
||||
def test_stream_chunk_builder_skips_stamp_when_cost_is_unpriceable():
|
||||
import time as time_module
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
|
||||
logging_obj: Final = LiteLLMLogging(
|
||||
model="us.anthropic.claude-opus-5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
call_type="completion",
|
||||
start_time=time_module.time(),
|
||||
litellm_call_id="stream-builder-alias-unpriceable",
|
||||
function_id="1",
|
||||
)
|
||||
logging_obj.model_call_details["custom_llm_provider"] = "bedrock"
|
||||
logging_obj.optional_params = {}
|
||||
usage_chunk: Final = _stream_builder_text_chunk("bedrock-claude-opus-5", "")
|
||||
usage_chunk.usage = Usage(prompt_tokens=40, completion_tokens=5, total_tokens=45)
|
||||
chunks: Final = [
|
||||
_stream_builder_text_chunk("bedrock-claude-opus-5", "Hello ", finish_reason="stop"),
|
||||
usage_chunk,
|
||||
]
|
||||
|
||||
response: Final = litellm.stream_chunk_builder(
|
||||
chunks=chunks, messages=[{"role": "user", "content": "hi"}], logging_obj=logging_obj
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
assert getattr(response.usage, "cost", None) is None
|
||||
assert response._hidden_params.get("response_cost") is None
|
||||
|
||||
|
||||
def test_stream_chunk_builder_keeps_provider_reported_usage_cost():
|
||||
usage_chunk: Final = _stream_builder_text_chunk("gpt-4o", "")
|
||||
usage_chunk.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, cost=0.5)
|
||||
chunks: Final = [
|
||||
_stream_builder_text_chunk("gpt-4o", "Hello "),
|
||||
_stream_builder_text_chunk("gpt-4o", "world.", finish_reason="stop"),
|
||||
usage_chunk,
|
||||
]
|
||||
|
||||
response: Final = litellm.stream_chunk_builder(
|
||||
|
|
@ -3180,4 +3214,26 @@ def test_stream_chunk_builder_defers_cost_to_logging_obj_when_usage_cost_absent(
|
|||
)
|
||||
|
||||
assert response is not None
|
||||
assert response._hidden_params.get("response_cost") is None
|
||||
assert getattr(response.usage, "cost", None) == pytest.approx(0.5)
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(0.5)
|
||||
|
||||
|
||||
def test_stream_chunk_builder_prices_alias_from_openai_sdk_usage_chunk():
|
||||
from openai.types.completion_usage import CompletionUsage
|
||||
|
||||
usage_chunk: Final = _stream_builder_text_chunk("mantle-claude", "")
|
||||
usage_chunk.usage = CompletionUsage(prompt_tokens=20, completion_tokens=60, total_tokens=80, cost=0.000704)
|
||||
assert type(usage_chunk.usage) is CompletionUsage
|
||||
chunks: Final = [
|
||||
_stream_builder_text_chunk("mantle-claude", "Hello "),
|
||||
_stream_builder_text_chunk("mantle-claude", "world.", finish_reason="stop"),
|
||||
usage_chunk,
|
||||
]
|
||||
|
||||
response: Final = litellm.stream_chunk_builder(chunks=chunks, messages=[{"role": "user", "content": "hi"}])
|
||||
|
||||
assert response is not None
|
||||
assert response.usage.prompt_tokens == 20
|
||||
assert response.usage.completion_tokens == 60
|
||||
assert getattr(response.usage, "cost", None) == pytest.approx(0.000704)
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(0.000704)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,156 @@
|
|||
import json
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[2]
|
||||
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
|
||||
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
|
||||
FLEX_LONG_CONTEXT = {
|
||||
"gpt-5.4": {
|
||||
"input_cost_per_token_above_272k_tokens_flex": 2.5e-06,
|
||||
"output_cost_per_token_above_272k_tokens_flex": 1.125e-05,
|
||||
"cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07,
|
||||
},
|
||||
"gpt-5.4-pro": {
|
||||
"input_cost_per_token_above_272k_tokens_flex": 3e-05,
|
||||
"output_cost_per_token_above_272k_tokens_flex": 0.000135,
|
||||
},
|
||||
"gpt-5.5": {
|
||||
"input_cost_per_token_above_272k_tokens_flex": 5e-06,
|
||||
"output_cost_per_token_above_272k_tokens_flex": 2.25e-05,
|
||||
"cache_read_input_token_cost_above_272k_tokens_flex": 5e-07,
|
||||
},
|
||||
}
|
||||
|
||||
PRIORITY_LONG_CONTEXT = {
|
||||
"gpt-5.6": {
|
||||
"input_cost_per_token_above_272k_tokens_priority": 1.6e-05,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 6e-05,
|
||||
"cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
|
||||
},
|
||||
"gpt-5.6-sol": {
|
||||
"input_cost_per_token_above_272k_tokens_priority": 1.6e-05,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 6e-05,
|
||||
"cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
|
||||
},
|
||||
"gpt-5.6-terra": {
|
||||
"input_cost_per_token_above_272k_tokens_priority": 8e-06,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 3.6e-05,
|
||||
"cache_read_input_token_cost_above_272k_tokens_priority": 8e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
|
||||
},
|
||||
"gpt-5.6-luna": {
|
||||
"input_cost_per_token_above_272k_tokens_priority": 8e-07,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 3.6e-06,
|
||||
"cache_read_input_token_cost_above_272k_tokens_priority": 8e-08,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06,
|
||||
},
|
||||
}
|
||||
|
||||
EXPECTED = {**FLEX_LONG_CONTEXT, **PRIORITY_LONG_CONTEXT}
|
||||
|
||||
NO_PUBLISHED_PRIORITY_LONG_CONTEXT = ("gpt-5.4", "gpt-5.5")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
|
||||
|
||||
@lru_cache(maxsize=2)
|
||||
def _load(path: Path) -> dict[str, dict[str, object]]:
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", [MAIN_PATH, BACKUP_PATH], ids=["main", "backup"])
|
||||
@pytest.mark.parametrize("model", sorted(EXPECTED))
|
||||
def test_service_tier_long_context_rates_are_published(model: str, path: Path) -> None:
|
||||
"""Each tier must carry its own above-272K rates, in both price files."""
|
||||
info = _load(path).get(model)
|
||||
assert info is not None, f"{model} not found in {path.name}"
|
||||
for key, expected in EXPECTED[model].items():
|
||||
assert info.get(key) == pytest.approx(expected), f"{model}.{key} is {info.get(key)!r}, expected {expected!r}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", sorted(EXPECTED))
|
||||
def test_tier_long_context_rate_is_half_or_double_the_standard(model: str) -> None:
|
||||
"""Flex is half the standard long-context rate; priority is double it."""
|
||||
info = _load(MAIN_PATH)[model]
|
||||
tier = "flex" if model in FLEX_LONG_CONTEXT else "priority"
|
||||
ratio = 0.5 if tier == "flex" else 2.0
|
||||
for base in ("input_cost_per_token", "output_cost_per_token"):
|
||||
standard = info[f"{base}_above_272k_tokens"]
|
||||
tiered = info[f"{base}_above_272k_tokens_{tier}"]
|
||||
assert tiered == pytest.approx(standard * ratio), (
|
||||
f"{model}.{base}_above_272k_tokens_{tier} is {tiered!r}, "
|
||||
f"expected {ratio}x the standard long-context rate {standard!r}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", NO_PUBLISHED_PRIORITY_LONG_CONTEXT)
|
||||
def test_no_priority_long_context_rates_where_openai_publishes_none(model: str) -> None:
|
||||
"""Guard against back-filling a rate OpenAI does not publish."""
|
||||
info = _load(MAIN_PATH)[model]
|
||||
assert "input_cost_per_token_above_272k_tokens_priority" not in info
|
||||
|
||||
|
||||
LONG_CONTEXT_PROMPT_TOKENS = 300_000
|
||||
COMPLETION_TOKENS = 1_000
|
||||
|
||||
TIERED_COST_CASES = [
|
||||
("gpt-5.4", "flex", 2.5e-06, 1.125e-05),
|
||||
("gpt-5.4-pro", "flex", 3e-05, 0.000135),
|
||||
("gpt-5.5", "flex", 5e-06, 2.25e-05),
|
||||
("gpt-5.6", "priority", 1.6e-05, 6e-05),
|
||||
("gpt-5.6-sol", "priority", 1.6e-05, 6e-05),
|
||||
("gpt-5.6-terra", "priority", 8e-06, 3.6e-05),
|
||||
("gpt-5.6-luna", "priority", 8e-07, 3.6e-06),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model,tier,input_rate,output_rate", TIERED_COST_CASES)
|
||||
def test_cost_per_token_bills_long_context_at_the_tier_rate(
|
||||
model: str, tier: str, input_rate: float, output_rate: float
|
||||
) -> None:
|
||||
"""A prompt over 272K on flex or priority must bill at that tier's long-context rate."""
|
||||
input_cost, output_cost = litellm.cost_per_token(
|
||||
model=model,
|
||||
prompt_tokens=LONG_CONTEXT_PROMPT_TOKENS,
|
||||
completion_tokens=COMPLETION_TOKENS,
|
||||
service_tier=tier,
|
||||
)
|
||||
assert input_cost == pytest.approx(LONG_CONTEXT_PROMPT_TOKENS * input_rate)
|
||||
assert output_cost == pytest.approx(COMPLETION_TOKENS * output_rate)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model,tier,input_rate,output_rate", TIERED_COST_CASES)
|
||||
def test_cost_per_token_tier_differs_from_the_standard_long_context_cost(
|
||||
model: str, tier: str, input_rate: float, output_rate: float
|
||||
) -> None:
|
||||
"""Flex halves the standard long-context bill and priority doubles it."""
|
||||
ratio = 0.5 if tier == "flex" else 2.0
|
||||
standard = sum(
|
||||
litellm.cost_per_token(
|
||||
model=model,
|
||||
prompt_tokens=LONG_CONTEXT_PROMPT_TOKENS,
|
||||
completion_tokens=COMPLETION_TOKENS,
|
||||
)
|
||||
)
|
||||
tiered = sum(
|
||||
litellm.cost_per_token(
|
||||
model=model,
|
||||
prompt_tokens=LONG_CONTEXT_PROMPT_TOKENS,
|
||||
completion_tokens=COMPLETION_TOKENS,
|
||||
service_tier=tier,
|
||||
)
|
||||
)
|
||||
assert tiered == pytest.approx(standard * ratio)
|
||||
|
|
@ -7271,6 +7271,71 @@ def test_get_configured_token_limits_coerces_numeric_strings():
|
|||
assert router.get_configured_token_limits("quoted-limits-model") == (32000, 8000)
|
||||
|
||||
|
||||
def test_get_configured_display_name_reads_deployment_model_info():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "Kimi K3-claude-compatible",
|
||||
"litellm_params": {"model": "openai/some-unmapped-model"},
|
||||
"model_info": {"display_name": "Kimi K3"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
assert router.get_configured_display_name("Kimi K3-claude-compatible") == "Kimi K3"
|
||||
|
||||
|
||||
def test_get_configured_display_name_returns_none_for_unset_or_unknown():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "no-display-model",
|
||||
"litellm_params": {"model": "openai/some-unmapped-model"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
assert router.get_configured_display_name("no-display-model") is None
|
||||
assert router.get_configured_display_name("not-a-real-model") is None
|
||||
|
||||
|
||||
def test_get_configured_display_name_skips_wildcard_pattern_matching():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "bedrock/*",
|
||||
"litellm_params": {"model": "bedrock/*"},
|
||||
"model_info": {"display_name": "Bedrock"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
router.pattern_router, "route", side_effect=AssertionError("pattern route called")
|
||||
):
|
||||
assert (
|
||||
router.get_configured_display_name("bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0")
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_get_configured_display_name_treats_malformed_values_as_absent():
|
||||
malformed = ["", " ", 12345, ["Kimi K3"], {"name": "Kimi K3"}, True]
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": f"bad-display-{i}",
|
||||
"litellm_params": {"model": "openai/some-unmapped-model"},
|
||||
"model_info": {"display_name": bad},
|
||||
}
|
||||
for i, bad in enumerate(malformed)
|
||||
]
|
||||
)
|
||||
|
||||
for i in range(len(malformed)):
|
||||
assert router.get_configured_display_name(f"bad-display-{i}") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acreate_batch_disable_fallbacks_surfaces_owning_provider_error():
|
||||
router = litellm.Router(
|
||||
|
|
|
|||
|
|
@ -4655,6 +4655,7 @@ GEMINI_4096_CACHE_MIN_MODELS: Final = tuple(
|
|||
"gemini-3.5-flash",
|
||||
"gemini-3.6-flash",
|
||||
"gemini-3.7-flash",
|
||||
"gemini-3.8-flash",
|
||||
"gemini-3.1-pro-preview",
|
||||
"gemini-3.1-pro-preview-customtools",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -217,3 +217,17 @@ bedrock/us-east-1/zai.glm-5
|
|||
bedrock/us-west-2/zai.glm-5
|
||||
bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0
|
||||
bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0
|
||||
bedrock/us-gov-west-1/nvidia.nemotron-nano-3-30b
|
||||
bedrock/us-gov-west-1/nvidia.nemotron-nano-12b-v2
|
||||
bedrock/us-gov-west-1/nvidia.nemotron-super-3-120b
|
||||
bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0
|
||||
bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0
|
||||
bedrock/us-gov-west-1/anthropic.claude-sonnet-5
|
||||
bedrock/us-gov-west-1/anthropic.claude-opus-4-8
|
||||
bedrock/us-gov-east-1/nvidia.nemotron-nano-3-30b
|
||||
bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2
|
||||
bedrock/us-gov-east-1/nvidia.nemotron-super-3-120b
|
||||
bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0
|
||||
bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0
|
||||
bedrock/us-gov-east-1/anthropic.claude-sonnet-5
|
||||
bedrock/us-gov-east-1/anthropic.claude-opus-4-8
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue