scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-03 02:22:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 81

Merge remote-tracking branch 'origin/litellm-oss-staging-04-25-2026' into litellm_oss_staging_04_25_2026

Browse source
This commit is contained in:
mateo-berri 2026-04-25 15:45:48 -07:00
commit 0644a1b02b
10 changed files with 477 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
litellm/constants.py
View file

@ -409,9 +409,6 @@ CACHED_STREAMING_CHUNK_DELAY = float(os.getenv("CACHED_STREAMING_CHUNK_DELAY", 0
AUDIO_SPEECH_CHUNK_SIZE = int(
os.getenv("AUDIO_SPEECH_CHUNK_SIZE", 8192)
) # chunk_size for audio speech streaming. Balance between latency and memory usage
MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int(
os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 512)
)
DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2000))
#### Networking settings ####
# Sentinel used when `REQUEST_TIMEOUT` is unset: `litellm.request_timeout` keeps this

42
litellm/integrations/arize/_utils.py
View file

@ -220,23 +220,57 @@ def _set_structured_outputs(span: "Span", response_obj, msg_attrs, span_attrs):
safe_set_attribute(span, f"{prefix}.{msg_attrs.MESSAGE_ROLE}", message_role)
def _safe_get(obj, key, default=None):
"""Read ``key`` from a dict-like or Pydantic-model-like object.
The arize/langfuse_otel logger receives ``usage`` objects from many sources:
plain dicts, litellm ``Usage`` (which exposes ``.get``), and raw OpenAI
Pydantic models (e.g. ``openai.types.completion_usage.CompletionUsage`` and
nested ``CompletionTokensDetails`` / ``OutputTokensDetails``) which do NOT
expose ``.get``. Calling ``.get`` on the latter raised ``AttributeError`` —
see https://github.com/BerriAI/litellm/issues/13672.
"""
if obj is None:
return default
getter = getattr(obj, "get", None)
if callable(getter):
try:
return getter(key, default)
except TypeError:
# Some objects expose `.get` with a different signature
pass
return getattr(obj, key, default)
def _set_usage_outputs(span: "Span", response_obj, span_attrs):
usage = response_obj and response_obj.get("usage")
if not usage:
return
safe_set_attribute(
span, span_attrs.LLM_TOKEN_COUNT_TOTAL, usage.get("total_tokens")
span, span_attrs.LLM_TOKEN_COUNT_TOTAL, _safe_get(usage, "total_tokens")
)
completion_tokens = _safe_get(usage, "completion_tokens") or _safe_get(
usage, "output_tokens"
)
completion_tokens = usage.get("completion_tokens") or usage.get("output_tokens")
if completion_tokens:
safe_set_attribute(
span, span_attrs.LLM_TOKEN_COUNT_COMPLETION, completion_tokens
)
prompt_tokens = usage.get("prompt_tokens") or usage.get("input_tokens")
prompt_tokens = _safe_get(usage, "prompt_tokens") or _safe_get(
usage, "input_tokens"
)
if prompt_tokens:
safe_set_attribute(span, span_attrs.LLM_TOKEN_COUNT_PROMPT, prompt_tokens)
reasoning_tokens = usage.get("output_tokens_details", {}).get("reasoning_tokens")
# Reasoning tokens live in `completion_tokens_details` for Chat Completions
# API (Usage) and in `output_tokens_details` for Responses API
# (ResponseAPIUsage). Both nested objects may be plain Pydantic models
# without `.get`.
token_details = _safe_get(usage, "completion_tokens_details") or _safe_get(
usage, "output_tokens_details"
)
reasoning_tokens = _safe_get(token_details, "reasoning_tokens")
if reasoning_tokens:
safe_set_attribute(
span,

6
litellm/llms/cloudflare/chat/transformation.py
View file

@ -147,9 +147,9 @@ class CloudflareChatConfig(BaseConfig):
) -> ModelResponse:
completion_response = raw_response.json()
model_response.choices[0].message.content = completion_response["result"][ # type: ignore
"response"
]
# Support both "response" and "response_text" keys (newer models like Nemotron use "response_text")
result = completion_response["result"]
model_response.choices[0].message.content = result.get("response") if result.get("response") is not None else result.get("response_text", "") # type: ignore
prompt_tokens = litellm.utils.get_token_count(messages=messages, model=model)
completion_tokens = len(

163
litellm/model_prices_and_context_window_backup.json
View file

@ -4645,6 +4645,169 @@
"supports_vision": true,
"supports_web_search": true
},
"azure/gpt-5.5": {
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1e-06,
"cache_read_input_token_cost_priority": 1e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 2e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_272k_tokens": 1e-05,
"input_cost_per_token_priority": 1e-05,
"input_cost_per_token_above_272k_tokens_priority": 2e-05,
"litellm_provider": "azure",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3e-05,
"output_cost_per_token_above_272k_tokens": 4.5e-05,
"output_cost_per_token_priority": 6e-05,
"output_cost_per_token_above_272k_tokens_priority": 9e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure/gpt-5.5-2026-04-23": {
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1e-06,
"cache_read_input_token_cost_priority": 1e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 2e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_272k_tokens": 1e-05,
"input_cost_per_token_priority": 1e-05,
"input_cost_per_token_above_272k_tokens_priority": 2e-05,
"litellm_provider": "azure",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3e-05,
"output_cost_per_token_above_272k_tokens": 4.5e-05,
"output_cost_per_token_priority": 6e-05,
"output_cost_per_token_above_272k_tokens_priority": 9e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true
},
"azure/gpt-5.5-pro": {
"cache_read_input_token_cost": 6e-06,
"cache_read_input_token_cost_above_272k_tokens": 1.2e-05,
"input_cost_per_token": 6e-05,
"input_cost_per_token_above_272k_tokens": 0.00012,
"litellm_provider": "azure",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 0.00036,
"output_cost_per_token_above_272k_tokens": 0.00054,
"supported_endpoints": [
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": false,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"supports_low_reasoning_effort": false
},
"azure/gpt-5.5-pro-2026-04-23": {
"cache_read_input_token_cost": 6e-06,
"cache_read_input_token_cost_above_272k_tokens": 1.2e-05,
"input_cost_per_token": 6e-05,
"input_cost_per_token_above_272k_tokens": 0.00012,
"litellm_provider": "azure",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 0.00036,
"output_cost_per_token_above_272k_tokens": 0.00054,
"supported_endpoints": [
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true
},
"azure/gpt-5.4-mini": {
"cache_read_input_token_cost": 7.5e-08,
"input_cost_per_token": 7.5e-07,

7
litellm/proxy/proxy_server.py
View file

@ -5845,10 +5845,15 @@ async def initialize( # noqa: PLR0915
if litellm_log_setting.upper() == "INFO":
import logging
from litellm._logging import verbose_proxy_logger, verbose_router_logger
from litellm._logging import (
verbose_logger,
verbose_proxy_logger,
verbose_router_logger,
)
# this must ALWAYS remain logging.INFO, DO NOT MODIFY THIS
verbose_logger.setLevel(level=logging.INFO) # set package log to info
verbose_router_logger.setLevel(
level=logging.INFO
) # set router logs to info

163
model_prices_and_context_window.json
View file

@ -4659,6 +4659,169 @@
"supports_vision": true,
"supports_web_search": true
},
"azure/gpt-5.5": {
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1e-06,
"cache_read_input_token_cost_priority": 1e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 2e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_272k_tokens": 1e-05,
"input_cost_per_token_priority": 1e-05,
"input_cost_per_token_above_272k_tokens_priority": 2e-05,
"litellm_provider": "azure",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3e-05,
"output_cost_per_token_above_272k_tokens": 4.5e-05,
"output_cost_per_token_priority": 6e-05,
"output_cost_per_token_above_272k_tokens_priority": 9e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure/gpt-5.5-2026-04-23": {
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1e-06,
"cache_read_input_token_cost_priority": 1e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 2e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_272k_tokens": 1e-05,
"input_cost_per_token_priority": 1e-05,
"input_cost_per_token_above_272k_tokens_priority": 2e-05,
"litellm_provider": "azure",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3e-05,
"output_cost_per_token_above_272k_tokens": 4.5e-05,
"output_cost_per_token_priority": 6e-05,
"output_cost_per_token_above_272k_tokens_priority": 9e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true
},
"azure/gpt-5.5-pro": {
"cache_read_input_token_cost": 6e-06,
"cache_read_input_token_cost_above_272k_tokens": 1.2e-05,
"input_cost_per_token": 6e-05,
"input_cost_per_token_above_272k_tokens": 0.00012,
"litellm_provider": "azure",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 0.00036,
"output_cost_per_token_above_272k_tokens": 0.00054,
"supported_endpoints": [
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": false,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"supports_low_reasoning_effort": false
},
"azure/gpt-5.5-pro-2026-04-23": {
"cache_read_input_token_cost": 6e-06,
"cache_read_input_token_cost_above_272k_tokens": 1.2e-05,
"input_cost_per_token": 6e-05,
"input_cost_per_token_above_272k_tokens": 0.00012,
"litellm_provider": "azure",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 0.00036,
"output_cost_per_token_above_272k_tokens": 0.00054,
"supported_endpoints": [
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true
},
"azure/gpt-5.4-mini": {
"cache_read_input_token_cost": 7.5e-08,
"input_cost_per_token": 7.5e-07,

85
tests/test_litellm/integrations/arize/test_arize_utils.py
View file

@ -273,6 +273,91 @@ def test_arize_set_attributes_responses_api():
)
def test_set_usage_outputs_pydantic_completion_usage():
"""
Regression test for https://github.com/BerriAI/litellm/issues/13672
`_set_usage_outputs` previously called `usage.get(...)` which crashes when
`usage` is a plain Pydantic model (e.g. openai.types.completion_usage.CompletionUsage)
that does not implement dict-style `.get()`. Same crash for nested
`output_tokens_details` / `completion_tokens_details`.
The function must:
1. Read total/prompt/completion tokens from a Pydantic usage without `.get`.
2. Read reasoning_tokens from `completion_tokens_details` (chat completions API)
OR `output_tokens_details` (responses API), even when those nested objects
are Pydantic models without `.get`.
3. Not raise AttributeError; not call span.record_exception.
"""
from unittest.mock import MagicMock
from openai.types.completion_usage import (
CompletionTokensDetails,
CompletionUsage,
)
from litellm.integrations.arize._utils import _set_usage_outputs
span = MagicMock()
# Plain OpenAI Pydantic model — has no `.get()`
usage = CompletionUsage(
completion_tokens=60,
prompt_tokens=40,
total_tokens=100,
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=25),
)
assert not hasattr(usage, "get"), "precondition: CompletionUsage must lack .get"
response_obj = {"usage": usage}
# Must not raise
_set_usage_outputs(span, response_obj, SpanAttributes)
span.set_attribute.assert_any_call(SpanAttributes.LLM_TOKEN_COUNT_TOTAL, 100)
span.set_attribute.assert_any_call(SpanAttributes.LLM_TOKEN_COUNT_PROMPT, 40)
span.set_attribute.assert_any_call(SpanAttributes.LLM_TOKEN_COUNT_COMPLETION, 60)
# reasoning_tokens for chat completions live in completion_tokens_details
span.set_attribute.assert_any_call(
SpanAttributes.LLM_TOKEN_COUNT_COMPLETION_DETAILS_REASONING, 25
)
def test_set_usage_outputs_pydantic_response_api_usage():
"""
Same crash also affects Responses API with `output_tokens_details` as a
Pydantic model that lacks `.get()`. Verifies the responses-API path.
"""
from unittest.mock import MagicMock
from litellm.integrations.arize._utils import _set_usage_outputs
from litellm.types.llms.openai import OutputTokensDetails
# Build an object that mimics openai ResponsesAPI usage but lacks `.get`
# (uses a plain class — not BaseLiteLLMOpenAIResponseObject)
class PlainResponsesUsage:
def __init__(self):
self.total_tokens = 370
self.input_tokens = 120
self.output_tokens = 250
self.output_tokens_details = OutputTokensDetails(reasoning_tokens=180)
usage = PlainResponsesUsage()
assert not hasattr(usage, "get")
span = MagicMock()
response_obj = {"usage": usage}
_set_usage_outputs(span, response_obj, SpanAttributes)
span.set_attribute.assert_any_call(SpanAttributes.LLM_TOKEN_COUNT_TOTAL, 370)
span.set_attribute.assert_any_call(SpanAttributes.LLM_TOKEN_COUNT_PROMPT, 120)
span.set_attribute.assert_any_call(SpanAttributes.LLM_TOKEN_COUNT_COMPLETION, 250)
span.set_attribute.assert_any_call(
SpanAttributes.LLM_TOKEN_COUNT_COMPLETION_DETAILS_REASONING, 180
)
class TestArizeLogger(CustomLogger):
"""
Custom logger implementation to capture standard_callback_dynamic_params.

1
tests/test_litellm/test_utils.py
View file

@ -769,6 +769,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"uses_embed_content": {"type": "boolean"},
"supports_reasoning": {"type": "boolean"},
"supports_minimal_reasoning_effort": {"type": "boolean"},
"supports_low_reasoning_effort": {"type": "boolean"},
"supports_none_reasoning_effort": {"type": "boolean"},
"supports_xhigh_reasoning_effort": {"type": "boolean"},
"supports_max_reasoning_effort": {"type": "boolean"},

14
ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx
View file

@ -68,6 +68,16 @@ describe("provider_info_helpers", () => {
expect(result.logo).toBe(providerLogoMap[Providers.OpenAI]);
});
it("should resolve the zai (Z.AI) provider value to the Z.AI display name", () => {
// Regression test for https://github.com/BerriAI/litellm/issues/25482 —
// the backend already returns `zai` from /public/providers and the docs
// have a dedicated page, but the UI dropdown was missing an entry, so
// `getProviderLogoAndName("zai")` previously returned the raw value as
// the display name (no mapping).
const result = getProviderLogoAndName("zai");
expect(result.displayName).toBe(Providers.ZAI);
});
it("should return provider value as display name when no mapping exists", () => {
const unknownProvider = "unknown_provider";
const result = getProviderLogoAndName(unknownProvider);
@ -156,6 +166,10 @@ describe("provider_info_helpers", () => {
expect(getPlaceholder(Providers.WATSONX)).toBe("watsonx/ibm/granite-3-3-8b-instruct");
});
it("should return zai/glm-4.5 placeholder for Z.AI provider", () => {
expect(getPlaceholder(Providers.ZAI)).toBe("zai/glm-4.5");
});
it("should return default gpt-3.5-turbo placeholder for unknown provider", () => {
expect(getPlaceholder("UnknownProvider" as any)).toBe("gpt-3.5-turbo");
});

4
ui/litellm-dashboard/src/components/provider_info_helpers.tsx
View file

@ -103,6 +103,7 @@ export enum Providers {
WATSONX_TEXT = "Watsonx Text",
xAI = "xAI",
XINFERENCE = "Xinference",
ZAI = "Z.AI (Zhipu AI)",
}
export const provider_map: Record<string, string> = {
@ -210,6 +211,7 @@ export const provider_map: Record<string, string> = {
WATSONX_TEXT: "watsonx_text",
xAI: "xai",
XINFERENCE: "xinference",
ZAI: "zai",
};
const asset_logos_folder = "../ui/assets/logos/";
@ -366,6 +368,8 @@ export const getPlaceholder = (selectedProvider: string): string => {
return "watsonx/ibm/granite-3-3-8b-instruct";
} else if (selectedProvider === Providers.Cursor) {
return "cursor/claude-4-sonnet";
} else if (selectedProvider === Providers.ZAI) {
return "zai/glm-4.5";
} else {
return "gpt-3.5-turbo";
}