From 278c9babc6ff75d7f0301b5b7f02afc266557d0b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 16 Mar 2026 15:32:20 -0700 Subject: [PATCH 1/4] [Infra] Merging RC Branch with Main (#23786) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(test): add missing mocks for test_streamable_http_mcp_handler_mock The test was missing mocks for extract_mcp_auth_context and set_auth_context, causing the handler to fail silently in the except block instead of reaching session_manager.handle_request. This mirrors the fix already applied to the sibling test_sse_mcp_handler_mock. Co-authored-by: Ishaan Jaff * fix(ci): route OpenAI models through chat completions in pass-through tests The test_anthropic_messages_openai_model_streaming_cost_injection test fails because the OpenAI Responses API returns 400 for requests routed through the Anthropic Messages endpoint. Setting LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true routes OpenAI models through the stable chat completions path instead. Cost injection still works since it happens at the proxy level. Co-authored-by: Ishaan Jaff * fix(ci): fix assemblyai custom auth and router wildcard test flakiness 1. custom_auth_basic.py: Add user_role='proxy_admin' so the custom auth user can access management endpoints like /key/generate. The test test_assemblyai_transcribe_with_non_admin_key was hidden behind an earlier -x failure and was never reached before. 2. test_router_utils.py: Add flaky(retries=3) and increase sleep from 1s to 2s for test_router_get_model_group_usage_wildcard_routes. The async callback needs time to write usage to cache, and 1s is insufficient on slower CI hardware. Co-authored-by: Ishaan Jaff * ci: retrigger CI pipeline Co-authored-by: Ishaan Jaff * fix(mypy): use LitellmUserRoles enum instead of raw string in custom_auth_basic Fixes mypy error: Argument 'user_role' has incompatible type 'str'; expected 'LitellmUserRoles | None' Co-authored-by: Ishaan Jaff * fix: don't close HTTP/SDK clients on LLMClientCache eviction (#22926) * fix: don't close HTTP/SDK clients on LLMClientCache eviction Removing the _remove_key override that eagerly called aclose()/close() on evicted clients. Evicted clients may still be held by in-flight streaming requests; closing them causes: RuntimeError: Cannot send a request, as the client has been closed. This is a regression from commit fb72979432. Clients that are no longer referenced will be garbage-collected naturally. Explicit shutdown cleanup happens via close_litellm_async_clients(). Fixes production crashes after the 1-hour cache TTL expires. * test: update LLMClientCache unit tests for no-close-on-eviction behavior Flip the assertions: evicted clients must NOT be closed. Replace test_remove_key_closes_async_client → test_remove_key_does_not_close_async_client and equivalents for sync/eviction paths. Add test_remove_key_removes_plain_values for non-client cache entries. Remove test_background_tasks_cleaned_up_after_completion (no more _background_tasks). Remove test_remove_key_no_event_loop variant that depended on old behavior. * test: add e2e tests for OpenAI SDK client surviving cache eviction Add two new e2e tests using real AsyncOpenAI clients: - test_evicted_openai_sdk_client_stays_usable: verifies size-based eviction doesn't close the client - test_ttl_expired_openai_sdk_client_stays_usable: verifies TTL expiry eviction doesn't close the client Both tests sleep after eviction so any create_task()-based close would have time to run, making the regression detectable. Also expand the module docstring to explain why the sleep is required. * docs(AGENTS.md): add rule — never close HTTP/SDK clients on cache eviction * docs(CLAUDE.md): add HTTP client cache safety guideline * [Fix] Install bsdmainutils for column command in security scans The security_scans.sh script uses `column` to format vulnerability output, but the package wasn't installed in the CI environment. Co-Authored-By: Claude Opus 4.6 * fix: handle string callback values in prometheus multiproc setup When callbacks are configured as a plain string (e.g., `callbacks: "my_callback"`) instead of a list, the proxy crashes on startup with: TypeError: can only concatenate str (not "list") to str Normalize each callback setting to a list before concatenating. Co-Authored-By: Claude Opus 4.6 * bump: version 1.82.2 → 1.82.3 * fix(test): update test_startup_fails_when_db_setup_fails for opt-in enforcement The --enforce_prisma_migration_check flag is now required to trigger sys.exit(1) on DB migration failure, after #23675 flipped the default behavior to warn-and-continue. Co-Authored-By: Claude Opus 4.6 * fix(cost_calculator): use model name for per-request custom pricing when router_model_id has no pricing When custom pricing is passed as per-request kwargs (input_cost_per_token/output_cost_per_token), completion() registers pricing under the model name, but _select_model_name_for_cost_calc was selecting the router deployment hash (which has no pricing data), causing response_cost to be 0.0. Now checks whether the router_model_id entry actually has pricing before preferring it. Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Cursor Agent Co-authored-by: Ishaan Jaff Co-authored-by: Ishaan Jaff Co-authored-by: Claude Opus 4.6 --- .circleci/config.yml | 1 + CLAUDE.md | 2 +- ci_cd/security_scans.sh | 2 +- litellm/cost_calculator.py | 9 ++- .../out/{404.html => 404/index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../{budgets.html => budgets/index.html} | 0 .../{caching.html => caching/index.html} | 0 .../index.html} | 0 .../{old-usage.html => old-usage/index.html} | 0 .../{prompts.html => prompts/index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../out/{login.html => login/index.html} | 0 .../out/{logs.html => logs/index.html} | 0 .../{callback.html => callback/index.html} | 0 .../{model-hub.html => model-hub/index.html} | 0 .../{model_hub.html => model_hub/index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../{policies.html => policies/index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../{ui-theme.html => ui-theme/index.html} | 0 .../out/{teams.html => teams/index.html} | 0 .../{test-key.html => test-key/index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../out/{usage.html => usage/index.html} | 0 .../out/{users.html => users/index.html} | 0 .../index.html} | 0 .../example_config_yaml/custom_auth_basic.py | 3 +- litellm/proxy/proxy_cli.py | 7 +++ pyproject.toml | 4 +- tests/local_testing/test_router_utils.py | 3 +- tests/mcp_tests/test_mcp_server.py | 3 + .../proxy/test_prometheus_cleanup.py | 24 ++++++++ tests/test_litellm/proxy/test_proxy_cli.py | 4 +- tests/test_litellm/test_cost_calculator.py | 59 +++++++++++++++++++ 45 files changed, 112 insertions(+), 9 deletions(-) rename litellm/proxy/_experimental/out/{404.html => 404/index.html} (100%) rename litellm/proxy/_experimental/out/{_not-found.html => _not-found/index.html} (100%) rename litellm/proxy/_experimental/out/{api-reference.html => api-reference/index.html} (100%) rename litellm/proxy/_experimental/out/experimental/{api-playground.html => api-playground/index.html} (100%) rename litellm/proxy/_experimental/out/experimental/{budgets.html => budgets/index.html} (100%) rename litellm/proxy/_experimental/out/experimental/{caching.html => caching/index.html} (100%) rename litellm/proxy/_experimental/out/experimental/{claude-code-plugins.html => claude-code-plugins/index.html} (100%) rename litellm/proxy/_experimental/out/experimental/{old-usage.html => old-usage/index.html} (100%) rename litellm/proxy/_experimental/out/experimental/{prompts.html => prompts/index.html} (100%) rename litellm/proxy/_experimental/out/experimental/{tag-management.html => tag-management/index.html} (100%) rename litellm/proxy/_experimental/out/{guardrails.html => guardrails/index.html} (100%) rename litellm/proxy/_experimental/out/{login.html => login/index.html} (100%) rename litellm/proxy/_experimental/out/{logs.html => logs/index.html} (100%) rename litellm/proxy/_experimental/out/mcp/oauth/{callback.html => callback/index.html} (100%) rename litellm/proxy/_experimental/out/{model-hub.html => model-hub/index.html} (100%) rename litellm/proxy/_experimental/out/{model_hub.html => model_hub/index.html} (100%) rename litellm/proxy/_experimental/out/{model_hub_table.html => model_hub_table/index.html} (100%) rename litellm/proxy/_experimental/out/{models-and-endpoints.html => models-and-endpoints/index.html} (100%) rename litellm/proxy/_experimental/out/{onboarding.html => onboarding/index.html} (100%) rename litellm/proxy/_experimental/out/{organizations.html => organizations/index.html} (100%) rename litellm/proxy/_experimental/out/{playground.html => playground/index.html} (100%) rename litellm/proxy/_experimental/out/{policies.html => policies/index.html} (100%) rename litellm/proxy/_experimental/out/settings/{admin-settings.html => admin-settings/index.html} (100%) rename litellm/proxy/_experimental/out/settings/{logging-and-alerts.html => logging-and-alerts/index.html} (100%) rename litellm/proxy/_experimental/out/settings/{router-settings.html => router-settings/index.html} (100%) rename litellm/proxy/_experimental/out/settings/{ui-theme.html => ui-theme/index.html} (100%) rename litellm/proxy/_experimental/out/{teams.html => teams/index.html} (100%) rename litellm/proxy/_experimental/out/{test-key.html => test-key/index.html} (100%) rename litellm/proxy/_experimental/out/tools/{mcp-servers.html => mcp-servers/index.html} (100%) rename litellm/proxy/_experimental/out/tools/{vector-stores.html => vector-stores/index.html} (100%) rename litellm/proxy/_experimental/out/{usage.html => usage/index.html} (100%) rename litellm/proxy/_experimental/out/{users.html => users/index.html} (100%) rename litellm/proxy/_experimental/out/{virtual-keys.html => virtual-keys/index.html} (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 38e6d1fc332..12e3cb1f6b6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3218,6 +3218,7 @@ jobs: -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \ diff --git a/CLAUDE.md b/CLAUDE.md index 5395d6d938e..d9061b5e2be 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -156,4 +156,4 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components: **Fix options:** 1. **Create a Prisma migration** (permanent) — run `prisma migrate dev --name ` in the worktree. The generated file will be picked up by `prisma migrate deploy` on next startup. 2. **Apply manually for local dev** — `psql -d litellm -c "ALTER TABLE ... ADD COLUMN IF NOT EXISTS ..."` after each proxy start. Fine for dev, not for production. -3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it. \ No newline at end of file +3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it. diff --git a/ci_cd/security_scans.sh b/ci_cd/security_scans.sh index 62440d13ebb..e0f370e0035 100755 --- a/ci_cd/security_scans.sh +++ b/ci_cd/security_scans.sh @@ -11,7 +11,7 @@ echo "Starting security scans for LiteLLM..." install_trivy() { echo "Installing Trivy and required tools..." sudo apt-get update - sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl + sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl bsdmainutils wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add - echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list sudo apt-get update diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index a3ef7b264ec..e0e1e35b94e 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -660,7 +660,14 @@ def _select_model_name_for_cost_calc( if custom_pricing is True: if router_model_id is not None and router_model_id in litellm.model_cost: - return_model = router_model_id + entry = litellm.model_cost[router_model_id] + if ( + entry.get("input_cost_per_token") is not None + or entry.get("input_cost_per_second") is not None + ): + return_model = router_model_id + else: + return_model = model else: return_model = model diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404/index.html similarity index 100% rename from litellm/proxy/_experimental/out/404.html rename to litellm/proxy/_experimental/out/404/index.html diff --git a/litellm/proxy/_experimental/out/_not-found.html b/litellm/proxy/_experimental/out/_not-found/index.html similarity index 100% rename from litellm/proxy/_experimental/out/_not-found.html rename to litellm/proxy/_experimental/out/_not-found/index.html diff --git a/litellm/proxy/_experimental/out/api-reference.html b/litellm/proxy/_experimental/out/api-reference/index.html similarity index 100% rename from litellm/proxy/_experimental/out/api-reference.html rename to litellm/proxy/_experimental/out/api-reference/index.html diff --git a/litellm/proxy/_experimental/out/experimental/api-playground.html b/litellm/proxy/_experimental/out/experimental/api-playground/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/api-playground.html rename to litellm/proxy/_experimental/out/experimental/api-playground/index.html diff --git a/litellm/proxy/_experimental/out/experimental/budgets.html b/litellm/proxy/_experimental/out/experimental/budgets/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/budgets.html rename to litellm/proxy/_experimental/out/experimental/budgets/index.html diff --git a/litellm/proxy/_experimental/out/experimental/caching.html b/litellm/proxy/_experimental/out/experimental/caching/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/caching.html rename to litellm/proxy/_experimental/out/experimental/caching/index.html diff --git a/litellm/proxy/_experimental/out/experimental/claude-code-plugins.html b/litellm/proxy/_experimental/out/experimental/claude-code-plugins/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/claude-code-plugins.html rename to litellm/proxy/_experimental/out/experimental/claude-code-plugins/index.html diff --git a/litellm/proxy/_experimental/out/experimental/old-usage.html b/litellm/proxy/_experimental/out/experimental/old-usage/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/old-usage.html rename to litellm/proxy/_experimental/out/experimental/old-usage/index.html diff --git a/litellm/proxy/_experimental/out/experimental/prompts.html b/litellm/proxy/_experimental/out/experimental/prompts/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/prompts.html rename to litellm/proxy/_experimental/out/experimental/prompts/index.html diff --git a/litellm/proxy/_experimental/out/experimental/tag-management.html b/litellm/proxy/_experimental/out/experimental/tag-management/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/tag-management.html rename to litellm/proxy/_experimental/out/experimental/tag-management/index.html diff --git a/litellm/proxy/_experimental/out/guardrails.html b/litellm/proxy/_experimental/out/guardrails/index.html similarity index 100% rename from litellm/proxy/_experimental/out/guardrails.html rename to litellm/proxy/_experimental/out/guardrails/index.html diff --git a/litellm/proxy/_experimental/out/login.html b/litellm/proxy/_experimental/out/login/index.html similarity index 100% rename from litellm/proxy/_experimental/out/login.html rename to litellm/proxy/_experimental/out/login/index.html diff --git a/litellm/proxy/_experimental/out/logs.html b/litellm/proxy/_experimental/out/logs/index.html similarity index 100% rename from litellm/proxy/_experimental/out/logs.html rename to litellm/proxy/_experimental/out/logs/index.html diff --git a/litellm/proxy/_experimental/out/mcp/oauth/callback.html b/litellm/proxy/_experimental/out/mcp/oauth/callback/index.html similarity index 100% rename from litellm/proxy/_experimental/out/mcp/oauth/callback.html rename to litellm/proxy/_experimental/out/mcp/oauth/callback/index.html diff --git a/litellm/proxy/_experimental/out/model-hub.html b/litellm/proxy/_experimental/out/model-hub/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model-hub.html rename to litellm/proxy/_experimental/out/model-hub/index.html diff --git a/litellm/proxy/_experimental/out/model_hub.html b/litellm/proxy/_experimental/out/model_hub/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model_hub.html rename to litellm/proxy/_experimental/out/model_hub/index.html diff --git a/litellm/proxy/_experimental/out/model_hub_table.html b/litellm/proxy/_experimental/out/model_hub_table/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model_hub_table.html rename to litellm/proxy/_experimental/out/model_hub_table/index.html diff --git a/litellm/proxy/_experimental/out/models-and-endpoints.html b/litellm/proxy/_experimental/out/models-and-endpoints/index.html similarity index 100% rename from litellm/proxy/_experimental/out/models-and-endpoints.html rename to litellm/proxy/_experimental/out/models-and-endpoints/index.html diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding/index.html similarity index 100% rename from litellm/proxy/_experimental/out/onboarding.html rename to litellm/proxy/_experimental/out/onboarding/index.html diff --git a/litellm/proxy/_experimental/out/organizations.html b/litellm/proxy/_experimental/out/organizations/index.html similarity index 100% rename from litellm/proxy/_experimental/out/organizations.html rename to litellm/proxy/_experimental/out/organizations/index.html diff --git a/litellm/proxy/_experimental/out/playground.html b/litellm/proxy/_experimental/out/playground/index.html similarity index 100% rename from litellm/proxy/_experimental/out/playground.html rename to litellm/proxy/_experimental/out/playground/index.html diff --git a/litellm/proxy/_experimental/out/policies.html b/litellm/proxy/_experimental/out/policies/index.html similarity index 100% rename from litellm/proxy/_experimental/out/policies.html rename to litellm/proxy/_experimental/out/policies/index.html diff --git a/litellm/proxy/_experimental/out/settings/admin-settings.html b/litellm/proxy/_experimental/out/settings/admin-settings/index.html similarity index 100% rename from litellm/proxy/_experimental/out/settings/admin-settings.html rename to litellm/proxy/_experimental/out/settings/admin-settings/index.html diff --git a/litellm/proxy/_experimental/out/settings/logging-and-alerts.html b/litellm/proxy/_experimental/out/settings/logging-and-alerts/index.html similarity index 100% rename from litellm/proxy/_experimental/out/settings/logging-and-alerts.html rename to litellm/proxy/_experimental/out/settings/logging-and-alerts/index.html diff --git a/litellm/proxy/_experimental/out/settings/router-settings.html b/litellm/proxy/_experimental/out/settings/router-settings/index.html similarity index 100% rename from litellm/proxy/_experimental/out/settings/router-settings.html rename to litellm/proxy/_experimental/out/settings/router-settings/index.html diff --git a/litellm/proxy/_experimental/out/settings/ui-theme.html b/litellm/proxy/_experimental/out/settings/ui-theme/index.html similarity index 100% rename from litellm/proxy/_experimental/out/settings/ui-theme.html rename to litellm/proxy/_experimental/out/settings/ui-theme/index.html diff --git a/litellm/proxy/_experimental/out/teams.html b/litellm/proxy/_experimental/out/teams/index.html similarity index 100% rename from litellm/proxy/_experimental/out/teams.html rename to litellm/proxy/_experimental/out/teams/index.html diff --git a/litellm/proxy/_experimental/out/test-key.html b/litellm/proxy/_experimental/out/test-key/index.html similarity index 100% rename from litellm/proxy/_experimental/out/test-key.html rename to litellm/proxy/_experimental/out/test-key/index.html diff --git a/litellm/proxy/_experimental/out/tools/mcp-servers.html b/litellm/proxy/_experimental/out/tools/mcp-servers/index.html similarity index 100% rename from litellm/proxy/_experimental/out/tools/mcp-servers.html rename to litellm/proxy/_experimental/out/tools/mcp-servers/index.html diff --git a/litellm/proxy/_experimental/out/tools/vector-stores.html b/litellm/proxy/_experimental/out/tools/vector-stores/index.html similarity index 100% rename from litellm/proxy/_experimental/out/tools/vector-stores.html rename to litellm/proxy/_experimental/out/tools/vector-stores/index.html diff --git a/litellm/proxy/_experimental/out/usage.html b/litellm/proxy/_experimental/out/usage/index.html similarity index 100% rename from litellm/proxy/_experimental/out/usage.html rename to litellm/proxy/_experimental/out/usage/index.html diff --git a/litellm/proxy/_experimental/out/users.html b/litellm/proxy/_experimental/out/users/index.html similarity index 100% rename from litellm/proxy/_experimental/out/users.html rename to litellm/proxy/_experimental/out/users/index.html diff --git a/litellm/proxy/_experimental/out/virtual-keys.html b/litellm/proxy/_experimental/out/virtual-keys/index.html similarity index 100% rename from litellm/proxy/_experimental/out/virtual-keys.html rename to litellm/proxy/_experimental/out/virtual-keys/index.html diff --git a/litellm/proxy/example_config_yaml/custom_auth_basic.py b/litellm/proxy/example_config_yaml/custom_auth_basic.py index 4d633a54fe2..0da6105a305 100644 --- a/litellm/proxy/example_config_yaml/custom_auth_basic.py +++ b/litellm/proxy/example_config_yaml/custom_auth_basic.py @@ -1,6 +1,6 @@ from fastapi import Request -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth: @@ -9,6 +9,7 @@ async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth: api_key="best-api-key-ever", user_id="best-user-id-ever", team_id="best-team-id-ever", + user_role=LitellmUserRoles.PROXY_ADMIN, ) except Exception: raise Exception diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 701762c834c..e5a34ae8bdd 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -340,9 +340,16 @@ class ProxyInitializationHelpers: return # Check if prometheus is in any callback list + # Each setting can be a list or a single string; normalize to list callbacks = litellm_settings.get("callbacks") or [] success_callbacks = litellm_settings.get("success_callback") or [] failure_callbacks = litellm_settings.get("failure_callback") or [] + if isinstance(callbacks, str): + callbacks = [callbacks] + if isinstance(success_callbacks, str): + success_callbacks = [success_callbacks] + if isinstance(failure_callbacks, str): + failure_callbacks = [failure_callbacks] all_callbacks = callbacks + success_callbacks + failure_callbacks if "prometheus" not in all_callbacks: return diff --git a/pyproject.toml b/pyproject.toml index 8efc6366128..07004f3ae16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.82.2" +version = "1.82.3" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -183,7 +183,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.82.2" +version = "1.82.3" version_files = [ "pyproject.toml:^version" ] diff --git a/tests/local_testing/test_router_utils.py b/tests/local_testing/test_router_utils.py index 9d51685751a..4f7f53cef02 100644 --- a/tests/local_testing/test_router_utils.py +++ b/tests/local_testing/test_router_utils.py @@ -199,6 +199,7 @@ def test_router_get_model_info_wildcard_routes(): @pytest.mark.asyncio +@pytest.mark.flaky(retries=3, delay=1) async def test_router_get_model_group_usage_wildcard_routes(): os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") @@ -219,7 +220,7 @@ async def test_router_get_model_group_usage_wildcard_routes(): ) print(resp) - await asyncio.sleep(1) + await asyncio.sleep(2) tpm, rpm = await router.get_model_group_usage(model_group="gemini/gemini-1.5-flash") diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index 2544e06598e..a4a28215e16 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -395,6 +395,7 @@ async def test_mcp_http_transport_tool_not_found(): @pytest.mark.asyncio async def test_streamable_http_mcp_handler_mock(): """Test the streamable HTTP MCP handler functionality""" + from litellm.proxy._types import UserAPIKeyAuth # Mock the session manager and its methods mock_session_manager = AsyncMock() @@ -425,6 +426,8 @@ async def test_streamable_http_mcp_handler_mock(): ), patch( "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", AsyncMock(return_value=mock_auth_context), + ), patch( + "litellm.proxy._experimental.mcp_server.server.set_auth_context", ): from litellm.proxy._experimental.mcp_server.server import ( handle_streamable_http_mcp, diff --git a/tests/test_litellm/proxy/test_prometheus_cleanup.py b/tests/test_litellm/proxy/test_prometheus_cleanup.py index b3d785f1133..0a67d5e64e0 100644 --- a/tests/test_litellm/proxy/test_prometheus_cleanup.py +++ b/tests/test_litellm/proxy/test_prometheus_cleanup.py @@ -67,6 +67,30 @@ class TestMaybeSetupPrometheusMultiprocDir: assert os.environ["PROMETHEUS_MULTIPROC_DIR"] == custom_dir assert os.path.isdir(custom_dir) + @pytest.mark.parametrize( + "litellm_settings", + [ + {"callbacks": "prometheus"}, + {"success_callback": "prometheus"}, + {"failure_callback": "prometheus"}, + {"callbacks": "custom_callback"}, # string but not prometheus + ], + ) + def test_handles_string_callbacks(self, litellm_settings): + """When callbacks are specified as a string instead of a list, should not crash.""" + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) + os.environ.pop("prometheus_multiproc_dir", None) + + # Should not raise TypeError + ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir( + num_workers=4, + litellm_settings=litellm_settings, + ) + + # Cleanup + os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) + @pytest.mark.parametrize( "num_workers, litellm_settings", [ diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 642d21a42f7..c5d6c45f9a5 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -677,7 +677,7 @@ class TestHealthAppFactory: mock_atexit_register, mock_subprocess_run, ): - """Test that proxy exits with code 1 when PrismaManager.setup_database returns False""" + """Test that proxy exits with code 1 when PrismaManager.setup_database returns False and --enforce_prisma_migration_check is set""" from litellm.proxy.proxy_cli import run_server mock_subprocess_run.return_value = MagicMock(returncode=0) @@ -717,7 +717,7 @@ class TestHealthAppFactory: with pytest.raises(SystemExit) as exc_info: run_server.main( - ["--local", "--skip_server_startup"], standalone_mode=False + ["--local", "--skip_server_startup", "--enforce_prisma_migration_check"], standalone_mode=False ) assert exc_info.value.code == 1 mock_setup_database.assert_called_once_with(use_migrate=True) diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 463f6952d23..8f5c3ece0ca 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -388,6 +388,65 @@ def test_custom_pricing_cost_calc_uses_router_model_id_from_litellm_metadata(): assert custom_model_id not in (selected_model_no_custom or "") +def test_per_request_custom_pricing_with_router(): + """When custom pricing is passed as per-request kwargs (not in model_list), + _select_model_name_for_cost_calc should fall back to the model name + (where register_model stored the pricing) instead of the router_model_id + (which has no pricing data). + + Regression test for the bug where response._hidden_params["response_cost"] + returned 0.0 for per-request custom pricing via Router. + """ + from litellm import Router + from litellm.cost_calculator import _select_model_name_for_cost_calc + + router = Router( + model_list=[ + { + "model_name": "openai/gpt-3.5-turbo", + "litellm_params": { + "model": "openai/gpt-3.5-turbo", + "api_key": "test_api_key", + }, + }, + ] + ) + + # Get the deployment's model_id (hash) that the router registered + deployment = router.model_list[0] + router_model_id = deployment["model_info"]["id"] + + # The router registered this hash in model_cost but without custom pricing + assert router_model_id in litellm.model_cost + entry = litellm.model_cost[router_model_id] + # No custom pricing was set in model_list, so these should be None + assert entry.get("input_cost_per_token") is None + + # Now simulate what completion() does: register custom pricing under the model name + litellm.register_model( + { + "openai/gpt-3.5-turbo": { + "input_cost_per_token": 2.0, + "output_cost_per_token": 2.0, + "litellm_provider": "openai", + } + } + ) + + # _select_model_name_for_cost_calc should pick the model name (which has pricing), + # NOT the router_model_id (which has no pricing) + selected = _select_model_name_for_cost_calc( + model="openai/gpt-3.5-turbo", + completion_response=None, + custom_pricing=True, + custom_llm_provider="openai", + router_model_id=router_model_id, + ) + assert selected is not None + assert router_model_id not in selected + assert "gpt-3.5-turbo" in selected + + def test_azure_realtime_cost_calculator(): os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") From d15c2d546ecf3448b2447b24aea0d7643b6efe3f Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Tue, 17 Mar 2026 10:54:29 +0530 Subject: [PATCH 2/4] fix: Register DynamoAI guardrail initializer and enum entry (#23752) * fix: Register DynamoAI guardrail initializer and enum entry Fix the "Unsupported guardrail: dynamoai" error by: 1. Adding DYNAMOAI to SupportedGuardrailIntegrations enum 2. Implementing initialize_guardrail() and registries in dynamoai/__init__.py The DynamoAI guardrail was added in PR #15920 but never properly registered in the initialization system. The __init__.py was missing the guardrail_initializer_registry and guardrail_class_registry dictionaries that the dynamic discovery mechanism looks for at module load time. Fixes #22773 Co-Authored-By: Claude Haiku 4.5 * Update litellm/proxy/guardrails/guardrail_hooks/dynamoai/__init__.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * Update litellm/proxy/guardrails/guardrail_hooks/dynamoai/__init__.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * test: Add tests for DynamoAI guardrail registration Verifies enum entry, initializer registry, class registry, instance creation, and global registry discovery. Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Haiku 4.5 Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../guardrail_hooks/dynamoai/__init__.py | 32 +++++++- litellm/types/guardrails.py | 1 + .../guardrail_hooks/test_dynamoai.py | 81 +++++++++++++++++++ 3 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 tests/test_litellm/proxy/guardrails/guardrail_hooks/test_dynamoai.py diff --git a/litellm/proxy/guardrails/guardrail_hooks/dynamoai/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/dynamoai/__init__.py index 79f1992da44..f9ebf46a270 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/dynamoai/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/dynamoai/__init__.py @@ -1,3 +1,33 @@ +from typing import TYPE_CHECKING + +from litellm.types.guardrails import SupportedGuardrailIntegrations + from .dynamoai import DynamoAIGuardrails -__all__ = ["DynamoAIGuardrails"] +if TYPE_CHECKING: + from litellm.types.guardrails import Guardrail, LitellmParams + + +def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"): + import litellm + + _dynamoai_callback = DynamoAIGuardrails( + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + guardrail_name=guardrail.get("guardrail_name", ""), + event_hook=litellm_params.mode, + default_on=litellm_params.default_on, + ) + litellm.logging_callback_manager.add_litellm_callback(_dynamoai_callback) + + return _dynamoai_callback + + +guardrail_initializer_registry = { + SupportedGuardrailIntegrations.DYNAMOAI.value: initialize_guardrail, +} + + +guardrail_class_registry = { + SupportedGuardrailIntegrations.DYNAMOAI.value: DynamoAIGuardrails, +} diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index d5abf5c8fbf..27fa27e6da3 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -44,6 +44,7 @@ guardrails: class SupportedGuardrailIntegrations(Enum): APORIA = "aporia" BEDROCK = "bedrock" + DYNAMOAI = "dynamoai" GUARDRAILS_AI = "guardrails_ai" LAKERA = "lakera" LAKERA_V2 = "lakera_v2" diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_dynamoai.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_dynamoai.py new file mode 100644 index 00000000000..7bc4e951a5f --- /dev/null +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_dynamoai.py @@ -0,0 +1,81 @@ +""" +Tests for DynamoAI guardrail registration and initialization. +""" + +import os +from unittest.mock import patch + +import pytest + + +class TestDynamoAIGuardrailRegistration: + """Tests for DynamoAI guardrail registration in the guardrail system.""" + + def test_supported_guardrail_enum_entry(self): + """Test that DYNAMOAI is in SupportedGuardrailIntegrations enum.""" + from litellm.types.guardrails import SupportedGuardrailIntegrations + + assert hasattr(SupportedGuardrailIntegrations, "DYNAMOAI") + assert SupportedGuardrailIntegrations.DYNAMOAI.value == "dynamoai" + + def test_initialize_guardrail_function_exists(self): + """Test that initialize_guardrail function is properly exported.""" + from litellm.proxy.guardrails.guardrail_hooks.dynamoai import ( + guardrail_initializer_registry, + initialize_guardrail, + ) + + assert initialize_guardrail is not None + assert "dynamoai" in guardrail_initializer_registry + + def test_guardrail_class_registry_exists(self): + """Test that guardrail_class_registry is properly exported.""" + from litellm.proxy.guardrails.guardrail_hooks.dynamoai import ( + guardrail_class_registry, + ) + from litellm.proxy.guardrails.guardrail_hooks.dynamoai.dynamoai import ( + DynamoAIGuardrails, + ) + + assert "dynamoai" in guardrail_class_registry + assert guardrail_class_registry["dynamoai"] == DynamoAIGuardrails + + def test_initialize_guardrail_creates_instance(self): + """Test that initialize_guardrail creates a DynamoAIGuardrails instance.""" + from litellm.proxy.guardrails.guardrail_hooks.dynamoai import ( + initialize_guardrail, + ) + from litellm.proxy.guardrails.guardrail_hooks.dynamoai.dynamoai import ( + DynamoAIGuardrails, + ) + from litellm.types.guardrails import LitellmParams + + litellm_params = LitellmParams( + guardrail="dynamoai", + mode="pre_call", + api_key="test-key", + api_base="https://test.dynamo.ai", + ) + + guardrail = { + "guardrail_name": "test-dynamoai-guard", + } + + with patch( + "litellm.logging_callback_manager.add_litellm_callback" + ) as mock_add: + result = initialize_guardrail(litellm_params, guardrail) + + assert isinstance(result, DynamoAIGuardrails) + assert result.api_key == "test-key" + assert result.api_base == "https://test.dynamo.ai" + assert result.guardrail_name == "test-dynamoai-guard" + mock_add.assert_called_once_with(result) + + def test_dynamoai_in_global_registry(self): + """Test that dynamoai is discoverable in the global guardrail registry.""" + from litellm.proxy.guardrails.guardrail_registry import ( + guardrail_initializer_registry, + ) + + assert "dynamoai" in guardrail_initializer_registry From 966124966f83e6e1091ad08d9dfee6e341d3ad85 Mon Sep 17 00:00:00 2001 From: Joe Reyna Date: Mon, 16 Mar 2026 22:25:55 -0700 Subject: [PATCH 3/4] docs: add v1.82.3 release notes and update provider_endpoints_support.json (#23816) --- docs/my-website/release_notes/v1.82.3.md | 374 +++++++++++++++++++++++ provider_endpoints_support.json | 55 ++-- 2 files changed, 406 insertions(+), 23 deletions(-) create mode 100644 docs/my-website/release_notes/v1.82.3.md diff --git a/docs/my-website/release_notes/v1.82.3.md b/docs/my-website/release_notes/v1.82.3.md new file mode 100644 index 00000000000..98351a30a27 --- /dev/null +++ b/docs/my-website/release_notes/v1.82.3.md @@ -0,0 +1,374 @@ +--- +title: "v1.82.3 - Nebius AI, gpt-5.4, Gemini 3.x, FLUX Kontext, and 116 New Models" +slug: "v1-82-3" +date: 2026-03-16T00:00:00 +authors: + - name: Krrish Dholakia + title: CEO, LiteLLM + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: CTO, LiteLLM + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +hide_table_of_contents: false +--- + +## Deploy this version + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + + + +``` showLineNumbers title="docker run litellm" +docker run \ +-e STORE_MODEL_IN_DB=True \ +-p 4000:4000 \ +ghcr.io/berriai/litellm:main-1.82.3-stable +``` + + + + +``` showLineNumbers title="pip install litellm" +pip install litellm==1.82.3 +``` + + + + +## Key Highlights + +- **Nebius AI — new provider** — [30 models across DeepSeek, Qwen, Llama, Mistral, NVIDIA, and BAAI available via Nebius AI cloud](../../docs/providers/nebius) - [PR #21542](https://github.com/BerriAI/litellm/pull/21542) +- **OpenAI gpt-5.4 / gpt-5.4-pro — day 0** — Full pricing and routing support for `gpt-5.4` (1M context, $2.50/$15.00) and `gpt-5.4-pro` ($30.00/$180.00) on OpenAI and Azure +- **Gemini 3.x models** — `gemini-3-flash-preview`, `gemini-3.1-pro-preview`, `gemini-3.1-flash-image-preview`, and `gemini-embedding-2-preview` added to cost map for Google AI and Vertex AI +- **FLUX Kontext image editing** — `flux-kontext-pro` and `flux-kontext-max` added to Black Forest Labs, alongside `flux-pro-1.0-fill` and `flux-pro-1.0-expand` for inpainting and outpainting +- **116 new models, 132 deprecated models cleaned up** — Major model map refresh including Mistral Magistral, Dashscope Qwen3 VL, xAI Grok via Azure AI, ZAI GLM-5, Serper Search; removal of OpenAI GPT-3.5/GPT-4 legacy variants, Gemini 1.5, and Vertex AI PaLM2 +- **SageMaker Nova provider** — [New `sagemaker_nova` provider for Amazon Nova models on SageMaker](../../docs/providers/aws_sagemaker) - [PR #21542](https://github.com/BerriAI/litellm/pull/21542) +- **Secret redaction in logs** — API keys, tokens, and credentials automatically scrubbed from all proxy log output. Enabled by default; opt out with `LITELLM_DISABLE_REDACT_SECRETS=true` - [PR #23668](https://github.com/BerriAI/litellm/pull/23668) +- **Streaming stability fix** — Critical fix for `RuntimeError: Cannot send a request, as the client has been closed.` crashes after ~1 hour in production - [PR #22926](https://github.com/BerriAI/litellm/pull/22926) + +--- + +## New Providers and Endpoints + +### New Providers (4 new providers) + +| Provider | Supported LiteLLM Endpoints | Description | +| -------- | --------------------------- | ----------- | +| [Nebius AI](../../docs/providers/nebius) (`nebius/`) | `/chat/completions`, `/embeddings` | EU-based AI cloud with 30+ open models — DeepSeek, Qwen3, Llama 3.1/3.3, NVIDIA Nemotron, BAAI embeddings | +| [ZAI](../../docs/providers/openai_compatible) (`zai/`) | `/chat/completions` | ZhipuAI GLM-5 models via ZAI cloud | +| [Black Forest Labs](../../docs/providers/black_forest_labs) (`black_forest_labs/`) | `/images/generations`, `/images/edits` | FLUX image generation and editing — Kontext Pro/Max, Pro 1.0 Fill/Expand | +| [Serper](../../docs/providers/serper) (`serper/`) | `/search` | Web search via Serper API | +| [SageMaker Nova](../../docs/providers/aws_sagemaker) (`sagemaker_nova/`) | `/chat/completions` | Amazon Nova models via SageMaker endpoint | + +--- + +## New Models / Updated Models + +#### New Model Support (116 new models) + +| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features | +| -------- | ----- | -------------- | ------------------- | -------------------- | -------- | +| OpenAI | `gpt-5.4` | 1.05M | $2.50 | $15.00 | chat, vision, tools, reasoning | +| OpenAI | `gpt-5.4-pro` | 1.05M | $30.00 | $180.00 | responses, vision, tools, reasoning | +| OpenAI | `gpt-5.3-chat-latest` | 128K | $1.75 | $14.00 | chat, vision, tools, reasoning | +| Azure OpenAI | `azure/gpt-5.4` | 1.05M | $2.50 | $15.00 | chat, vision, tools, reasoning | +| Azure OpenAI | `azure/gpt-5.4-pro` | 1.05M | $30.00 | $180.00 | responses, vision, tools, reasoning | +| Azure OpenAI | `azure/gpt-5.3-chat` | 128K | $1.75 | $14.00 | chat, vision, tools, reasoning | +| Google Gemini | `gemini/gemini-3-flash-preview` | 1M | $0.50 | $3.00 | chat, vision, tools, reasoning | +| Google Gemini | `gemini/gemini-3.1-pro-preview` | 1M | $2.00 | $12.00 | chat, vision, tools, reasoning | +| Google Gemini | `gemini/gemini-3.1-flash-image-preview` | 65K | $0.25 | $1.50 | image generation, vision | +| Google Gemini | `gemini/gemini-3.1-flash-lite-preview` | - | - | - | chat | +| Google Gemini | `gemini/gemini-3-pro-image-preview` | - | - | - | image generation | +| Google Gemini | `gemini/gemini-embedding-2-preview` | 8K | $0.20 | - | embeddings | +| Google Vertex AI | `vertex_ai/gemini-3-flash-preview` | - | - | - | chat | +| Google Vertex AI | `vertex_ai/gemini-3.1-pro-preview` | - | - | - | chat | +| Google Vertex AI | `vertex_ai/gemini-3.1-flash-lite-preview` | - | - | - | chat | +| Google Vertex AI | `vertex_ai/gemini-embedding-2-preview` | - | $0.20 | - | embeddings | +| Mistral | `mistral/magistral-medium-1-2-2509` | 40K | $2.00 | $5.00 | chat, tools, reasoning | +| Mistral | `mistral/magistral-small-1-2-2509` | 40K | $0.50 | $1.50 | chat, tools, reasoning | +| Mistral | `mistral/mistral-large-2512` | 262K | $0.50 | $1.50 | chat, vision, tools | +| Mistral | `mistral/mistral-medium-3-1-2508` | - | - | - | chat | +| Mistral | `mistral/mistral-small-3-2-2506` | - | - | - | chat | +| Mistral | `mistral/ministral-3-3b-2512` | - | - | - | chat | +| Mistral | `mistral/ministral-3-8b-2512` | - | - | - | chat | +| Mistral | `mistral/ministral-3-14b-2512` | - | - | - | chat | +| Black Forest Labs | `black_forest_labs/flux-kontext-pro` | - | - | - | image editing | +| Black Forest Labs | `black_forest_labs/flux-kontext-max` | - | - | - | image editing | +| Black Forest Labs | `black_forest_labs/flux-pro-1.0-fill` | - | - | - | image editing (inpaint) | +| Black Forest Labs | `black_forest_labs/flux-pro-1.0-expand` | - | - | - | image editing (outpaint) | +| Black Forest Labs | `black_forest_labs/flux-pro-1.1` | - | - | - | image generation | +| Black Forest Labs | `black_forest_labs/flux-pro-1.1-ultra` | - | - | - | image generation | +| Black Forest Labs | `black_forest_labs/flux-dev` | - | - | - | image generation | +| Black Forest Labs | `black_forest_labs/flux-pro` | - | - | - | image generation | +| Azure AI | `azure_ai/grok-4-1-fast-non-reasoning` | 131K | $0.20 | $0.50 | chat, tools | +| Azure AI | `azure_ai/grok-4-1-fast-reasoning` | 131K | $0.20 | $0.50 | chat, tools, reasoning | +| Azure AI | `azure_ai/mistral-document-ai-2512` | - | - | - | OCR | +| Dashscope | `dashscope/qwen3-next-80b-a3b-instruct` | 262K | $0.15 | $1.20 | chat | +| Dashscope | `dashscope/qwen3-next-80b-a3b-thinking` | 262K | $0.15 | $1.20 | chat, reasoning | +| Dashscope | `dashscope/qwen3-vl-235b-a22b-instruct` | 131K | $0.40 | $1.60 | chat, vision | +| Dashscope | `dashscope/qwen3-vl-235b-a22b-thinking` | 131K | $0.40 | $4.00 | chat, vision, reasoning | +| Dashscope | `dashscope/qwen3-vl-32b-instruct` | 131K | $0.16 | $0.64 | chat, vision | +| Dashscope | `dashscope/qwen3-vl-32b-thinking` | 131K | $0.16 | $2.87 | chat, vision, reasoning | +| Dashscope | `dashscope/qwen3-vl-plus` | 260K | - | - | chat, vision | +| Dashscope | `dashscope/qwen3.5-plus` | 992K | - | - | chat | +| Dashscope | `dashscope/qwen3-max-2026-01-23` | 258K | - | - | chat | +| Nebius AI | `nebius/deepseek-ai/DeepSeek-R1` | 128K | $0.80 | $2.40 | chat, reasoning | +| Nebius AI | `nebius/deepseek-ai/DeepSeek-R1-0528` | 164K | $0.80 | $2.40 | chat, reasoning | +| Nebius AI | `nebius/deepseek-ai/DeepSeek-V3` | 128K | $0.50 | $1.50 | chat | +| Nebius AI | `nebius/deepseek-ai/DeepSeek-V3-0324` | 128K | $0.50 | $1.50 | chat | +| Nebius AI | `nebius/deepseek-ai/DeepSeek-R1-Distill-Llama-70B` | 128K | $0.25 | $0.75 | chat | +| Nebius AI | `nebius/Qwen/Qwen3-235B-A22B` | 262K | $0.20 | $0.60 | chat | +| Nebius AI | `nebius/Qwen/Qwen3-32B` | 32K | $0.10 | $0.30 | chat | +| Nebius AI | `nebius/Qwen/Qwen3-30B-A3B` | 32K | $0.10 | $0.30 | chat | +| Nebius AI | `nebius/Qwen/Qwen3-14B` | 32K | $0.08 | $0.24 | chat | +| Nebius AI | `nebius/Qwen/Qwen3-4B` | 32K | $0.08 | $0.24 | chat | +| Nebius AI | `nebius/Qwen/QwQ-32B` | 32K | $0.15 | $0.45 | chat | +| Nebius AI | `nebius/Qwen/Qwen2.5-72B-Instruct` | 128K | $0.13 | $0.40 | chat | +| Nebius AI | `nebius/Qwen/Qwen2.5-32B-Instruct` | 128K | $0.06 | $0.20 | chat | +| Nebius AI | `nebius/Qwen/Qwen2.5-VL-72B-Instruct` | 131K | $0.13 | $0.40 | chat, vision | +| Nebius AI | `nebius/Qwen/Qwen2-VL-72B-Instruct` | 131K | $0.13 | $0.40 | chat, vision | +| Nebius AI | `nebius/Qwen/Qwen2-VL-7B-Instruct` | 131K | $0.02 | $0.06 | chat, vision | +| Nebius AI | `nebius/meta-llama/Meta-Llama-3.1-405B-Instruct` | 128K | $1.00 | $3.00 | chat | +| Nebius AI | `nebius/meta-llama/Meta-Llama-3.1-70B-Instruct` | 128K | $0.13 | $0.40 | chat | +| Nebius AI | `nebius/meta-llama/Meta-Llama-3.1-8B-Instruct` | 128K | $0.02 | $0.06 | chat | +| Nebius AI | `nebius/meta-llama/Llama-3.3-70B-Instruct` | 128K | $0.13 | $0.40 | chat | +| Nebius AI | `nebius/meta-llama/Llama-Guard-3-8B` | 128K | $0.02 | $0.06 | chat | +| Nebius AI | `nebius/nvidia/Llama-3.1-Nemotron-Ultra-253B-v1` | 128K | $0.60 | $1.80 | chat | +| Nebius AI | `nebius/nvidia/Llama-3.3-Nemotron-Super-49B-v1` | 131K | $0.10 | $0.40 | chat | +| Nebius AI | `nebius/NousResearch/Hermes-3-Llama-3.1-405B` | 128K | $1.00 | $3.00 | chat | +| Nebius AI | `nebius/google/gemma-3-27b-it` | 128K | $0.06 | $0.20 | chat | +| Nebius AI | `nebius/mistralai/Mistral-Nemo-Instruct-2407` | 128K | $0.04 | $0.12 | chat | +| Nebius AI | `nebius/Qwen/Qwen2.5-Coder-7B` | 32K | $0.01 | $0.03 | chat | +| Nebius AI | `nebius/BAAI/bge-en-icl` | 32K | $0.01 | - | embeddings | +| Nebius AI | `nebius/BAAI/bge-multilingual-gemma2` | 8K | $0.01 | - | embeddings | +| Nebius AI | `nebius/intfloat/e5-mistral-7b-instruct` | 32K | $0.01 | - | embeddings | +| AWS Bedrock | `mistral.devstral-2-123b` | 256K | $0.40 | $2.00 | chat, tools | +| AWS Bedrock | `zai.glm-4.7-flash` | 200K | $0.07 | $0.40 | chat, tools, reasoning | +| ZAI | `zai/glm-5` | 200K | $1.00 | $3.20 | chat, tools, reasoning | +| ZAI | `zai/glm-5-code` | 200K | $1.20 | $5.00 | chat, tools, reasoning | +| OpenRouter | `openrouter/anthropic/claude-sonnet-4.6` | - | - | - | chat | +| OpenRouter | `openrouter/google/gemini-3.1-pro-preview` | - | - | - | chat | +| OpenRouter | `openrouter/openai/gpt-5.1-codex-max` | - | - | - | chat | +| OpenRouter | `openrouter/qwen/qwen3-coder-plus` | - | - | - | chat | +| OpenRouter | `openrouter/qwen/qwen3.5-*` (5 models) | - | - | - | chat | +| OpenRouter | `openrouter/z-ai/glm-5` | - | - | - | chat | +| Together AI | `together_ai/Qwen/Qwen3.5-397B-A17B` | - | - | - | chat | +| Perplexity | `perplexity/pplx-embed-v1-0.6b` | 32K | $0.00 | - | embeddings | +| Perplexity | `perplexity/pplx-embed-v1-4b` | 32K | $0.03 | - | embeddings | +| Serper | `serper/search` | - | - | - | search | + +#### Updated Models + +- **[AWS Bedrock](../../docs/providers/bedrock)** + - Add `cache_read_input_token_cost` and `cache_creation_input_token_cost` to Bedrock-hosted Anthropic models (`claude-3-opus`, `claude-3-sonnet`, `claude-3-haiku`, and APAC/EU variants) — prompt caching is now tracked for cost estimation + - Rename `apac.anthropic.claude-sonnet-4-6` → `au.anthropic.claude-sonnet-4-6` to reflect correct regional identifier + +- **[Azure OpenAI](../../docs/providers/azure)** + - Add `supports_none_reasoning_effort` to all `gpt-5.1-chat`, `gpt-5.1-codex`, and `gpt-5.4` variants (global, EU, standard deployments) — allows passing `reasoning_effort: null` to disable reasoning + +- **[Azure OpenAI](../../docs/providers/azure)** — Removed deprecated models + - Remove `azure/gpt-35-turbo-0301` (deprecated 2025-02-13) + - Remove `azure/gpt-35-turbo-0613` (deprecated 2025-02-13) + +#### Features + +- **[OpenAI](../../docs/providers/openai)** + - Day 0 support for `gpt-5.4` and `gpt-5.4-pro` on OpenAI and Azure + +- **[Google Gemini](../../docs/providers/gemini)** + - Add Gemini 3.x model cost map entries — `gemini-3-flash-preview`, `gemini-3.1-pro-preview`, `gemini-3.1-flash-lite-preview`, `gemini-3-pro-image-preview`, `gemini-embedding-2-preview` + - Add Gemini 2.0 Flash and Flash Lite to cost map (re-added with updated pricing) + +- **[Google Vertex AI](../../docs/providers/vertex)** + - Add `gemini-3-flash-preview`, `gemini-3.1-flash-lite-preview`, `gemini-flash-experimental`, and `gemini-embedding-2-preview` to Vertex AI model cost map + +- **[Mistral](../../docs/providers/mistral)** + - Add Magistral reasoning models (`magistral-medium-1-2-2509`, `magistral-small-1-2-2509`) + - Add `mistral-large-2512`, `mistral-medium-3-1-2508`, `mistral-small-3-2-2506`, `ministral-3-*` variants + +- **[Dashscope / Qwen](../../docs/providers/dashscope)** + - Add Qwen3 VL multimodal models (`qwen3-vl-235b`, `qwen3-vl-32b` — instruct and thinking variants) + - Add `qwen3-next-80b-a3b` (instruct + thinking), `qwen3.5-plus`, `qwen3-max-2026-01-23` + +- **[Black Forest Labs](../../docs/providers/black_forest_labs)** + - Add FLUX Kontext image editing models (`flux-kontext-pro`, `flux-kontext-max`) + - Add FLUX Pro 1.0 Fill (inpainting) and Expand (outpainting) + - Add `flux-pro-1.1`, `flux-pro-1.1-ultra`, `flux-dev`, `flux-pro` + +- **[Azure AI](../../docs/providers/azure_ai)** + - Add xAI Grok models via Azure AI Foundry (`grok-4-1-fast-non-reasoning`, `grok-4-1-fast-reasoning`) + - Add Mistral Document AI (`mistral-document-ai-2512`) — OCR mode + +- **[AWS Bedrock](../../docs/providers/bedrock)** + - Add `mistral.devstral-2-123b` (256K context, tools) + - Add `zai.glm-4.7-flash` via Bedrock Converse (200K context, tools, reasoning) + +- **[SageMaker](../../docs/providers/aws_sagemaker)** + - Add `sagemaker_nova` provider for Amazon Nova models on SageMaker - [PR #21542](https://github.com/BerriAI/litellm/pull/21542) + +#### Deprecated / Removed Models + +**OpenAI** — Legacy models removed from cost map: +- `gpt-3.5-turbo-0301`, `gpt-3.5-turbo-0613`, `gpt-3.5-turbo-16k-0613` +- `gpt-4-0314`, `gpt-4-32k`, `gpt-4-32k-0314`, `gpt-4-32k-0613`, `gpt-4-1106-vision-preview`, `gpt-4-vision-preview` +- `gpt-4.5-preview`, `gpt-4.5-preview-2025-02-27` +- `gpt-4o-audio-preview-2024-10-01`, `gpt-4o-realtime-preview-2024-10-01` +- `o1-mini`, `o1-mini-2024-09-12`, `o1-preview`, `o1-preview-2024-09-12` + +**Google Gemini** — Gemini 1.5 and legacy 2.0 variants removed: +- All `gemini-1.5-*` variants (flash, flash-8b, pro, and dated versions) +- `gemini-2.0-flash-exp`, `gemini-2.0-pro-exp-02-05`, `gemini-2.5-flash-preview-04-17`, `gemini-2.5-flash-preview-05-20` + +**Google Vertex AI** — PaLM 2 / legacy models removed: +- All `chat-bison`, `text-bison`, `codechat-bison`, `code-bison`, `code-gecko` variants +- Gemini 1.0 Pro, 1.5 Flash/Pro, 2.0 Flash experimental, and preview variants + +**Perplexity** — Legacy Llama-sonar models removed: +- `llama-3.1-sonar-huge-128k-online`, `llama-3.1-sonar-large/small-128k-chat/online` + +--- + +## LLM API Endpoints + +#### Features + +- **[Responses API](../../docs/response_api)** + - Handle `response.failed`, `response.incomplete`, and `response.cancelled` terminal event types in background streaming — previously only `response.completed` was handled - [PR #23492](https://github.com/BerriAI/litellm/pull/23492) + +#### Bug Fixes + +- **[Anthropic](../../docs/providers/anthropic)** + - Preserve native tool format (web_search, bash, tool_search, etc.) when guardrails convert tools for the Anthropic Messages API - [PR #23526](https://github.com/BerriAI/litellm/pull/23526) + +- **[Moonshot / Kimi](../../docs/providers/openai_compatible)** + - Auto-fill `reasoning_content` for Moonshot Kimi reasoning models - [PR #23580](https://github.com/BerriAI/litellm/pull/23580) + +- **[HuggingFace](../../docs/providers/huggingface)** + - Forward `extra_headers` to HuggingFace embedding API - [PR #23525](https://github.com/BerriAI/litellm/pull/23525) + +- **General** + - Normalize `content_filtered` finish reason across providers - [PR #23564](https://github.com/BerriAI/litellm/pull/23564) + - Fix custom cost tracking on deployments for `/v1/messages` and `/v1/responses` - [PR #23647](https://github.com/BerriAI/litellm/pull/23647) + - Fix per-request custom pricing when `router_model_id` has no pricing data — now falls back to model name + +--- + +## Management Endpoints / UI + +#### Features + +- **Virtual Keys** + - Add Organization dropdown to Create/Edit Key form — `organization_id` is now a first-class field in Key Ownership - [PR #23595](https://github.com/BerriAI/litellm/pull/23595) + - Allow setting `organization_id` on `/key/update` — keys can be assigned or moved to a different organization after creation - [PR #23557](https://github.com/BerriAI/litellm/pull/23557) + +- **Internal Users** + - Add/Remove Team Membership directly from the Internal Users info page — includes searchable dropdown and role selector; no longer requires navigating to each team - [PR #23638](https://github.com/BerriAI/litellm/pull/23638) + +- **Default Team Settings** + - Modernize page to antd (consistent with rest of app) - [PR #23614](https://github.com/BerriAI/litellm/pull/23614) + - Fix: default team params (budget, duration, tpm, rpm, permissions) now correctly applied on `/team/new` - [PR #23614](https://github.com/BerriAI/litellm/pull/23614) + - Fix: settings persist across proxy restarts (`default_team_params` added to `LITELLM_SETTINGS_SAFE_DB_OVERRIDES`) - [PR #23614](https://github.com/BerriAI/litellm/pull/23614) + - Fix: resolved race condition in `_update_litellm_setting` where `get_config()` could overwrite freshly saved values - [PR #23614](https://github.com/BerriAI/litellm/pull/23614) + +- **Usage** + - Auto-paginate daily spend data — all entity views (teams, orgs, customers, tags, agents, users) fetch pages progressively with charts updating after each page - [PR #23622](https://github.com/BerriAI/litellm/pull/23622) + +- **Models / Cost** + - Azure Model Router cost breakdown in UI — show per-sub-model `additional_costs` from `hidden_params` in `CostBreakdownViewer` - [PR #23550](https://github.com/BerriAI/litellm/pull/23550) + +- **User Management** + - New `/user/info/v2` endpoint — scoped, paginated replacement for the existing god endpoint that caused memory and stability issues on large installs - [PR #23437](https://github.com/BerriAI/litellm/pull/23437) + +#### Bugs + +- Fix Tag list endpoint returning 500 due to invalid Prisma `group_by` kwargs - [PR #23606](https://github.com/BerriAI/litellm/pull/23606) +- Fix Team Admin getting 403 on `/user/filter/ui` when `scope_user_search_to_org` is enabled - [PR #23671](https://github.com/BerriAI/litellm/pull/23671) +- Fix Public Model Hub not showing config-defined models after save - [PR #23501](https://github.com/BerriAI/litellm/pull/23501) +- Fix fallback popup model dropdown z-index issue - [PR #23516](https://github.com/BerriAI/litellm/pull/23516) +- Fix double-counting bug in org/team key limit checks on `/key/update` + +--- + +## AI Integrations + +### Logging + +- **[Vantage](https://vantage.sh)** + - Add Vantage integration for FOCUS 1.2 CSV export — export LiteLLM proxy spend data as FinOps Open Cost & Usage Specification reports, with time-windowed filenames to prevent overwrites - [PR #23333](https://github.com/BerriAI/litellm/pull/23333) + +- **General** + - Fix silent metrics race condition causing metric collision across experiments - [PR #23542](https://github.com/BerriAI/litellm/pull/23542) + +### Guardrails + +No major guardrail changes in this release. + +### Prompt Management + +No major prompt management changes in this release. + +### Secret Managers + +No major secret manager changes in this release. + +--- + +## Performance / Loadbalancing / Reliability improvements + +- **Fix streaming crashes after ~1 hour** — `LLMClientCache._remove_key()` no longer calls `close()`/`aclose()` on evicted HTTP/SDK clients. In-flight requests were crashing with `RuntimeError: Cannot send a request, as the client has been closed.` after the 1-hour TTL expired. Cleanup now happens only at shutdown via `close_litellm_async_clients()` - [PR #22926](https://github.com/BerriAI/litellm/pull/22926) +- **Fix OOM / Prisma connection loss** on large installs — unbounded managed-object poll was exhausting Prisma connections after ~60–70 minutes on instances with 336K+ queued response rows - [PR #23472](https://github.com/BerriAI/litellm/pull/23472) +- **Centralize logging kwarg updates** — root cause fix migrating all logging updates to a single function, eliminating kwarg inconsistencies across logging paths - [PR #23659](https://github.com/BerriAI/litellm/pull/23659) +- **Fix tiktoken cache for non-root offline containers** — tiktoken cache now works correctly in offline environments running as non-root users - [PR #23498](https://github.com/BerriAI/litellm/pull/23498) +- **Add CodSpeed continuous performance benchmarks** — automated performance regression tracking on CI - [PR #23676](https://github.com/BerriAI/litellm/pull/23676) + +--- + +## Security + +- **Secret redaction in proxy logs** — Adds a `SecretRedactionFilter` to all LiteLLM loggers that scrubs API keys, tokens, and credentials from log messages, format args, exception tracebacks, and extra fields. Enabled by default; opt out with `LITELLM_DISABLE_REDACT_SECRETS=true` - [PR #23668](https://github.com/BerriAI/litellm/pull/23668), [PR #23667](https://github.com/BerriAI/litellm/pull/23667) +- **Bump PyJWT to `^2.12.0`** — addresses security vulnerability in `^2.10.1` - [PR #23678](https://github.com/BerriAI/litellm/pull/23678) +- **Bump `tar` to 7.5.11 and `tornado` to 6.5.5** — addresses CVEs in transitive dependencies - [PR #23602](https://github.com/BerriAI/litellm/pull/23602) + +--- + +## Database / Proxy Operations + +- **Fix Prisma migrate deploy on pre-existing instances** — resolved multiple bugs in migration recovery logic: missing return in the P3018 idempotent error handler and unhandled exceptions in `_roll_back_migration` that caused silent failures even after successful recovery - [PR #23655](https://github.com/BerriAI/litellm/pull/23655) +- **Make DB migration failure exit opt-in** — proxy no longer exits on `prisma migrate deploy` failure by default; enable with `--enforce_prisma_migration_check` - [PR #23675](https://github.com/BerriAI/litellm/pull/23675) + +--- + +## New Contributors + +* @ryanh-ai made their first contribution in [PR #21542](https://github.com/BerriAI/litellm/pull/21542) +* @ryan-crabbe made their first contribution in [PR #23668](https://github.com/BerriAI/litellm/pull/23668) +* @Jah-yee made their first contribution in [PR #23525](https://github.com/BerriAI/litellm/pull/23525) +* @gambletan made their first contribution in [PR #23516](https://github.com/BerriAI/litellm/pull/23516) +* @awais786 made their first contribution in [PR #23183](https://github.com/BerriAI/litellm/pull/23183) +* @pradyyadav made their first contribution in [PR #23580](https://github.com/BerriAI/litellm/pull/23580) +* @xianzongxie-stripe made their first contribution in [PR #23492](https://github.com/BerriAI/litellm/pull/23492) +* @Harshit28j made their first contribution in [PR #23333](https://github.com/BerriAI/litellm/pull/23333) +* @codspeed-hq[bot] made their first contribution in [PR #23676](https://github.com/BerriAI/litellm/pull/23676) + +--- + +## Diff Summary + +## 03/16/2026 +* New Providers: 5 +* New Models / Updated Models: 116 new, 132 removed +* LLM API Endpoints: 5 +* Management Endpoints / UI: 11 +* AI Integrations: 2 +* Performance / Reliability: 5 +* Security: 3 +* Database / Proxy Operations: 2 + +--- + +## Full Changelog +[v1.82.0-stable...v1.82.3-stable](https://github.com/BerriAI/litellm/compare/v1.82.0-stable...v1.82.3-stable) diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 2f3302bb574..473edb5a6b2 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -471,9 +471,7 @@ "audio_speech": false, "moderations": false, "batches": false, - "rerank": false, - "a2a": false, - "interactions": false + "rerank": false } }, "chutes": { @@ -1486,12 +1484,12 @@ } }, "nebius": { - "display_name": "Nebius AI Studio (`nebius`)", + "display_name": "Nebius AI (`nebius`)", "url": "https://docs.litellm.ai/docs/providers/nebius", "endpoints": { "chat_completions": true, - "messages": true, - "responses": true, + "messages": false, + "responses": false, "embeddings": true, "image_generations": false, "audio_transcriptions": false, @@ -1499,8 +1497,7 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "a2a": false } }, "nlp_cloud": { @@ -2081,9 +2078,20 @@ }, "serper": { "display_name": "Serper (`serper`)", - "url": "https://docs.litellm.ai/docs/search/serper", + "url": "https://docs.litellm.ai/docs/providers/serper", "endpoints": { - "search": true + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true, + "a2a": false } }, "triton": { @@ -2265,12 +2273,12 @@ } }, "zai": { - "display_name": "Z.AI (Zhipu AI) (`zai`)", - "url": "https://docs.litellm.ai/docs/providers/zai", + "display_name": "ZAI (`zai`)", + "url": "https://docs.litellm.ai/docs/providers/openai_compatible", "endpoints": { "chat_completions": true, - "messages": true, - "responses": true, + "messages": false, + "responses": false, "embeddings": false, "image_generations": false, "audio_transcriptions": false, @@ -2278,8 +2286,7 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "a2a": false } }, "ragflow": { @@ -2551,23 +2558,25 @@ "audio_speech": false, "moderations": false, "batches": false, - "rerank": false + "rerank": false, + "a2a": false } }, - "charity_engine": { - "display_name": "Charity Engine (`charity_engine`)", - "url": "https://docs.litellm.ai/docs/providers/charity_engine", + "sagemaker_nova": { + "display_name": "AWS SageMaker Nova (`sagemaker_nova`)", + "url": "https://docs.litellm.ai/docs/providers/aws_sagemaker", "endpoints": { "chat_completions": true, - "messages": true, - "responses": true, + "messages": false, + "responses": false, "embeddings": false, "image_generations": false, "audio_transcriptions": false, "audio_speech": false, "moderations": false, "batches": false, - "rerank": false + "rerank": false, + "a2a": false } } }, From 245a3d2b2658fe1cfcfa2138961a51dfde593812 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Mon, 16 Mar 2026 22:26:45 -0700 Subject: [PATCH 4/4] =?UTF-8?q?Revert=20"docs:=20add=20v1.82.3=20release?= =?UTF-8?q?=20notes=20and=20update=20provider=5Fendpoints=5Fsupport?= =?UTF-8?q?=E2=80=A6"=20(#23817)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 966124966f83e6e1091ad08d9dfee6e341d3ad85. --- docs/my-website/release_notes/v1.82.3.md | 374 ----------------------- provider_endpoints_support.json | 55 ++-- 2 files changed, 23 insertions(+), 406 deletions(-) delete mode 100644 docs/my-website/release_notes/v1.82.3.md diff --git a/docs/my-website/release_notes/v1.82.3.md b/docs/my-website/release_notes/v1.82.3.md deleted file mode 100644 index 98351a30a27..00000000000 --- a/docs/my-website/release_notes/v1.82.3.md +++ /dev/null @@ -1,374 +0,0 @@ ---- -title: "v1.82.3 - Nebius AI, gpt-5.4, Gemini 3.x, FLUX Kontext, and 116 New Models" -slug: "v1-82-3" -date: 2026-03-16T00:00:00 -authors: - - name: Krrish Dholakia - title: CEO, LiteLLM - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: CTO, LiteLLM - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg -hide_table_of_contents: false ---- - -## Deploy this version - -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - - - - -``` showLineNumbers title="docker run litellm" -docker run \ --e STORE_MODEL_IN_DB=True \ --p 4000:4000 \ -ghcr.io/berriai/litellm:main-1.82.3-stable -``` - - - - -``` showLineNumbers title="pip install litellm" -pip install litellm==1.82.3 -``` - - - - -## Key Highlights - -- **Nebius AI — new provider** — [30 models across DeepSeek, Qwen, Llama, Mistral, NVIDIA, and BAAI available via Nebius AI cloud](../../docs/providers/nebius) - [PR #21542](https://github.com/BerriAI/litellm/pull/21542) -- **OpenAI gpt-5.4 / gpt-5.4-pro — day 0** — Full pricing and routing support for `gpt-5.4` (1M context, $2.50/$15.00) and `gpt-5.4-pro` ($30.00/$180.00) on OpenAI and Azure -- **Gemini 3.x models** — `gemini-3-flash-preview`, `gemini-3.1-pro-preview`, `gemini-3.1-flash-image-preview`, and `gemini-embedding-2-preview` added to cost map for Google AI and Vertex AI -- **FLUX Kontext image editing** — `flux-kontext-pro` and `flux-kontext-max` added to Black Forest Labs, alongside `flux-pro-1.0-fill` and `flux-pro-1.0-expand` for inpainting and outpainting -- **116 new models, 132 deprecated models cleaned up** — Major model map refresh including Mistral Magistral, Dashscope Qwen3 VL, xAI Grok via Azure AI, ZAI GLM-5, Serper Search; removal of OpenAI GPT-3.5/GPT-4 legacy variants, Gemini 1.5, and Vertex AI PaLM2 -- **SageMaker Nova provider** — [New `sagemaker_nova` provider for Amazon Nova models on SageMaker](../../docs/providers/aws_sagemaker) - [PR #21542](https://github.com/BerriAI/litellm/pull/21542) -- **Secret redaction in logs** — API keys, tokens, and credentials automatically scrubbed from all proxy log output. Enabled by default; opt out with `LITELLM_DISABLE_REDACT_SECRETS=true` - [PR #23668](https://github.com/BerriAI/litellm/pull/23668) -- **Streaming stability fix** — Critical fix for `RuntimeError: Cannot send a request, as the client has been closed.` crashes after ~1 hour in production - [PR #22926](https://github.com/BerriAI/litellm/pull/22926) - ---- - -## New Providers and Endpoints - -### New Providers (4 new providers) - -| Provider | Supported LiteLLM Endpoints | Description | -| -------- | --------------------------- | ----------- | -| [Nebius AI](../../docs/providers/nebius) (`nebius/`) | `/chat/completions`, `/embeddings` | EU-based AI cloud with 30+ open models — DeepSeek, Qwen3, Llama 3.1/3.3, NVIDIA Nemotron, BAAI embeddings | -| [ZAI](../../docs/providers/openai_compatible) (`zai/`) | `/chat/completions` | ZhipuAI GLM-5 models via ZAI cloud | -| [Black Forest Labs](../../docs/providers/black_forest_labs) (`black_forest_labs/`) | `/images/generations`, `/images/edits` | FLUX image generation and editing — Kontext Pro/Max, Pro 1.0 Fill/Expand | -| [Serper](../../docs/providers/serper) (`serper/`) | `/search` | Web search via Serper API | -| [SageMaker Nova](../../docs/providers/aws_sagemaker) (`sagemaker_nova/`) | `/chat/completions` | Amazon Nova models via SageMaker endpoint | - ---- - -## New Models / Updated Models - -#### New Model Support (116 new models) - -| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features | -| -------- | ----- | -------------- | ------------------- | -------------------- | -------- | -| OpenAI | `gpt-5.4` | 1.05M | $2.50 | $15.00 | chat, vision, tools, reasoning | -| OpenAI | `gpt-5.4-pro` | 1.05M | $30.00 | $180.00 | responses, vision, tools, reasoning | -| OpenAI | `gpt-5.3-chat-latest` | 128K | $1.75 | $14.00 | chat, vision, tools, reasoning | -| Azure OpenAI | `azure/gpt-5.4` | 1.05M | $2.50 | $15.00 | chat, vision, tools, reasoning | -| Azure OpenAI | `azure/gpt-5.4-pro` | 1.05M | $30.00 | $180.00 | responses, vision, tools, reasoning | -| Azure OpenAI | `azure/gpt-5.3-chat` | 128K | $1.75 | $14.00 | chat, vision, tools, reasoning | -| Google Gemini | `gemini/gemini-3-flash-preview` | 1M | $0.50 | $3.00 | chat, vision, tools, reasoning | -| Google Gemini | `gemini/gemini-3.1-pro-preview` | 1M | $2.00 | $12.00 | chat, vision, tools, reasoning | -| Google Gemini | `gemini/gemini-3.1-flash-image-preview` | 65K | $0.25 | $1.50 | image generation, vision | -| Google Gemini | `gemini/gemini-3.1-flash-lite-preview` | - | - | - | chat | -| Google Gemini | `gemini/gemini-3-pro-image-preview` | - | - | - | image generation | -| Google Gemini | `gemini/gemini-embedding-2-preview` | 8K | $0.20 | - | embeddings | -| Google Vertex AI | `vertex_ai/gemini-3-flash-preview` | - | - | - | chat | -| Google Vertex AI | `vertex_ai/gemini-3.1-pro-preview` | - | - | - | chat | -| Google Vertex AI | `vertex_ai/gemini-3.1-flash-lite-preview` | - | - | - | chat | -| Google Vertex AI | `vertex_ai/gemini-embedding-2-preview` | - | $0.20 | - | embeddings | -| Mistral | `mistral/magistral-medium-1-2-2509` | 40K | $2.00 | $5.00 | chat, tools, reasoning | -| Mistral | `mistral/magistral-small-1-2-2509` | 40K | $0.50 | $1.50 | chat, tools, reasoning | -| Mistral | `mistral/mistral-large-2512` | 262K | $0.50 | $1.50 | chat, vision, tools | -| Mistral | `mistral/mistral-medium-3-1-2508` | - | - | - | chat | -| Mistral | `mistral/mistral-small-3-2-2506` | - | - | - | chat | -| Mistral | `mistral/ministral-3-3b-2512` | - | - | - | chat | -| Mistral | `mistral/ministral-3-8b-2512` | - | - | - | chat | -| Mistral | `mistral/ministral-3-14b-2512` | - | - | - | chat | -| Black Forest Labs | `black_forest_labs/flux-kontext-pro` | - | - | - | image editing | -| Black Forest Labs | `black_forest_labs/flux-kontext-max` | - | - | - | image editing | -| Black Forest Labs | `black_forest_labs/flux-pro-1.0-fill` | - | - | - | image editing (inpaint) | -| Black Forest Labs | `black_forest_labs/flux-pro-1.0-expand` | - | - | - | image editing (outpaint) | -| Black Forest Labs | `black_forest_labs/flux-pro-1.1` | - | - | - | image generation | -| Black Forest Labs | `black_forest_labs/flux-pro-1.1-ultra` | - | - | - | image generation | -| Black Forest Labs | `black_forest_labs/flux-dev` | - | - | - | image generation | -| Black Forest Labs | `black_forest_labs/flux-pro` | - | - | - | image generation | -| Azure AI | `azure_ai/grok-4-1-fast-non-reasoning` | 131K | $0.20 | $0.50 | chat, tools | -| Azure AI | `azure_ai/grok-4-1-fast-reasoning` | 131K | $0.20 | $0.50 | chat, tools, reasoning | -| Azure AI | `azure_ai/mistral-document-ai-2512` | - | - | - | OCR | -| Dashscope | `dashscope/qwen3-next-80b-a3b-instruct` | 262K | $0.15 | $1.20 | chat | -| Dashscope | `dashscope/qwen3-next-80b-a3b-thinking` | 262K | $0.15 | $1.20 | chat, reasoning | -| Dashscope | `dashscope/qwen3-vl-235b-a22b-instruct` | 131K | $0.40 | $1.60 | chat, vision | -| Dashscope | `dashscope/qwen3-vl-235b-a22b-thinking` | 131K | $0.40 | $4.00 | chat, vision, reasoning | -| Dashscope | `dashscope/qwen3-vl-32b-instruct` | 131K | $0.16 | $0.64 | chat, vision | -| Dashscope | `dashscope/qwen3-vl-32b-thinking` | 131K | $0.16 | $2.87 | chat, vision, reasoning | -| Dashscope | `dashscope/qwen3-vl-plus` | 260K | - | - | chat, vision | -| Dashscope | `dashscope/qwen3.5-plus` | 992K | - | - | chat | -| Dashscope | `dashscope/qwen3-max-2026-01-23` | 258K | - | - | chat | -| Nebius AI | `nebius/deepseek-ai/DeepSeek-R1` | 128K | $0.80 | $2.40 | chat, reasoning | -| Nebius AI | `nebius/deepseek-ai/DeepSeek-R1-0528` | 164K | $0.80 | $2.40 | chat, reasoning | -| Nebius AI | `nebius/deepseek-ai/DeepSeek-V3` | 128K | $0.50 | $1.50 | chat | -| Nebius AI | `nebius/deepseek-ai/DeepSeek-V3-0324` | 128K | $0.50 | $1.50 | chat | -| Nebius AI | `nebius/deepseek-ai/DeepSeek-R1-Distill-Llama-70B` | 128K | $0.25 | $0.75 | chat | -| Nebius AI | `nebius/Qwen/Qwen3-235B-A22B` | 262K | $0.20 | $0.60 | chat | -| Nebius AI | `nebius/Qwen/Qwen3-32B` | 32K | $0.10 | $0.30 | chat | -| Nebius AI | `nebius/Qwen/Qwen3-30B-A3B` | 32K | $0.10 | $0.30 | chat | -| Nebius AI | `nebius/Qwen/Qwen3-14B` | 32K | $0.08 | $0.24 | chat | -| Nebius AI | `nebius/Qwen/Qwen3-4B` | 32K | $0.08 | $0.24 | chat | -| Nebius AI | `nebius/Qwen/QwQ-32B` | 32K | $0.15 | $0.45 | chat | -| Nebius AI | `nebius/Qwen/Qwen2.5-72B-Instruct` | 128K | $0.13 | $0.40 | chat | -| Nebius AI | `nebius/Qwen/Qwen2.5-32B-Instruct` | 128K | $0.06 | $0.20 | chat | -| Nebius AI | `nebius/Qwen/Qwen2.5-VL-72B-Instruct` | 131K | $0.13 | $0.40 | chat, vision | -| Nebius AI | `nebius/Qwen/Qwen2-VL-72B-Instruct` | 131K | $0.13 | $0.40 | chat, vision | -| Nebius AI | `nebius/Qwen/Qwen2-VL-7B-Instruct` | 131K | $0.02 | $0.06 | chat, vision | -| Nebius AI | `nebius/meta-llama/Meta-Llama-3.1-405B-Instruct` | 128K | $1.00 | $3.00 | chat | -| Nebius AI | `nebius/meta-llama/Meta-Llama-3.1-70B-Instruct` | 128K | $0.13 | $0.40 | chat | -| Nebius AI | `nebius/meta-llama/Meta-Llama-3.1-8B-Instruct` | 128K | $0.02 | $0.06 | chat | -| Nebius AI | `nebius/meta-llama/Llama-3.3-70B-Instruct` | 128K | $0.13 | $0.40 | chat | -| Nebius AI | `nebius/meta-llama/Llama-Guard-3-8B` | 128K | $0.02 | $0.06 | chat | -| Nebius AI | `nebius/nvidia/Llama-3.1-Nemotron-Ultra-253B-v1` | 128K | $0.60 | $1.80 | chat | -| Nebius AI | `nebius/nvidia/Llama-3.3-Nemotron-Super-49B-v1` | 131K | $0.10 | $0.40 | chat | -| Nebius AI | `nebius/NousResearch/Hermes-3-Llama-3.1-405B` | 128K | $1.00 | $3.00 | chat | -| Nebius AI | `nebius/google/gemma-3-27b-it` | 128K | $0.06 | $0.20 | chat | -| Nebius AI | `nebius/mistralai/Mistral-Nemo-Instruct-2407` | 128K | $0.04 | $0.12 | chat | -| Nebius AI | `nebius/Qwen/Qwen2.5-Coder-7B` | 32K | $0.01 | $0.03 | chat | -| Nebius AI | `nebius/BAAI/bge-en-icl` | 32K | $0.01 | - | embeddings | -| Nebius AI | `nebius/BAAI/bge-multilingual-gemma2` | 8K | $0.01 | - | embeddings | -| Nebius AI | `nebius/intfloat/e5-mistral-7b-instruct` | 32K | $0.01 | - | embeddings | -| AWS Bedrock | `mistral.devstral-2-123b` | 256K | $0.40 | $2.00 | chat, tools | -| AWS Bedrock | `zai.glm-4.7-flash` | 200K | $0.07 | $0.40 | chat, tools, reasoning | -| ZAI | `zai/glm-5` | 200K | $1.00 | $3.20 | chat, tools, reasoning | -| ZAI | `zai/glm-5-code` | 200K | $1.20 | $5.00 | chat, tools, reasoning | -| OpenRouter | `openrouter/anthropic/claude-sonnet-4.6` | - | - | - | chat | -| OpenRouter | `openrouter/google/gemini-3.1-pro-preview` | - | - | - | chat | -| OpenRouter | `openrouter/openai/gpt-5.1-codex-max` | - | - | - | chat | -| OpenRouter | `openrouter/qwen/qwen3-coder-plus` | - | - | - | chat | -| OpenRouter | `openrouter/qwen/qwen3.5-*` (5 models) | - | - | - | chat | -| OpenRouter | `openrouter/z-ai/glm-5` | - | - | - | chat | -| Together AI | `together_ai/Qwen/Qwen3.5-397B-A17B` | - | - | - | chat | -| Perplexity | `perplexity/pplx-embed-v1-0.6b` | 32K | $0.00 | - | embeddings | -| Perplexity | `perplexity/pplx-embed-v1-4b` | 32K | $0.03 | - | embeddings | -| Serper | `serper/search` | - | - | - | search | - -#### Updated Models - -- **[AWS Bedrock](../../docs/providers/bedrock)** - - Add `cache_read_input_token_cost` and `cache_creation_input_token_cost` to Bedrock-hosted Anthropic models (`claude-3-opus`, `claude-3-sonnet`, `claude-3-haiku`, and APAC/EU variants) — prompt caching is now tracked for cost estimation - - Rename `apac.anthropic.claude-sonnet-4-6` → `au.anthropic.claude-sonnet-4-6` to reflect correct regional identifier - -- **[Azure OpenAI](../../docs/providers/azure)** - - Add `supports_none_reasoning_effort` to all `gpt-5.1-chat`, `gpt-5.1-codex`, and `gpt-5.4` variants (global, EU, standard deployments) — allows passing `reasoning_effort: null` to disable reasoning - -- **[Azure OpenAI](../../docs/providers/azure)** — Removed deprecated models - - Remove `azure/gpt-35-turbo-0301` (deprecated 2025-02-13) - - Remove `azure/gpt-35-turbo-0613` (deprecated 2025-02-13) - -#### Features - -- **[OpenAI](../../docs/providers/openai)** - - Day 0 support for `gpt-5.4` and `gpt-5.4-pro` on OpenAI and Azure - -- **[Google Gemini](../../docs/providers/gemini)** - - Add Gemini 3.x model cost map entries — `gemini-3-flash-preview`, `gemini-3.1-pro-preview`, `gemini-3.1-flash-lite-preview`, `gemini-3-pro-image-preview`, `gemini-embedding-2-preview` - - Add Gemini 2.0 Flash and Flash Lite to cost map (re-added with updated pricing) - -- **[Google Vertex AI](../../docs/providers/vertex)** - - Add `gemini-3-flash-preview`, `gemini-3.1-flash-lite-preview`, `gemini-flash-experimental`, and `gemini-embedding-2-preview` to Vertex AI model cost map - -- **[Mistral](../../docs/providers/mistral)** - - Add Magistral reasoning models (`magistral-medium-1-2-2509`, `magistral-small-1-2-2509`) - - Add `mistral-large-2512`, `mistral-medium-3-1-2508`, `mistral-small-3-2-2506`, `ministral-3-*` variants - -- **[Dashscope / Qwen](../../docs/providers/dashscope)** - - Add Qwen3 VL multimodal models (`qwen3-vl-235b`, `qwen3-vl-32b` — instruct and thinking variants) - - Add `qwen3-next-80b-a3b` (instruct + thinking), `qwen3.5-plus`, `qwen3-max-2026-01-23` - -- **[Black Forest Labs](../../docs/providers/black_forest_labs)** - - Add FLUX Kontext image editing models (`flux-kontext-pro`, `flux-kontext-max`) - - Add FLUX Pro 1.0 Fill (inpainting) and Expand (outpainting) - - Add `flux-pro-1.1`, `flux-pro-1.1-ultra`, `flux-dev`, `flux-pro` - -- **[Azure AI](../../docs/providers/azure_ai)** - - Add xAI Grok models via Azure AI Foundry (`grok-4-1-fast-non-reasoning`, `grok-4-1-fast-reasoning`) - - Add Mistral Document AI (`mistral-document-ai-2512`) — OCR mode - -- **[AWS Bedrock](../../docs/providers/bedrock)** - - Add `mistral.devstral-2-123b` (256K context, tools) - - Add `zai.glm-4.7-flash` via Bedrock Converse (200K context, tools, reasoning) - -- **[SageMaker](../../docs/providers/aws_sagemaker)** - - Add `sagemaker_nova` provider for Amazon Nova models on SageMaker - [PR #21542](https://github.com/BerriAI/litellm/pull/21542) - -#### Deprecated / Removed Models - -**OpenAI** — Legacy models removed from cost map: -- `gpt-3.5-turbo-0301`, `gpt-3.5-turbo-0613`, `gpt-3.5-turbo-16k-0613` -- `gpt-4-0314`, `gpt-4-32k`, `gpt-4-32k-0314`, `gpt-4-32k-0613`, `gpt-4-1106-vision-preview`, `gpt-4-vision-preview` -- `gpt-4.5-preview`, `gpt-4.5-preview-2025-02-27` -- `gpt-4o-audio-preview-2024-10-01`, `gpt-4o-realtime-preview-2024-10-01` -- `o1-mini`, `o1-mini-2024-09-12`, `o1-preview`, `o1-preview-2024-09-12` - -**Google Gemini** — Gemini 1.5 and legacy 2.0 variants removed: -- All `gemini-1.5-*` variants (flash, flash-8b, pro, and dated versions) -- `gemini-2.0-flash-exp`, `gemini-2.0-pro-exp-02-05`, `gemini-2.5-flash-preview-04-17`, `gemini-2.5-flash-preview-05-20` - -**Google Vertex AI** — PaLM 2 / legacy models removed: -- All `chat-bison`, `text-bison`, `codechat-bison`, `code-bison`, `code-gecko` variants -- Gemini 1.0 Pro, 1.5 Flash/Pro, 2.0 Flash experimental, and preview variants - -**Perplexity** — Legacy Llama-sonar models removed: -- `llama-3.1-sonar-huge-128k-online`, `llama-3.1-sonar-large/small-128k-chat/online` - ---- - -## LLM API Endpoints - -#### Features - -- **[Responses API](../../docs/response_api)** - - Handle `response.failed`, `response.incomplete`, and `response.cancelled` terminal event types in background streaming — previously only `response.completed` was handled - [PR #23492](https://github.com/BerriAI/litellm/pull/23492) - -#### Bug Fixes - -- **[Anthropic](../../docs/providers/anthropic)** - - Preserve native tool format (web_search, bash, tool_search, etc.) when guardrails convert tools for the Anthropic Messages API - [PR #23526](https://github.com/BerriAI/litellm/pull/23526) - -- **[Moonshot / Kimi](../../docs/providers/openai_compatible)** - - Auto-fill `reasoning_content` for Moonshot Kimi reasoning models - [PR #23580](https://github.com/BerriAI/litellm/pull/23580) - -- **[HuggingFace](../../docs/providers/huggingface)** - - Forward `extra_headers` to HuggingFace embedding API - [PR #23525](https://github.com/BerriAI/litellm/pull/23525) - -- **General** - - Normalize `content_filtered` finish reason across providers - [PR #23564](https://github.com/BerriAI/litellm/pull/23564) - - Fix custom cost tracking on deployments for `/v1/messages` and `/v1/responses` - [PR #23647](https://github.com/BerriAI/litellm/pull/23647) - - Fix per-request custom pricing when `router_model_id` has no pricing data — now falls back to model name - ---- - -## Management Endpoints / UI - -#### Features - -- **Virtual Keys** - - Add Organization dropdown to Create/Edit Key form — `organization_id` is now a first-class field in Key Ownership - [PR #23595](https://github.com/BerriAI/litellm/pull/23595) - - Allow setting `organization_id` on `/key/update` — keys can be assigned or moved to a different organization after creation - [PR #23557](https://github.com/BerriAI/litellm/pull/23557) - -- **Internal Users** - - Add/Remove Team Membership directly from the Internal Users info page — includes searchable dropdown and role selector; no longer requires navigating to each team - [PR #23638](https://github.com/BerriAI/litellm/pull/23638) - -- **Default Team Settings** - - Modernize page to antd (consistent with rest of app) - [PR #23614](https://github.com/BerriAI/litellm/pull/23614) - - Fix: default team params (budget, duration, tpm, rpm, permissions) now correctly applied on `/team/new` - [PR #23614](https://github.com/BerriAI/litellm/pull/23614) - - Fix: settings persist across proxy restarts (`default_team_params` added to `LITELLM_SETTINGS_SAFE_DB_OVERRIDES`) - [PR #23614](https://github.com/BerriAI/litellm/pull/23614) - - Fix: resolved race condition in `_update_litellm_setting` where `get_config()` could overwrite freshly saved values - [PR #23614](https://github.com/BerriAI/litellm/pull/23614) - -- **Usage** - - Auto-paginate daily spend data — all entity views (teams, orgs, customers, tags, agents, users) fetch pages progressively with charts updating after each page - [PR #23622](https://github.com/BerriAI/litellm/pull/23622) - -- **Models / Cost** - - Azure Model Router cost breakdown in UI — show per-sub-model `additional_costs` from `hidden_params` in `CostBreakdownViewer` - [PR #23550](https://github.com/BerriAI/litellm/pull/23550) - -- **User Management** - - New `/user/info/v2` endpoint — scoped, paginated replacement for the existing god endpoint that caused memory and stability issues on large installs - [PR #23437](https://github.com/BerriAI/litellm/pull/23437) - -#### Bugs - -- Fix Tag list endpoint returning 500 due to invalid Prisma `group_by` kwargs - [PR #23606](https://github.com/BerriAI/litellm/pull/23606) -- Fix Team Admin getting 403 on `/user/filter/ui` when `scope_user_search_to_org` is enabled - [PR #23671](https://github.com/BerriAI/litellm/pull/23671) -- Fix Public Model Hub not showing config-defined models after save - [PR #23501](https://github.com/BerriAI/litellm/pull/23501) -- Fix fallback popup model dropdown z-index issue - [PR #23516](https://github.com/BerriAI/litellm/pull/23516) -- Fix double-counting bug in org/team key limit checks on `/key/update` - ---- - -## AI Integrations - -### Logging - -- **[Vantage](https://vantage.sh)** - - Add Vantage integration for FOCUS 1.2 CSV export — export LiteLLM proxy spend data as FinOps Open Cost & Usage Specification reports, with time-windowed filenames to prevent overwrites - [PR #23333](https://github.com/BerriAI/litellm/pull/23333) - -- **General** - - Fix silent metrics race condition causing metric collision across experiments - [PR #23542](https://github.com/BerriAI/litellm/pull/23542) - -### Guardrails - -No major guardrail changes in this release. - -### Prompt Management - -No major prompt management changes in this release. - -### Secret Managers - -No major secret manager changes in this release. - ---- - -## Performance / Loadbalancing / Reliability improvements - -- **Fix streaming crashes after ~1 hour** — `LLMClientCache._remove_key()` no longer calls `close()`/`aclose()` on evicted HTTP/SDK clients. In-flight requests were crashing with `RuntimeError: Cannot send a request, as the client has been closed.` after the 1-hour TTL expired. Cleanup now happens only at shutdown via `close_litellm_async_clients()` - [PR #22926](https://github.com/BerriAI/litellm/pull/22926) -- **Fix OOM / Prisma connection loss** on large installs — unbounded managed-object poll was exhausting Prisma connections after ~60–70 minutes on instances with 336K+ queued response rows - [PR #23472](https://github.com/BerriAI/litellm/pull/23472) -- **Centralize logging kwarg updates** — root cause fix migrating all logging updates to a single function, eliminating kwarg inconsistencies across logging paths - [PR #23659](https://github.com/BerriAI/litellm/pull/23659) -- **Fix tiktoken cache for non-root offline containers** — tiktoken cache now works correctly in offline environments running as non-root users - [PR #23498](https://github.com/BerriAI/litellm/pull/23498) -- **Add CodSpeed continuous performance benchmarks** — automated performance regression tracking on CI - [PR #23676](https://github.com/BerriAI/litellm/pull/23676) - ---- - -## Security - -- **Secret redaction in proxy logs** — Adds a `SecretRedactionFilter` to all LiteLLM loggers that scrubs API keys, tokens, and credentials from log messages, format args, exception tracebacks, and extra fields. Enabled by default; opt out with `LITELLM_DISABLE_REDACT_SECRETS=true` - [PR #23668](https://github.com/BerriAI/litellm/pull/23668), [PR #23667](https://github.com/BerriAI/litellm/pull/23667) -- **Bump PyJWT to `^2.12.0`** — addresses security vulnerability in `^2.10.1` - [PR #23678](https://github.com/BerriAI/litellm/pull/23678) -- **Bump `tar` to 7.5.11 and `tornado` to 6.5.5** — addresses CVEs in transitive dependencies - [PR #23602](https://github.com/BerriAI/litellm/pull/23602) - ---- - -## Database / Proxy Operations - -- **Fix Prisma migrate deploy on pre-existing instances** — resolved multiple bugs in migration recovery logic: missing return in the P3018 idempotent error handler and unhandled exceptions in `_roll_back_migration` that caused silent failures even after successful recovery - [PR #23655](https://github.com/BerriAI/litellm/pull/23655) -- **Make DB migration failure exit opt-in** — proxy no longer exits on `prisma migrate deploy` failure by default; enable with `--enforce_prisma_migration_check` - [PR #23675](https://github.com/BerriAI/litellm/pull/23675) - ---- - -## New Contributors - -* @ryanh-ai made their first contribution in [PR #21542](https://github.com/BerriAI/litellm/pull/21542) -* @ryan-crabbe made their first contribution in [PR #23668](https://github.com/BerriAI/litellm/pull/23668) -* @Jah-yee made their first contribution in [PR #23525](https://github.com/BerriAI/litellm/pull/23525) -* @gambletan made their first contribution in [PR #23516](https://github.com/BerriAI/litellm/pull/23516) -* @awais786 made their first contribution in [PR #23183](https://github.com/BerriAI/litellm/pull/23183) -* @pradyyadav made their first contribution in [PR #23580](https://github.com/BerriAI/litellm/pull/23580) -* @xianzongxie-stripe made their first contribution in [PR #23492](https://github.com/BerriAI/litellm/pull/23492) -* @Harshit28j made their first contribution in [PR #23333](https://github.com/BerriAI/litellm/pull/23333) -* @codspeed-hq[bot] made their first contribution in [PR #23676](https://github.com/BerriAI/litellm/pull/23676) - ---- - -## Diff Summary - -## 03/16/2026 -* New Providers: 5 -* New Models / Updated Models: 116 new, 132 removed -* LLM API Endpoints: 5 -* Management Endpoints / UI: 11 -* AI Integrations: 2 -* Performance / Reliability: 5 -* Security: 3 -* Database / Proxy Operations: 2 - ---- - -## Full Changelog -[v1.82.0-stable...v1.82.3-stable](https://github.com/BerriAI/litellm/compare/v1.82.0-stable...v1.82.3-stable) diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 473edb5a6b2..2f3302bb574 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -471,7 +471,9 @@ "audio_speech": false, "moderations": false, "batches": false, - "rerank": false + "rerank": false, + "a2a": false, + "interactions": false } }, "chutes": { @@ -1484,12 +1486,12 @@ } }, "nebius": { - "display_name": "Nebius AI (`nebius`)", + "display_name": "Nebius AI Studio (`nebius`)", "url": "https://docs.litellm.ai/docs/providers/nebius", "endpoints": { "chat_completions": true, - "messages": false, - "responses": false, + "messages": true, + "responses": true, "embeddings": true, "image_generations": false, "audio_transcriptions": false, @@ -1497,7 +1499,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": false + "a2a": true, + "interactions": true } }, "nlp_cloud": { @@ -2078,20 +2081,9 @@ }, "serper": { "display_name": "Serper (`serper`)", - "url": "https://docs.litellm.ai/docs/providers/serper", + "url": "https://docs.litellm.ai/docs/search/serper", "endpoints": { - "chat_completions": false, - "messages": false, - "responses": false, - "embeddings": false, - "image_generations": false, - "audio_transcriptions": false, - "audio_speech": false, - "moderations": false, - "batches": false, - "rerank": false, - "search": true, - "a2a": false + "search": true } }, "triton": { @@ -2273,12 +2265,12 @@ } }, "zai": { - "display_name": "ZAI (`zai`)", - "url": "https://docs.litellm.ai/docs/providers/openai_compatible", + "display_name": "Z.AI (Zhipu AI) (`zai`)", + "url": "https://docs.litellm.ai/docs/providers/zai", "endpoints": { "chat_completions": true, - "messages": false, - "responses": false, + "messages": true, + "responses": true, "embeddings": false, "image_generations": false, "audio_transcriptions": false, @@ -2286,7 +2278,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": false + "a2a": true, + "interactions": true } }, "ragflow": { @@ -2558,25 +2551,23 @@ "audio_speech": false, "moderations": false, "batches": false, - "rerank": false, - "a2a": false + "rerank": false } }, - "sagemaker_nova": { - "display_name": "AWS SageMaker Nova (`sagemaker_nova`)", - "url": "https://docs.litellm.ai/docs/providers/aws_sagemaker", + "charity_engine": { + "display_name": "Charity Engine (`charity_engine`)", + "url": "https://docs.litellm.ai/docs/providers/charity_engine", "endpoints": { "chat_completions": true, - "messages": false, - "responses": false, + "messages": true, + "responses": true, "embeddings": false, "image_generations": false, "audio_transcriptions": false, "audio_speech": false, "moderations": false, "batches": false, - "rerank": false, - "a2a": false + "rerank": false } } },